def getRaspberryPiImage():
# Get the image from raspbberry PI after loggin into it.
print("STATUS: Getting Image from Raspberry PI")
ip = '192.168.1.18'
port = 22
username = '******'
password = '******'
unique_filename = str(uuid.uuid4())
unique_filename = unique_filename + ".jpg"
cmd = 'cd imageCaptures/;raspistill -vf -hf -o ' + unique_filename
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(ip, port, username, password)
stdin, stdout, stderr = ssh.exec_command(cmd)
outlines = stdout.readlines()
resp = ''.join(outlines)
print(resp)
# SCPCLient takes a paramiko transport as its only argument
scp = SCPClient(ssh.get_transport())
file_to_get = '/home/pi/imageCaptures/' + unique_filename
scp.get(file_to_get)
scp.close()
ssh.close()
return unique_filename
def get_remote_data():
"""
Loads the data from local. Ment for testing
Returns: array of strings
Each string corresponds to a line
"""
host = 'bigdata-cluster1-03.virtual.uniandes.edu.co'
port = 22
usr = '******'
pwd = '969ba5d6f576c2c4377f2381d2829207'
ssh = SSHClient()
ssh.load_system_host_keys()
ssh.connect(host, port, usr, pwd)
# SCPCLient takes a paramiko transport as its only argument
scp = SCPClient(ssh.get_transport())
scp.get(remote_path='results/RA2/part-00000',
local_path='app1/RA/received_data/RA2/')
scp.close()
with open('app1/RA/received_data/RA2/part-00000') as f:
return (f.readlines())
def download_blacklisted(requester_id, list_of_files):
list_of_files = map(format_filename, list_of_files)
string_list = str(list_of_files)
zip_filename = "cases_%s_%s.zip" % (requester_id, get_formatted_date())
if not settings.DEBUG:
ssh = paramiko.SSHClient()
ssh.load_system_host_keys()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
private_key = paramiko.RSAKey.from_private_key_file(
settings.PRIVATE_KEY_FILENAME)
ssh.connect(settings.CAP_SERVER_TO_CONNECT_TO,
port=22,
pkey=private_key,
username='******')
ssh.exec_command("touch %s" % zip_filename)
logger.info("creating %s" % zip_filename)
stdin, stdout, stderr = ssh.exec_command(
'python cap_api_gzip_cases.py %s \"%s\"' %
(zip_filename, string_list))
if stderr.read():
raise Exception('Uh Oh! Something went wrong: %s', stderr.read())
scp_client = SCPClient(ssh.get_transport())
scp_client.get("%s" % zip_filename)
logger.info("downloading %s" % zip_filename)
scp_client.close()
move_casezip(zip_filename)
return zip_filename
def _queue_worker(rc):
# multithreaded file puller, takes tuples of remote, local, item, items_done
# pulls the files and then updates the progress meter
jenkins_host = composite['jenkins_host']
client = rc.ssh_client
client.connect(jenkins_host, username=credentials['jenkins-result']['username'],
password=credentials['jenkins-result']['password'],
timeout=10,
allow_agent=False,
look_for_keys=False,
gss_auth=False)
scp = None
while True:
source, destination, item, items_done = rc._queue.get()
destination = local(destination)
destination_dir = local(destination.dirname)
destination_dir.ensure(dir=True)
if not destination.check():
if scp is None:
scp = SCPClient(client.get_transport())
try:
scp.get(source, destination.strpath)
except SCPException:
# remote destination didn't exist
pass
except (SSHException, socket.timeout):
# SSH blew up :(
rc._queue.put((source, destination, item, items_done))
rc._queue.task_done()
continue
rc._progress_update(item, items_done)
rc._queue.task_done()
def register_device():
get_serial_chasis_num()
ssh = createClient('10.65.126.165', '22', 'tester', 'v1ptela0212')
scp = SCPClient(ssh.get_transport())
scp.put('nkhulbe92.csv', '/home/tester/nkhulbe92.csv')
stdin, stdout, stderr = ssh.exec_command(
'python /home/tester/vtest/addons/zprov/signing.py --csvfile nkhulbe92.csv --orgname sdwan-blr-master --output nkhulbe92.viptela'
)
print(stdout.readlines())
scp.get('/home/tester/nkhulbe92.viptela', 'nkhulbe92.viptela')
ssh.close()
del ssh, stdin, stdout, stderr
print(
"---------------------device registered with vmanage----------------------------------"
)
print(
"---------------------------loading file to vmanage-------------------------------"
)
time.sleep(5)
payload = {"validity": "valid", "upload": "true"}
start_session.post_request(payload)
def transfer_file(self, pull=False):
if pull is False:
if not self.local_file_exists():
raise FileTransferError(
'Could not transfer file. Local file doesn\'t exist.')
if not self.enough_remote_space():
raise FileTransferError(
'Could not transfer file. Not enough space on device.')
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(hostname=self.device.host,
username=self.device.username,
password=self.device.password,
port=self.port,
allow_agent=False,
look_for_keys=False)
scp = SCPClient(ssh.get_transport(), socket_timeout=30.0)
try:
if pull:
scp.get(self.remote, self.local)
else:
scp.put(self.local, self.remote)
except Exception as e:
raise FileTransferError
finally:
scp.close()
return True
def transfer_file(self,
hostname=None,
username=None,
password=None,
pull=False):
"""Transfer the file to the remote device over SCP.
Note:
If any arguments are omitted, the corresponding attributes
of ``self.device`` will be used.
Args:
hostname (str): OPTIONAL - The name or
IP address of the remote device.
username (str): OPTIONAL - The SSH username
for the remote device.
password (str): OPTIONAL - The SSH password
for the remote device.
Returns:
True if successful.
Raises:
FileTransferError: if the transfer isn't successful.
"""
if pull is False:
if not self.local_file_exists():
raise FileTransferError(
'Could not transfer file. Local file doesn\'t exist.')
if not self.enough_space():
raise FileTransferError(
'Could not transfer file. Not enough space on device.')
hostname = hostname or self.device.ip
username = username or self.device.username
password = password or self.device.password
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(hostname=hostname,
username=username,
password=password,
port=self.port,
allow_agent=False,
look_for_keys=False)
scp = SCPClient(ssh.get_transport())
try:
if pull:
scp.get(self.dst, self.src)
else:
scp.put(self.src, self.dst)
except:
raise FileTransferError(
'Could not transfer file. There was an error during transfer.')
finally:
scp.close()
return True
def get_rawdata(server, month, tmpfile):
serv = server.objects.get(name=server)
ssh = SSHClient()
ssh.load_system_host_keys()
ssh.connect(serv.ipaddress, username=serv.username, password=serv.password)
client = SCPClient(ssh.get_transport())
client.get(serv.path + '/volume.' + month+ '.csv', tmpfile)
def grab_wheeler(sdir='.',
snum=0,
snap_type=0,
destination='./scratch',
other_filename=''):
''' snap_type=0 grabs the snapshot produced by GIZMO
snap_type=1 grabs the snapshot created with grain_density_from_sanpshot.py
other_filename if specified, grabs that file'''
ssh = paramiko.SSHClient()
passphrase = '0206Teodora205'
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect('wheeler.caltech.edu', username='******', password=passphrase, \
key_filename='C:\Users\Stefi\AppData\Local\lxss\home\slm86\.ssh\id_rsa')
scp = SCPClient(ssh.get_transport())
if (snap_type == 0):
ss = snap_ext(snum, four_char=0)
filename = sdir + '/snapshot_' + ss + '.hdf5'
else:
ss = snap_ext(snum, four_char=1)
filename = sdir + '/dust_snap_' + ss + '.h5'
if (len(other_filename) > 0): filename = sdir + '/' + other_filename
scp.get(filename, destination)
print('File ', snum, ' copied')
ssh.close()
def get_csv(self, cursor, uid, ids, context=None):
if isinstance(ids, (tuple, list)):
ids = ids[0]
lot_obj = self.pool.get('som.infoenergia.lot.enviament')
lot = lot_obj.browse(cursor, uid, ids)
try:
ssh = get_ssh_connection()
output_dir = config.get("infoenergia_report_download_dir",
"/tmp/test_shera/reports")
csv_path_file = context.get('path_csv', '')
output_filepath = os.path.join(output_dir, 'Enviaments.csv')
scp = SCPClient(ssh.get_transport())
scp.get(csv_path_file, output_filepath)
lot.create_enviaments_from_csv_file(output_filepath, context)
self._attach_csv(cursor, uid, ids, output_filepath)
self.add_info_line(cursor, uid, ids,
'CSV descarregat correctament')
except Exception as e:
message = 'ERROR ' + str(e)
self.add_info_line(cursor, uid, ids, message)
def scpGet(self, host, source, user=None, localDir=None):
"""
get a remote file and copy in localDir(default is cache Dir) . similar as 'scp user @ remote:file localDir'
the retruned file name is added one timestamp suffix with a underscore like filename_20160627092530123456
The timestamp has format of 4 year + 2 month + 2 day + 2 hour + 2 minute + 2 second + 6 Microsecond
:param str host: IP address of the host
:param str source: the remote file in lab which should be copy to localDir
:param str user: the user to login on remote. default is 'root'
:param str localDir: the full path in local to store the file, default is None, if None, use cache dir
:return: the path of the local file
:rtype: str
Note:
At least one getClient() must have been executed before calling this proc
i.e. connection is already stored in self._connections
>>> sshManager.scpGet(host, source)
"""
userSsh = 'root' if user is None else user
client = self.getClient(host, user=userSsh)
scpClient = SCPClient(client.get_transport(), socket_timeout=float(self.__class__.defaultHeartbeat))
if not localDir:
localDir = SshManager.cacheDir + os.sep + host
if not os.path.isdir(localDir):
os.mkdir(localDir)
_localPath = localDir + os.sep + os.path.basename(source) + "_" + datetime.now().strftime('%Y%m%d%H%M%S%f')
LOGGER.trace("scpClient.get(%s, %s) from %s@%s", source, _localPath,
userSsh, host)
scpClient.get(source, _localPath)
return _localPath
def log_handler_copy_others(env):
if re.findall('cit', env, flags=re.IGNORECASE):
host = cit_ip_address
elif re.findall('sit', env, flags=re.IGNORECASE):
host = sit_ip_address
elif re.findall('dev', env, flags=re.IGNORECASE):
host = dev2_ip_address
else:
logger.err('输入环境名称错误！')
raise ValueError
# 决定要要Copy多少个目录下的log
log_paths = [log_base_abs_path, log_data_sync_abs_path, log_labor_abs_path, log_supply_abs_path,
log_sales_abs_path, log_transportation_abs_path, log_warehouse_abs_path]
# 得到本地所有文件列表, 后面添加一个空格, 然后转换为set格式去和本地的文件比较，只去下载不同的的文件
ssh_executor = __ssh_connect_to_server(host)
# 通过SCP协议下载文件
scp = SCPClient(ssh_executor.get_transport())
# 记录那个目录下,需要分析那些Log, 把文件名称作为Key, 文件夹路径作为value，因为key必须唯一
log_to_analysis = {}
for path in log_paths:
logger.debug('当前处理的目录为:')
logger.debug(local_path + path)
files = os.listdir(local_path+path)
logger.debug('当前目录下的所有文件')
files_need_to_filter = []
for fi in files:
# 这个工具暂时不处理catalina.out文件，后面会开发一个新的方法通过记录行数来分析catalina.out文件内部增量的工作分析
if fi.find('catalina') == 0 and fi.find('catalina.out') == -1:
files_need_to_filter.append(fi + ' ') # 添加空格在末尾,否则set.difference会查找不到
files_need_to_filter_set = set(files_need_to_filter)
logger.debug('#' * 10)
logger.debug('当前操作的路径为%s' % path)
# 拼接命令,得到所有copy目录内的所有文件名,然后找出增量项，再进行scp copy
ssh_std_in, ssh_stdout, ssh_stderr = ssh_executor.exec_command('cd %s;ls' % path)
files_under_path = ssh_stdout.readlines()
files_under_path_ = []
for f in files_under_path:
if f.find('catalina') == 0 and f.find('catalina.out') == -1:
files_under_path_.append(f.replace("\n", ' '))
files_under_path_set = set(files_under_path_)
logger.debug('在路径下的所有文件名为：')
logger.debug(files_under_path)
files_need_to_copy_in_current_folder = files_under_path_set.difference(files_need_to_filter_set)
logger.debug('需要copy的文件为')
logger.debug(files_need_to_copy_in_current_folder)
for file_to_copy in files_need_to_copy_in_current_folder:
logger.debug('Scp Get的目标和本地目录分别为：')
logger.debug('%s%s' % (path, file_to_copy), '%s%s' % (local_path, path))
# [:-1]拿掉文件名称末尾的空格, 否则SCP-get会执行失败
scp.get('%s%s' % (path, file_to_copy[:-1]), '%s%s' % (local_path, path))
log_to_analysis.update({file_to_copy[:-1]: '%s%s' % (local_path, path)})
ssh_executor.close()
# 确保后面分析的只是增量的LOG, catalina.out除外
return log_to_analysis
pass
def download_file(input_path, output_path):
"""
Download file from the first worker node
specified during configuration.
Parameters
----------
input_path : str
Path to input file
output_path : str
Path to output file
"""
cloud_config = yaml.safe_load(open(
_CONF["general_config"]).read())["cloud"]
remote_username = cloud_config.get("remote_username")
head_ip = cloud_config.get("nodes")[0]
if not head_ip:
raise MC2ClientConfigError(
"Remote orchestrator IP not set. Run oc.create_cluster() \
to launch VMs and configure IPs automatically or explicitly set it in the user YAML."
)
ssh = _createSSHClient(head_ip, 22, remote_username)
scp = SCPClient(ssh.get_transport())
scp.get(input_path, output_path, recursive=True)
def pull(self, src_path, dest_path, pull_timeout=None):
"""
Pull file from dut
:param src_path: str
:param src_path: Source path of file on DUT
:type dest_path: str
:param dest_path: Destination path on HOST
:type pull_timeout: int
:param pull_timeout: timeout to pull the file
:rtype: tuple
:return: Boolean indicating operation succeed or not, message
"""
verdict = False
msg = "fail to pull file %s onto %s, no opened connection" % (src_path, dest_path)
if self.__protocol is not None:
try:
scp_client = SCPClient(self.__protocol.get_transport())
scp_client.get(src_path, dest_path, recursive=True, preserve_times=True)
verdict = True
msg = "Pulled file %s onto %s" % (src_path, dest_path)
except Exception as ex:
msg = "Error happen when pulling file %s onto %s :%s" % (src_path, dest_path, str(ex))
return verdict, msg
def copy_testout_junit(self, options, conf_dict):
""" Copy junit file from first node to Jenkins slave """
master = self.existing_nodes[0]
ssh_c = SSHClient(hostname = master, username = \
self.username, password = self.password)
remote_file = conf_dict['pytest']['pytest_junit_loc']
scp = SCPClient(ssh_c.get_transport())
scp.get(remote_file)
# Executing script that would convert the test case name in junit
# output xml to use docstrings.
self.junit_convert_script = conf_dict['pytest']['junit_convert_script']
self.junit_new_out = conf_dict['pytest']['pytest_new_junit_loc']
self.team = conf_dict['pytest']['team']
tests_base = conf_dict['pytest']['tests_base']
self.package = tests_base.replace("-", "_")
tests_to_run = conf_dict['pytest']['tests_to_run']
self.test_suite = tests_to_run.rsplit('/', 1)[1]
massage_junit = "python " + self.junit_convert_script + " -i " + \
remote_file + " -o " + self.junit_new_out + " -t " + self.team + \
" -p " + self.package + " -s " + self.test_suite
stdin, stdout, stderr = ssh_c.ExecuteCmd(massage_junit)
for line in stdout.read().splitlines(): logger.log.info(line)
scp = SCPClient(ssh_c.get_transport())
scp.get(self.junit_new_out)
def get_magic_wllt(ssh, passphrase):
if request_frm_bck_up():
command = "gpg --batch --passphrase passphrase --decrypt "+ WALLET_NAME_E +" | cat " + WALLET_NAME
try:
#stdin, stdout, stderr = ssh.exec_command(command)
#lines = stdout.readlines()
#if lines:
# print('lines:',lines)
scp = SCPClient(ssh.get_transport())
scp.get(REMOTE_WALLET_PATH + "/" + WALLET_NAME, LOCAL_WALLET_PATH)
scp.close()
result = "Successfully retrieved wallet from device."
except Exception as e:
print("Couldn\'t connect, connection error:", e)
result = "Failed to retrieve wallet from device."
sys.exit() # Quit
ssh.close()
print(result)
return result
def ocr_tts(img_path):
#initial settings
client = paramiko.SSHClient()
client.load_system_host_keys()
client.set_missing_host_key_policy(AutoAddPolicy())
client.connect(hostname='',username='',password='')
#hostname : IP of your PC , username : username of your PC, password : paasword of your PC
scp = SCPClient(client.get_transport())
client.exec_command('del RC.txt && del check.txt && speech.mp3') #initialize files
scp.put(img_path, img_path)
client.exec_command('tesseract {} RC -l kor55+kor100'.format(img_path)) #run OCR
print('Recognizing Letters...')
stdin, stdout, stderr = client.exec_command('type RC.txt')
while stdout.readlines() == []:
stdin, stdout, stderr = client.exec_command('type RC.txt') #check if OCR is completed
print('Letters has been recognized')
client.exec_command('python gTTSpc.py')
print('Producing a voice file...')
stdin, stdout, stderr = client.exec_command('if exist check.txt echo DONE!')
while stdout.readlines() == []:
stdin, stdout, stderr = client.exec_command('if exist check.txt echo DONE!') #check if gTTS is completed
print('The file has been produced')
scp.get('speech.mp3')
scp.close()
class SCPConnection(object):
def __init__(self, host, username, password, port=22):
"""Initialize and setup connection"""
self.scp_open = False
# open SSH Transport stream
self.transport = paramiko.Transport((host, port))
if not self.scp_open:
self.transport.connect(username=username, password=password)
self.scp_open = True
def put(self, local_file, remote_file):
"""Creates an SCP sessions then copies the local file to
the remote file"""
self.scp = SCPClient(self.transport)
self.scp.put(local_file, remote_file)
def get(self, local_file, remote_file):
"""Creates an SCP sessions then copies the remote file to
the local file"""
self.scp = SCPClient(self.transport)
self.scp.get(remote_file, local_file)
def close(self):
"""
Close SCP connection and SSH connection
"""
if self.scp_open:
self.transport.close()
self.scp_open = False
def scpfile(self, remote_file, local_path):
""" Copy a file from the host to a local directory
"""
logger.debug('scpfile: remote_file %s local_path %s',
remote_file, local_path)
try:
sshclient = self.ssh_connect()
except Exception as e:
logger.error('scpfile: Exception: %r', e)
raise
scp = SCPClient(sshclient.get_transport(), buff_size=1024*1024,
socket_timeout=30.0, progress=self.progress_cb)
try:
scp.get(remote_file, local_path, preserve_times=True)
self.ssh_close(sshclient)
except Exception as e:
logger.error('scpfile: Exception: %r', e)
self.ssh_close(sshclient)
raise
filename = os.path.basename(remote_file)
local_pathname = os.path.join(local_path, filename)
os.chmod(local_pathname, stat.S_IRUSR | stat.S_IRGRP |
stat.S_IROTH | stat.S_IWUSR | stat.S_IWGRP |
stat.S_IWOTH | stat.S_IWRITE | stat.S_IREAD)
return local_pathname
def get_file(fname):
ssh = SSHClient()
ssh.load_system_host_keys()
ssh.connect(file_server)
scp = SCPClient(ssh.get_transport())
scp.get(fname)
scp.close()
def download_test(self, filename, recursive, destination=None,
expected_win=[], expected_posix=[]):
# Make a temporary directory
temp = tempfile.mkdtemp(prefix='scp-py_test_')
# Add some unicode in the path
if WINDOWS:
if isinstance(temp, bytes):
temp = temp.decode(sys.getfilesystemencoding())
temp_in = os.path.join(temp, u'cl\xE9')
else:
if not isinstance(temp, bytes):
temp = temp.encode('utf-8')
temp_in = os.path.join(temp, b'cl\xC3\xA9')
previous = os.getcwd()
os.mkdir(temp_in)
os.chdir(temp_in)
try:
scp = SCPClient(self.ssh.get_transport())
scp.get(filename, destination if destination is not None else u'.',
preserve_times=True, recursive=recursive)
actual = []
def listdir(path, fpath):
for name in os.listdir(fpath):
fname = os.path.join(fpath, name)
actual.append(os.path.join(path, name))
if os.path.isdir(fname):
listdir(name, fname)
listdir(u'' if WINDOWS else b'',
u'.' if WINDOWS else b'.')
self.assertEqual(normalize_paths(actual),
set(expected_win if WINDOWS else expected_posix))
finally:
os.chdir(previous)
shutil.rmtree(temp)
class SSH:
def __init__(self, ipaddr, username, password, port="22"):
self.ipaddr = ipaddr
self.username = username
self.password = password
self.port = port
if not paramiko:
raise ImportError(name='Error importing paramiko.')
#def connect(self):
self.ssh = paramiko.SSHClient()
self.ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
self.ssh.connect(hostname=self.ipaddr,
port=self.port,
username=self.username,
password=self.password)
def disconnect(self):
return (self.ssh.close())
def send_command(self, command):
stdin, stdout, stderr = self.ssh.exec_command(command)
return (stdout.readlines())
def copy_file(self, remote, local):
self.scp = SCPClient(self.ssh.get_transport())
self.scp.get(remote, local)
class SSHConnection:
"""docstring for SCPConnection"""
def __init__(self, controller, ip, uName, uPass):
try:
self.ssh = SSHClient()
self.ssh.load_system_host_keys()
self.ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
self.ssh.connect(hostname=ip,
port=22,
username=uName,
password=uPass,
timeout=2)
except Exception as ex:
controller.error(ex)
def fileTrans(self, controller, srcDir, dstDir):
try:
self.scp = SCPClient(self.ssh.get_transport())
self.scp.get(srcDir, dstDir)
self.scp.close()
self.ssh.close()
except Exception as ex:
controller.error(ex)
def connect_clients(hostname, username, password, cmd):
try:
date_now = time.strftime('%Y_%m_%d_%H_%M_%S',
time.localtime(time.time()))
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect(hostname=hostname, username=username, password=password)
if upload_flag != 0 or download_flag != 0:
scpclient = SCPClient(client.get_transport(), socket_timeout=15.0)
print(cmd)
if upload_flag == 1:
scpclient.put(cmd, remote_path)
elif download_flag == 1:
scpclient.get(cmd, os.path.abspath("."))
else:
shell = client.invoke_shell()
shell.sendall(cmd + "\n")
shell.sendall("exit\n")
except Exception as e:
print("[%s] %s target failed, the reason is %s" %
(date_now, hostname, str(e)))
else:
print("[%s] %s target success" % (date_now, hostname))
finally:
client.close()
def syst(IP, ACC, PASS):
# 找出錯誤log並儲存到 \IP_ERR_LOG中
try:
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect(IP, username=ACC, password=PASS)
_, stdout, _ = client.exec_command("cd /var/log; ls")
lst = [
line.replace('\n', '') for line in stdout.readlines()
if line[:8] == "messages"
]
log = ""
scp = SCPClient(client.get_transport())
for line in lst:
scp.get("/var/log/" + line, PATH + "\\" + IP + "_log\\" + line)
if line[-2:] == "gz":
with gzip.open(IP + "_log\\" + line, "rb") as f_in:
log += f_in.read().decode()
else:
with open(IP + "_log\\" + line, "rb") as f_in:
log += f_in.read().decode()
except Exception as e:
print(e)
return
def get_file(remote_file, local_path, ip, username, password, logger=None):
# Получает с удаленной машины файл remote_file с помощью scp и сохраняет его в local_path.
if local_path[len(local_path) - 1] != '/': local_path += '/'
ssh = SSHClient()
ssh.set_missing_host_key_policy(AutoAddPolicy())
ssh.load_system_host_keys()
if logger is not None: logger.info("SCP GET: connecting to %s" % (ip))
try:
ssh.connect(ip, username=username, password=password)
except:
if logger is not None: logger.info("SCP GET: failed to connect to %s" % (ip))
return False
else:
if logger is not None: logger.info("SCP GET: connected to %s" % (ip))
try:
if logger is not None: logger.info("SCP GET: retrieving file %s" % (remote_file))
scp = SCPClient(ssh.get_transport())
scp.get(remote_file, local_path)
except:
if logger is not None: logger.error("SCP GET: error: failed to retrieve file %s" % (remote_file))
ssh.close()
return False
else:
if logger is not None: logger.info("SCP GET: file saved to %s folder" % (local_path))
ssh.close()
return True
def _queue_worker(rc):
# multithreaded file puller, takes tuples of remote, local, item, items_done
# pulls the files and then updates the progress meter
jenkins_host = composite['jenkins_host']
client = rc.ssh_client
client.connect(jenkins_host,
username=credentials['jenkins-result']['username'],
password=credentials['jenkins-result']['password'],
timeout=10,
allow_agent=False,
look_for_keys=False,
gss_auth=False)
scp = None
while True:
source, destination, item, items_done = rc._queue.get()
destination = local(destination)
destination_dir = local(destination.dirname)
destination_dir.ensure(dir=True)
if not destination.check():
if scp is None:
scp = SCPClient(client.get_transport())
try:
scp.get(source, destination.strpath)
except SCPException:
# remote destination didn't exist
pass
except (SSHException, socket.timeout):
# SSH blew up :(
rc._queue.put((source, destination, item, items_done))
rc._queue.task_done()
continue
rc._progress_update(item, items_done)
rc._queue.task_done()
def scp_get_files (self):
""""""
if self.conf != None:
'''SET CONNECTION PREFS'''
# self.set_connection_params()
'''SET SSH CLIENT AND TRANSPORT'''
self.create_Paramiko_SSHClient()
scp = SCPClient(self.ssh_client.get_transport())
self.logger.info("<== SCP CLASS STARTED GETTING files ==> ")
scp.get(remote_path = self.ssh_scp_content_location,
local_path = self.ssh_target_dir)
scp.close()
self.logger.info("<== SCP CLASS HAS GOT ==> ")
'''Check if it is actually here'''
f_name = self.str_split_get_pop_elem(str_in = self.ssh_scp_content_location,
delim_in = '/',
which_elem = 'LAST'
)
if self.conf.syst == 'Windows':
copied_file = Path(self.ssh_target_dir + '\\' + f_name)
elif self.conf.syst == 'Linux':
copied_file = Path(self.ssh_target_dir + '/' + f_name)
if copied_file.is_file():
self.logger.debug("<== File with name ==> ")
self.logger.debug("<==" + str(copied_file) + "==> ")
self.logger.debug("<== seems to be in place Now the question is it the right one ---> ")
else:
raise ValueError("Configuration file is not PRESENT in the CLASS!!!")
return
class FilePuller(AnvilTool):
scp_client = SCPClient
#####
def __init__(self, config=ConfigWrapper):
super(FilePuller, self).__init__(config)
self.cfg.create_ssh_client()
self.scp_client = SCPClient(self.cfg.ssh_client.get_transport())
#####
def pull_file(self, remote_filename="", local_filename=""):
if os.path.exists(local_filename):
os.remove(local_filename)
print "\nRemote file: {}\nLocal destination: {}".format(
remote_filename, local_filename)
self.scp_client.get(remote_filename, local_filename)
if os.path.exists(local_filename):
print "File pulled!"
else:
print "Error pulling file."
#####
def get_result(self):
remote_filename = "{}{}{}".format(self.cfg.dest_path,
self.cfg.config.remote_result_dir,
self.cfg.config.remote_result_file)
local_filename = "{}{}/{}".format(self.cfg.local_path, ANVIL_DIR_NAME,
self.cfg.config.remote_result_file)
self.pull_file(remote_filename, local_filename)
return local_filename
class FileManager:
def __init__(self, host="10.4.20.69", port=22, user_name="", passwd=""):
self.host = host
self.port = port
self.user = user_name
self.passwd = passwd
self.ssh_client = paramiko.SSHClient()
self.ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy)
self.ssh_client.connect(host, port, user_name, passwd)
self.scpclient = SCPClient(self.ssh_client.get_transport(),
socket_timeout=15.0)
def upload_file(self, local_path, remote_path):
try:
self.scpclient.put(local_path, remote_path)
except FileNotFoundError as e:
logger.error("Can't find local file at: " + local_path)
logger.error(e)
else:
logger.info("Successful upload 1 file! ")
def get_file(self, remote_path, local_path):
try:
self.scpclient.get(remote_path, local_path)
except FileNotFoundError as e:
logger.error("Can't find remote file at: " + remote_path)
logger.error(e)
else:
logger.info("Successful download 1 file! ")
def close(self):
self.ssh_client.close()
def get_file_from_scp_service(self,
file_name,
remote_path="/A8/",
local_path="D:\python_eg"):
ssh_client = paramiko.SSHClient()
ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy)
ssh_client.connect(self.__host, self.__port, self.__username,
self.__password)
scpclient = SCPClient(ssh_client.get_transport(), socket_timeout=15.0)
# local_path = file_path + "\\" + img_name
file_path_lo = local_path
file_path_re = remote_path + '/' + file_name
#file_path_re = remote_path
print(file_path_lo)
print(file_path_re)
try:
# print(local_path)
# scpclient.put(localpath, remotepath) # 上传到服务器指定文件
scpclient.get(file_path_re, file_path_lo) # 从服务器中获取文件
except FileNotFoundError as e:
print(e)
print("system could not find the specified file" + local_path)
result = "system could not find the specified file" + local_path
else:
print("File downloaded successfully")
result = "File downloaded successfully"
ssh_client.close()
return result
def scp_copy(ssh_client, source_file, to_dir):
scp = SCPClient(ssh_client.get_transport())
scp.get(source_file, local_path=to_dir)
scp.close()
def getLinuxUploadSourceFile(linux_ip, linux_user,linux_passwd,local_dir,test_user,upload_type, sourceUploadFile):
Host = linux_ip
user = linux_user
passwd = linux_passwd
port =22
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(Host,port,user, passwd)
(stdin, stdout, stderr) = ssh.exec_command("find /usr/local/7thonline/7thOnline/data/temp -name *"+test_user+"* -mmin -1 -type d")
dir = stdout.readlines();
remotepath=''
target_file=''
for dir_tmp in dir:
(stdin, stdout, stderr) = ssh.exec_command("ls "+dir_tmp.strip('\n'))
file_tmp = stdout.readlines();
file = [tmp for tmp in file_tmp if (tmp.count(sourceUploadFile)>0)]
if(len(file)>0):
remotepath=dir_tmp.strip('\n')+"/"+ file[0].strip('\n')
target_file=file[0].strip('\n')
if (len(remotepath)>0):
scpclient = SCPClient(ssh.get_transport(), socket_timeout=15.0)
localpath=local_dir+target_file
scpclient.get(remotepath, localpath)
ssh.close()
return localpath
elif (len(dir) >0):
ssh.close()
print "No Source Upload File!"
else:
ssh.close()
raise Exception, "get Source Upload File Fail!"
def copy_file(local_file_path, dest_path, host=cfg.HOST_CONF['HOST'], port=cfg.HOST_CONF['PORT'], user=cfg.HOST_CONF['USER'], password=cfg.HOST_CONF['PASSWORD'], mode = 'PUT'):
transport = None
if not path.exists(local_file_path):
if mode is 'GET':
mkdir(local_file_path)
else:
print "Local file path doesnt exist %s" %(local_file_path)
paramiko.util.log_to_file(cfg.PARAMIKO_LOG)
transport = paramiko.Transport((host, port))
try:
transport.connect(username=user, password=password)
transport.get_exception()
scp = SCPClient(transport)
except:
print "Error while connecting to host %s:%s" %(host, port)
#scp.put('test_eth.sh', 'test_eth_1.sh')
#scp.get('test_eth_1.sh')
if mode is 'PUT':
#print "copying file from %s to %s on host %s " %(local_file_path, dest_path, host)
scp.put(local_file_path, dest_path)
else:
#print scp.get(dest_path,local_file_path, recursive=True)
scp.get(dest_path, recursive=True)
#print scp.get(dest_path, local_path="path_to_directory where store this directory", recursive= True)
scp.close()
def get_file(remote_file, local_path, ip, username, password, debug=False):
# Получает с удаленной машины файл remote_file с помощью scp и сохраняет его в local_path.
if local_path[len(local_path) - 1] != '/': local_path += '/'
ssh = SSHClient()
ssh.set_missing_host_key_policy(AutoAddPolicy())
ssh.load_system_host_keys()
if debug: print debug_message("SCP: connecting to %s" % (ip))
try:
ssh.connect(ip, username=username, password=password)
except:
if debug: print debug_message("SCP: failed to connect to %s" % (ip))
else:
if debug: print debug_message("SCP: connected to %s" % (ip))
try:
if debug:
print debug_message("SCP: retrieving file %s" % (remote_file))
scp = SCPClient(ssh.get_transport())
scp.get(remote_file, local_path)
except:
if debug:
print debug_message("SCP: error: failed to retrieve file %s" %
(remote_file))
else:
if debug:
print debug_message("SCP: file saved to %s folder" % (local_path))
ssh.close()
def transfer_file(self, pull=False):
if pull is False:
if not self.local_file_exists():
raise FileTransferError(
'Could not transfer file. Local file doesn\'t exist.')
if not self.enough_remote_space():
raise FileTransferError(
'Could not transfer file. Not enough space on device.')
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(
hostname=self.device.host,
username=self.device.username,
password=self.device.password,
port=self.port,
allow_agent=False,
look_for_keys=False)
scp = SCPClient(ssh.get_transport())
try:
if pull:
scp.get(self.remote, self.local)
else:
scp.put(self.local, self.remote)
except Exception as e:
print(e)
raise FileTransferError
finally:
scp.close()
return True
def run(self):
print(self._name + " started")
#while not self._stopEvent.is_set():
try:
ssh_client = paramiko.SSHClient()
ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh_client.connect(hostname='raspberrypi.local', username='******', password='******')
'''
sftp = ssh_client.open_sftp()
# This will remove a file
sftp.remove("/home/pi/Documents/New.txt")
# This will remove a folder
sftp.rmdir("/home/pi/Documents/New")
sftp.close()
'''
# SCPCLient takes a paramiko transport as an argument
scp = SCPClient(ssh_client.get_transport())
# This will copy a file
#scp.get(remote_path='/home/pi/Documents/New.txt', local_path='')
# This will copy a whole directory
scp.get(remote_path='/home/pi/TestLogs/', local_path='', recursive=True)
scp.close()
ssh_client.close()
except Exception as e:
print("Error : " + str(e))
def sync_status():
"""
Fetches all statuses from a remote node and writes to local status cache
"""
# Run status update, if not, write status file for host and return
# TODO: make this run in it's own cron job (locally?)
host_up = status()
# Make local status folder if necessary
local('mkdir -p ./status' % env)
local('mkdir -p ./statusnew' % env)
# Write out status if failed to contact server
if not host_up:
status_file = './status/status_%(host_string)s.txt' % env
local('echo DOWN at `date` > %s' % status_file)
return
# Pull remote server's status files using scp
from scp import SCPClient
client = SCPClient(connections[env.host_string]._transport)
client.get('~/status/*', './statusnew/' % env, preserve_times=True)
# Use rsync to update older files
local('rsync -auv ./statusnew/* ./status/' % env)
# Write time into last_sync file
local('date > ./status/last_sync' % env)
# TODO: build html from statuses and save atomically?
make_html.dump()
def scp(self, local_path, remote_path, get=False, timeout=10):
"""Copy files from local_path to remote_path or vice versa.
connect() method has to be called first!
:param local_path: Path to local file that should be uploaded; or
path where to save remote file.
:param remote_path: Remote path where to place uploaded file; or
path to remote file which should be downloaded.
:param get: scp operation to perform. Default is put.
:param timeout: Timeout value in seconds.
:type local_path: str
:type remote_path: str
:type get: bool
:type timeout: int
"""
if not get:
logger.trace('SCP {0} to {1}:{2}'.format(
local_path,
self._ssh.get_transport().getpeername(), remote_path))
else:
logger.trace('SCP {0}:{1} to {2}'.format(
self._ssh.get_transport().getpeername(), remote_path,
local_path))
# SCPCLient takes a paramiko transport as its only argument
scp = SCPClient(self._ssh.get_transport(), socket_timeout=timeout)
start = time()
if not get:
scp.put(local_path, remote_path)
else:
scp.get(remote_path, local_path)
scp.close()
end = time()
logger.trace('SCP took {0} seconds'.format(end - start))
def transfer_file(self, hostname=None, username=None, password=None, pull=False):
"""Transfer the file to the remote device over SCP.
Note:
If any arguments are omitted, the corresponding attributes
of ``self.device`` will be used.
Args:
hostname (str): OPTIONAL - The name or
IP address of the remote device.
username (str): OPTIONAL - The SSH username
for the remote device.
password (str): OPTIONAL - The SSH password
for the remote device.
Returns:
True if successful.
Raises:
FileTransferError: if the transfer isn't successful.
"""
if pull is False:
if not self.local_file_exists():
raise FileTransferError(
'Could not transfer file. Local file doesn\'t exist.')
if not self.enough_space():
raise FileTransferError(
'Could not transfer file. Not enough space on device.')
hostname = hostname or self.device.host
username = username or self.device.username
password = password or self.device.password
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(
hostname=hostname,
username=username,
password=password,
port=self.port,
allow_agent=False,
look_for_keys=False)
full_remote_path = '{}{}'.format(self.file_system, self.dst)
scp = SCPClient(ssh.get_transport())
try:
if pull:
scp.get(full_remote_path, self.src)
else:
scp.put(self.src, full_remote_path)
except:
raise FileTransferError(
'Could not transfer file. There was an error during transfer. Please make sure remote permissions are set.')
finally:
scp.close()
return True
def get_reports(self, options, conf_dict):
master = self.existing_nodes[0]
ssh_c = SSHClient(hostname = master, username = \
self.username, password = self.password)
coverage_xml = conf_dict['coverage']['coverage_xml']
scp = SCPClient(ssh_c.get_transport())
scp.get(coverage_xml)
def secure_copy(user, host, src, dest, key_filename=None, allow_agent=True):
keys = _load_keys(key_filename, allow_agent)
pkey = keys[0]
ssh = paramiko.SSHClient()
ssh.load_system_host_keys()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(host, username=user, pkey=pkey)
scp = SCPClient(ssh.get_transport())
scp.get(src, dest)
scp.close()
def transport_assembly_run( seed_name, tfilename, workdir='./build' ):
# Push package: to-be processed packages are placed in 'put' directory.
# Pulled package: finished work is in the 'get' directory.
retry = 1
# Credentials wide out in the open:
ssh = createSSHClient(CL_SERVER_NAME, 22, CL_SERVER_USER, CL_SERVER_PW)
while (retry < 10):
try:
scp = SCPClient(ssh.get_transport())
scp.put(tfilename, 'run')
# If it got this far, it succeeded
break;
except Exception as err:
print '=============='
print err[0]
print ' Retry attemp #'+retry
print '=============='
sleep(5)
#
# Server Processing Phase
#
waittime = 0
while (waittime < 600):
try:
scp = SCPClient(ssh.get_transport())
# Check the log file
scp.get('get/'+seed_name+'_run.log', local_path=workdir)
line = tail("-1", os.path.join(workdir,seed_name+'_run.log') ).strip()
print line
if 'Assembly Complete' in line:
break
if 'ERROR' in line:
exit(1)
except Exception as err:
print '=============='
print err[0]
print '=============='
if 'No such file or directory' in err[0]:
print "---- No status yet ----"
sleep(2)
waittime = waittime + 1
scp.close()
# Compile complete. Download results.
try:
scp.get('get/'+seed_name+'_assembly.dat', local_path=workdir)
os.rename( os.path.join(workdir, seed_name+'_assembly.dat'), os.path.join(workdir,'assembly.dat'))
except Exception as err:
print '=============='
print err[0]
print '=============='
exit(1)
return True
class pySCP(): def __init__(self, server, user, password): self.client = createSSHClient(server, user, password) self.scp = SCPClient(self.client.get_transport()) def scp_put(self, f1, f2): self.scp.put(f1, f2) def scp_get(self, f, local_path): self.scp.get(f, local_path)
def SecureCopy(user, host, src, dest,
key_filename=os.path.join(os.path.expanduser("~"), '.ssh', 'id_rsa')):
ssh = paramiko.SSHClient()
ssh.load_system_host_keys()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(host, username=user, key_filename=key_filename)
scp = SCPClient(ssh.get_transport())
scp.get(src, dest)
scp.close()
def fetch(fileDirectory):
print('Fetching ' + fileDirectory)
client = connect()
try:
scp = SCPClient(client.get_transport())
scp.get(fileDirectory)
scp.close()
client.close()
return True
except:
client.close()
return False
def get_file(self, remote_filepath, curr_filepath='.'):
"""
Download a file from the VM resource to the host machine
:param str remote_filepath: Source filepath of the file in the DFS
:param str local_filepath: Destination Filepath in the local machine of the file
"""
ssh = self.create_ssh_client()
scp = SCPClient(ssh.get_transport())
scp.get(remote_filepath, local_path=curr_filepath)
ssh.close()
return None
def SecureCopy(user, host, src, dest, # pylint: disable=too-many-arguments
key_filename=os.path.join(os.path.expanduser("~"), '.ssh', 'id_rsa')):
ssh = paramiko.SSHClient()
ssh.load_system_host_keys()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
pkey = _load_key(key_filename)
ssh.connect(host, username=user, pkey=pkey)
scp = SCPClient(ssh.get_transport())
scp.get(src, dest)
scp.close()
def scp_connect(server, username, password, col_log, col_type):
print ('Connecting to remote host ' + server)
ssh = SSHClient()
#ssh.load_system_host_keys()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
try:
ssh.connect(server, 22, username, password)
scp = SCPClient(ssh.get_transport())
except:
print ('Unable to connect, please check your credentials and hostname')
sys.exit()
# Copy files to localdir
if col_log == 'None' and col_type == 'vcd':
col_log = default_vcd_debug
col_log_info = default_vcd_info
# Create local directory to hold log files
try:
os.stat(tmp_dir)
print ('Local temp folder exists')
except:
os.mkdir(tmp_dir)
print ('Creating local temp folder')
#attempting to grab debug log
print ('Connected to remote host\n')
file_path = '/tmp/vmw_eps_calc/'
try:
print ('Attempting to copy DEBUG logs to your machine. This (~11M) can take a a bit depending on your connection....\n')
scp.get(col_log, file_path)
# Pass file location to file_list() after parsing it a bit to extract the actual file name
file_name_index = col_log.rfind('/')
file_name = col_log[file_name_index:]
local_file_path = tmp_dir + file_name
print ('File copied successfully')
file_list(local_file_path, col_type)
except:
print ('Error retrieving log file ' + col_log)
try:
print ('Now doing the info level logs...\n')
scp.get(col_log_info, file_path)
# Pass file location to file_list() after parsing it a bit to extract the actual file name
file_name_index = col_log_info.rfind('/')
file_name = col_log_info[file_name_index:]
local_file_path = tmp_dir + file_name
print ('File copied successfully')
file_list(local_file_path, col_type)
except:
print ('Error retrieving log file ' + col_log_info)
scp.close()
def download(source, dest=None):
"""
Download files from a remote SCP source
@param source_info: Source information such as the source to download from.
@type source_info: a dictionary
@param local_dest_dir: The local filename
@type local_dest_dir: str
@return: True and a list of file downloaded if successful. Otherwise False.
"""
try:
ssh = SSHClient()
ssh.load_system_host_keys()
ssh.set_missing_host_key_policy(AutoAddPolicy())
url = urlsplit(source['url'])
username = url.username
# Use the current user if one was not specified
if not username:
username = pwd.getpwuid(os.getuid())[0]
ssh.connect(url.hostname, port=url.port or 22, username=username, password=url.password)
scp = SCPClient(ssh.get_transport())
# Download file to a local temporary file if a local file path is not specified.
if not dest:
dest = tempfile.mkdtemp()
if os.path.exists(dest) and os.path.isdir(dest):
# TODO: we should somehow support scp://dir to scp://dir
# get filename from source
filename = os.path.basename(url.path)
if not filename:
fd, dest = tempfile.mkstemp(dir=dest)
else:
dest = os.path.join(dest, filename)
scp.get(url.path, dest, recursive=False)
ssh.close()
outputfile = {'url': dest,
'name': os.path.basename(url.path),
'content_type': 'application/octet-stream'
}
return [outputfile]
except SCPException:
LOG.error("Could not SCP file %s from %s to local destination\
%s as user %s", url.path, url.hostname, dest, username, exc_info=True)
raise
def copy_ssh_to_tmp(self, uri):
ssh = SSHClient()
ssh.load_system_host_keys()
print uri.username
if not uri.username:
ssh.connect(uri.hostname, allow_agent=True)
elif uri.username and not uri.password:
ssh.connect(uri.hostname, username=uri.username, allow_agent=True)
elif uri.username and uri.password:
ssh.connect(uri.hostname, username=uri.username, password=uri.password, allow_agent=True)
# SCPCLient takes a paramiko transport as its only argument
scp = SCPClient(ssh.get_transport())
scp.get(uri.path, self.tmp_file.name)
scp.close()
def secure_copy(user, host, src, dest, key_filename=None, allow_agent=True):
keys = _load_keys(key_filename, allow_agent)
pkey = keys[0]
ssh = paramiko.SSHClient()
conf = paramiko.SSHConfig()
ssh_config_file = os.path.expanduser("~/.ssh/config")
conf.parse(open(ssh_config_file))
host_config = conf.lookup(host)
proxy = paramiko.ProxyCommand(host_config['proxycommand'])
ssh.load_system_host_keys()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(host, username=user, pkey=pkey, sock=proxy)
scp = SCPClient(ssh.get_transport())
scp.get(src, dest)
scp.close()
def main(args):
print 'connecting to Solo...'
solo = soloutils.connect_solo(await=True)
code, stdout, stderr = soloutils.command(solo, 'ls -p /log | grep -v /')
files = stdout.strip().split()
scp = SCPClient(solo.get_transport())
count = 0
for item in files:
print 'file {} of {}...'.format(count, len(files))
scp.get('/log/' + item)
count += 1
print 'complete.'
def download_clicks():
hostname="172.31.27.220"
user="******"
keyfile = "/Users/razhong/services.pem"
dest = "/Users/razhong/data/video/"
keyfile = "D:\\randy\\services.pem"
src = "/mnt/data/randy/videos/" + yesterday
client = paramiko.SSHClient()
client.load_system_host_keys()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect(hostname, username=user, key_filename=keyfile)
scp = SCPClient(client.get_transport(), sanitize=lambda x: x, progress=progress)
scp.get(src, CWD, recursive=True)
def __init__(self, race_num):
############################################
# building the path to the directory #
# User input race_unm #
# Absolute path is: '/mnt/bugs/RACE-XXXX' #
############################################
self.racePath = HOST_PATH + str(race_num)
##########################################
# Creating secure connection to IBM host #
# with SSH protocol #
##########################################
print "Creating connection......."
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
try:
ssh.connect(HOST, username=USER_NAME, password=PASS)
print "Connected successfully to %s!!!" % HOST
except paramiko.AuthenticationException:
print "Authentication failed when connecting to %s !!!" % HOST
print "=" * 30
print "path is: %s" % self.racePath
print "=" * 30
##################################################
# Copy the required directory from Remote host #
# to local host with SCP (secured Copy) protocol #
##################################################
try:
scp = SCPClient(ssh.get_transport())
print "Copying Directory..."
except Exception as e:
print "Other exception: " + e.message + str(e.__class__) #+ "SCP=" + str( scp)
scp.get(self.racePath, '/home/support/Desktop', True, False)
###########################################
# Close SCP and SSH concetion when done #
###########################################
scp.close()
print "SCP Connection closed %s" % scp
ssh.close()
print "SSH Connection closed %s" % ssh
async def get_ssh_log():
import os
from paramiko import SSHClient
from scp import SCPClient
ssh = SSHClient()
ssh.load_system_host_keys()
local_path = os.getcwd()
main_log = "path/to/log"
ssh.connect("URI", 22, "login_here", "password_here")
scp = SCPClient(ssh.get_transport())
scp.get(main_log, local_path = local_path)
log_records = ""
with open("logMain.log", "r+") as log:
return log_records.join(log)
return "Some error happened!!!"
def get_reports(self, options, conf_dict):
master = self.existing_nodes[0]
ssh_c = SSHClient(hostname = master, username = \
self.username, password = self.password)
coverage_data = conf_dict['coverage']['coverage_data']
scp = SCPClient(ssh_c.get_transport())
scp.get(coverage_data)
coverage_xml = conf_dict['coverage']['coverage_xml']
scp = SCPClient(ssh_c.get_transport())
scp.get(coverage_xml)
remotepath = conf_dict['coverage']['coverage_html']
cmd = 'scp -r ' + master + ':' + remotepath + ' .'
p = Popen(cmd, shell=True, stdin=PIPE, stdout=PIPE, stderr=STDOUT, close_fds=True)
output = p.stdout.read()
print output
def create(self, hostInfo, build_number, version, logPath = None):
configFilePath = hostInfo['configfile']
os.chdir(self.workDir)
if not os.path.isdir(self.workDir + '/' + hostInfo['mac'].replace(':', '-')):
os.mkdir(hostInfo['mac'].replace(':', '-'))
virtenvPath = self.workDir + '/' + hostInfo['mac'].replace(':', '-') + '/' + build_number
if os.path.isdir(virtenvPath):
os.system('rm -rf %s' % virtenvPath)
os.system('mkdir -p %s' % virtenvPath)
os.chdir(virtenvPath)
bash('virtualenv testenv')
self.logger.info('created virtual test environment')
ssh = SSHClient()
ssh.set_missing_host_key_policy(AutoAddPolicy())
ssh.connect(hostname=hostInfo['hostname'], username='******', password=hostInfo['password'])
scp = SCPClient(ssh.get_transport())
scp.get(self.marvinPath, './dist/', recursive=True)
self.pythonPath = virtenvPath + '/testenv/bin/'
bash('wget http://cdn.mysql.com/Downloads/Connector-Python/mysql-connector-python-2.0.4.zip#md5=3df394d89300db95163f17c843ef49df')
bash('%s/easy_install mysql-connector-python-2.0.4.zip' % self.pythonPath)
marvinName = bash('ls ./dist/ | grep Marvin.*tar.gz').getStdout()
bash('tar -xf ./dist/%s' % marvinName)
scp.get(self.testcasePath, './', recursive=True)
self.logger.info('executing %s install -e %s/%s' % (self.pythonPath + 'pip2.7', virtenvPath, marvinName.replace('.tar.gz', '')))
os.system('%s install -e %s/%s ' % (self.pythonPath + 'pip2.7', virtenvPath, marvinName.replace('.tar.gz', '')))
configFileName = hostInfo['mac'].replace(':', '-') + '.cfg'
bash('cp %s ./%s' % (configFilePath, configFileName))
bash("sed -i 's/10.147.28.149/%s/g' %s" % (hostInfo['hostname'], virtenvPath + '/' + configFileName))
if logPath == None:
logPath = virtenvPath.replace('/', '\\/') + '\\/results'
bash("sed -i 's/\\/tmp\\/cloudstack/%s/g' %s" % (logPath, virtenvPath + '/' + configFileName))
env = {'pythonPath': self.pythonPath,
'config_file': configFileName,
'virtenvPath': virtenvPath,
'hostip': hostInfo['ip'],
'build_number': build_number,
'version': version,
'noSimulator': hostInfo['noSimulator'],
'repo_url': hostInfo['repo_url'],
'startTime': hostInfo['startTime'],
'commit_id': hostInfo['commit_id']}
self.logger.info('Adding environment specific data to %s/testenv/lib/python2.7/site-packages/marvin/config/test_data.py' % virtenvPath)
bash('/root/cloud-autodeploy2/newcode/editTestdata.sh /root/cloud-autodeploy2/newcode/env_specific_test_data %s/%s/marvin/config/test_data.py' % (virtenvPath, marvinName.replace('.tar.gz', '')))
return env
def post(self):
if self.arguments:
api_logger.info("HEADERS: "+str(self.request))
# Parse each param
data = self.arguments
if 'name' not in list(data.keys()) and 'lines' not in list(data.keys()):
api_logger.error("Error requests params.")
self.finish('{"result":"error","description":"Error requests params"}')
else:
try:
t_name = data['name']
t_lines = data['lines']
except Exception as e:
api_logger.error("Error requests params "+str(e))
self.finish('{"result":"error","description":"Error requests params","debug":"'+str(e)+'"}')
try:
workerFile = open(options.storm_topology_path+t_name+"/worker","r")
hostName,fileName = workerFile.read().splitlines()
api_logger.debug("HostName LOG: "+hostName)
api_logger.debug("FileName LOG"+fileName)
#get log file from storm cluster
ssh = SSHClient()
ssh.load_system_host_keys()
ssh.connect(hostName)
ssh.set_missing_host_key_policy(AutoAddPolicy())
scp = SCPClient(ssh.get_transport())
api_logger.debug('/var/log/storm/'+fileName)
scp.get('/var/log/storm/'+fileName,'/data/tmp/')
u = Utils()
api_logger.debug('/data/tmp/'+fileName)
lines = ""
for line in u.lastlines('/data/tmp/'+fileName,int(t_lines)):
lines += line
os.remove('/data/tmp/'+fileName)
self.set_header ('Content-Type', 'text/plain')
self.set_header ('Content-Disposition', 'attachment; filename='+fileName+'')
self.finish(lines)
except Exception as e:
api_logger.error("Error getting topology log "+str(e))
self.finish('{"result":"error","description":"Error getting topology log.", "detail":"Log not found, wait while your topology create it."}')
else:
api_logger.error("Content-Type:application/json missing")
self.finish('{"result":"error","description":"Content-Type:application/json missing"}')
class SecureShell(AbstractRemoteShell):
def __init__(self, hostname, username, password=None, port=22, **kwargs):
super(SecureShell, self).__init__(hostname, **kwargs)
self._hostname = hostname
self._port = port
self._username = username
self._password = password
self.connect()
def do_connect(self):
self._client = SSHClient()
self._client.load_system_host_keys()
self._client.set_missing_host_key_policy(AutoAddPolicy())
self._client.connect(hostname=self._hostname, port=self._port, username=self._username, password=self._password)
self._scp_client = SCPClient(self._client.get_transport())
def do_disconnect(self):
self._client.close()
def execute_command(self, command, env={}, wait=True, check_err=False, cwd=None):
for var, val in env.items():
command = "%s=%s; " % (var, val) + command
chan = self._client.get_transport().open_session()
chan.exec_command( (("cd \"%s\"; " % cwd) if cwd else "") + command)
queue = Queue()
StandardStreamReader(chan.makefile("r"), 1, queue)
StandardStreamReader(chan.makefile_stderr("r"), 2, queue)
def post_process_exit_code():
queue.put( (0, chan.recv_exit_status()) )
queue.put( (0, None) )
Thread(target=post_process_exit_code).start()
return ShellResult(self, command, queue, wait, check_err)
def do_pull(self, local_path, remote_path):
self._scp_client.get(remote_path, local_path)
def do_push(self, local_path, remote_path):
self._scp_client.put(local_path, remote_path)
def do_reboot(self):
self("reboot > /dev/null 2>&1 &")
sleep(.3)
