def execute(self, client, stage_number):
"""
Move the tar file into place, unpack and run bootstrap.sh passing any given arguments.
"""
sftp = SFTPClient.from_transport(client.get_transport())
stage_directory = 'stage-' + str(stage_number)
try:
sftp.mkdir(stage_directory)
except:
logger.exception('')
sftp.chdir(stage_directory)
# Stage complete so nothing to do.
if 'stage-complete' in sftp.listdir():
return
# upload, untar, execute bootstrap.sh
sftp.put(self.tarfile, '{0}/{1}.tar'.format(sftp.getcwd(), stage_directory))
untar = 'cd {0} && tar xf {1}.tar'.format(stage_directory, stage_directory)
untar_result = self.execute_command(untar, client)
logger.debug('Command results: {0}.'.format(untar_result))
logger.info('Executing contents of tar file: {0}.'.format(self.tarfile))
bootstrap_arguments = ' '.join(self.args)
run_bootstrap = 'cd {0} '.format(
stage_directory) + '&& sudo -u root -H bash -l -c "bash bootstrap.sh {0}" > output '.format(
bootstrap_arguments) + '&& touch stage-complete'
bootstrap_result = self.execute_command(run_bootstrap, client)
if bootstrap_result[1] != 0:
logger.error("Stage did not complete: {0}.".format(stage_number))
logger.debug('Command results: {0}.'.format(bootstrap_result[0]))
def apply_ericomshield_version():
with tarfile.open('./system.tar.gz'.format(ericom_shield_install_dir),
mode='w:gz') as tar:
for file in glob.glob("{}/*".format(ericom_shield_install_dir)):
tar.add(file)
for file in glob.glob('{}/.*'.format(ericom_shield_install_dir)):
if not '.es_ip_address' in file:
tar.add(file)
transport = client.get_transport()
sftp_client = SFTPClient.from_transport(transport)
sftp_client.put(
'./system.tar.gz'.format(ericom_shield_install_dir),
'/home/{}/system.tar.gz'.format(os.environ['MACHINE_USER']))
sftp_client.close()
_, stdout, stderr = client.exec_command(
"tar xfz ./system.tar.gz && sudo rsync -avh ./usr/local/ericomshield/ /usr/local/ericomshield/ && rm -rf ./usr ./system.tar.gz"
)
while not stdout.channel.exit_status_ready():
one_line = ''
if stdout.channel.recv_ready():
one_line = stdout.channel.recv(2048).decode("ascii")
sys.stdout.write(one_line)
sys.stdout.flush()
output = subprocess.check_output(
'rm -f ./system.tar.gz'.format(ericom_shield_install_dir), shell=True)
def remote_scp(ftp_type, host_ip, remote_path, local_path, username, password):
ssh_port = 22
try:
conn = Transport((host_ip, ssh_port))
conn.connect(username=username, password=password)
sftp = SFTPClient.from_transport(conn)
if ftp_type == 'remoteRead':
print('read')
if not local_path:
filename = os.path.split(remote_path)
local_path = os.path.join('/tmp', filename[-1])
print('开始从服务器下载文件......')
sftp.get(remote_path, local_path)
print(f'文件{filename[-1]}已经下载到本地')
if ftp_type == "remoteWrite":
print('write')
sftp.put(local_path, remote_path)
conn.close()
return True
except IOError as e:
print('没有找到目录', e)
except Exception as e:
print('error!!!', e)
def restore(path):
dbinfo = choicedb(path+'/conf/db_info.yaml')
editData(dbinfo, [['reset master;']])
if sys.platform == 'win32':
t = Transport(dbinfo[0], 22)
t.connect(username=dbinfo[5], password=dbinfo[6])
sftp = SFTPClient.from_transport(t)
src = '/home/mysql/basedata/initdata.sql'
des = path + '/database/initdata.sql'
sftp.put(des,src )
t.close()
else:
child = spawn(
'scp '+path+'/database/initdata.sql '+ dbinfo[5]+'@'+dbinfo[0]+':/home/mysql/basedata/')
child.expect("password:"******"'"+dbinfo[3]+"'"' '+dbinfo[4]+' </home/mysql/basedata/initdata.sql')
return stderr.read().decode('gbk', errors='ignore')
def dbbackup(path):
dbinfo = choicedb(path + '/conf/db_info.yaml')
s = sshconnect(hostname=dbinfo[0], port=dbinfo[7], username=dbinfo[5], password=dbinfo[6])
st, std, stde = s.exec_command('cd /home/mysql/backup'+'\n' 'ls')
filelist = std.read().decode('gbk',errors='ignore').split('\n')
if 'databackup.sql' in filelist:
s.exec_command('cd /home/mysql/backup'+'\n' 'rm -rf databackup1.sql')
stdin, stdout, stderr = s.exec_command('mysqldump -t -u'+dbinfo[2]+' -p'+dbinfo[3]+' '+dbinfo[4]+' >/home/mysql/backup/databackup.sql')
if 'error' not in stderr.read().decode('gbk', errors='ignore'):
t1,t2,t3=s.exec_command('du -b /home/mysql/databackup.sql')
if sys.platform == 'win32':
t = Transport(dbinfo[0], 22)
t.connect(username=dbinfo[5], password=dbinfo[6])
sftp = SFTPClient.from_transport(t)
src = '/home/mysql/backup/databackup.sql'
des = path + '/database/databackup.sql'
sftp.get(src, des)
t.close()
else:
child = spawn(
'scp ' + dbinfo[5] + '@' + dbinfo[0] + ':/home/mysql/backup/databackup.sql ' + path + '/database/')
child.expect("password:")
child.sendline(dbinfo[6])
child.read()
with zipfile.ZipFile(path + '/database/backupdatabase.zip', 'w') as z:
z.write(path + '/database/databackup.sql')
return t2.read().decode('gbk',errors='ignore').split('\t')[0].strip()
else:
return False
s.close
def _authenticate(self):
self._transport = SFTPTransport((self.config['sftp_host'],
self.config['sftp_port']))
self._transport.connect(username=self.config['sftp_user'],
password=self.config['sftp_password'])
self.session = SFTPClient.from_transport(self._transport)
logging.info('SFTP Authorization succeed')
def prepare_machine_to_docker_node(ip):
_, stdout, _ = client.exec_command('hostname')
hostname = (stdout.read()).decode('ascii').replace('\n', '')
with open('hosts', mode='w') as file:
file.write("{0}\t{1}\n".format(ip, hostname))
for manager in get_managers_ip_and_name():
file.write('{}\n'.format(manager))
file.close()
transport = client.get_transport()
sftp_client = SFTPClient.from_transport(transport)
# sftp_client.put(os.path.abspath('./hosts'),'/home/{}/hosts'.format(os.environ['MACHINE_USER']))
# stdin, stdout, stderr = client.exec_command('cat ./hosts | sudo tee -a /etc/hosts && rm -f ./hosts')
# stdout.channel.recv_exit_status()
# logger.error(stderr.read())
sftp_client.put(
os.path.abspath('./sysctl_shield.conf'),
'/home/{}/sysctl_shield.conf'.format(os.environ['MACHINE_USER']))
sftp_client.put(
os.path.abspath('./mount-tmpfs-volume.sh'),
'/home/{}/mount-tmpfs-volume.sh'.format(os.environ['MACHINE_USER']))
sftp_client.close()
stdin, stdout, stderr = client.exec_command(
'chmod +x ./mount-tmpfs-volume.sh && sudo ./mount-tmpfs-volume.sh && rm -f ./mount-tmpfs-volume.sh'
)
stdout.channel.recv_exit_status()
logger.error(stderr.read())
stdin, stdout, stderr = client.exec_command(
'cat ./sysctl_shield.conf | sudo tee "/etc/sysctl.d/30-ericom-shield.conf"; sudo sysctl --load="/etc/sysctl.d/30-ericom-shield.conf"'
)
stdout.channel.recv_exit_status()
logger.error(stderr.read())
return hostname
def install_docker():
transport = client.get_transport()
sftp_client = SFTPClient.from_transport(transport)
sftp_client.put(
os.path.abspath('./install-docker.sh'),
'/home/{}/install-docker.sh'.format(os.environ['MACHINE_USER']))
sftp_client.close()
running = True
channel = client.get_transport().open_session()
channel.exec_command(
'chmod +x install-docker.sh && sudo ./install-docker.sh && rm -f ./install-docker.sh'
)
print('Install docker please wait...', end='')
while running:
sys.stdout.write('.')
sys.stdout.flush()
time.sleep(1)
if channel.exit_status_ready():
print(' ')
if channel.recv_exit_status() == 0:
print("Docker installed")
else:
print("Docker installation failed")
running = False
if test_docker_on_machine():
logger.info("Docker installation success")
else:
logger.error("Docker installation failed")
sys.exit(1)
def check_destination_path(self, path, mkdir_destination=None, ssh=None):
"""Returns the destination_dir or raises an Exception if destination_dir does
not exist on the remote host.
@param path: path on remote host
@type path: byte or str
@param mkdir_destination: if True attempts to create the remote folder.
@type mkdir_destination: boolean
@raise FileNotFoundError: if path does not exist and mkdir_destination is False
@return path
"""
ssh = ssh if ssh else self.ssh
if path[0:1] == b'~' or path[0:1] == '~':
_, stdout, _ = ssh.exec_command("pwd")
path = os.path.join(stdout.readlines()[0].strip(),
path.replace('~/', ''))
with SFTPClient.from_transport(ssh.get_transport()) as sftp:
try:
sftp.chdir(path)
except IOError:
if mkdir_destination or self.mkdir_destination:
self.mkdir_p(sftp, path)
else:
raise FileNotFoundError(
'{} not found on remote host.'.format(path))
return path
return None
def connect(self):
host = self.server.host.split(':')
port = 21 if self.mode == 0 else 22
if len(host) > 1:
try:
port = int(host[1])
except:
message = log.Message('port: %s is invalid.' % host[1],
log.Level.ERROR)
message.log()
return False
if self.mode == 0:
self.session = ftplib.FTP()
self.session.connect(host[0], port)
self.session.login(self.server.user, self.server.password)
else:
ssh = SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(host[0],
port,
username=self.server.user,
password=self.server.password)
self.session = SFTPClient.from_transport(ssh.get_transport())
return True
def connect(self):
""" Get a handle to a remote connection """
# Check URL
schema = urlparse(self.url)
if schema.scheme == 'sftp':
self.transport = Transport((schema.hostname, int(schema.port)))
else:
raise SFTPError('Not a valid sftp url %s, type is %s' %
(self.url, schema.scheme))
# Add authentication to transport
try:
if self.password:
self.transport.connect(username=self.user_name,
password=self.password)
elif self.private_key:
self.transport.connect(username=self.user_name,
pkey=RSAKey.from_private_key(
StringIO(self.private_key)))
else:
raise SFTPError("No password or private_key defined")
# Connect
self.conn = SFTPClient.from_transport(self.transport)
except (socket.gaierror, error), msg:
raise SFTPError(
str(msg) + ' while establishing connection to %s' %
(self.url, ))
def _connect(self):
"""Open connection to remote host."""
if self.conn is None:
try:
self.client = SSHClient()
self.client.load_system_host_keys()
self.client.set_missing_host_key_policy(AutoAddPolicy())
self.client.connect(self.host,
username=self.user,
key_filename=self.ssh_key_filepath,
look_for_keys=True,
banner_timeout=2000,
auth_timeout=2000,
timeout=5000)
# Define progress callback that prints the current percentage completed for the file
def progress(filename, size, sent):
print("%s's progress: %.2f%% \r" %
(filename, float(sent) / float(size) * 100),
end='\x1b[1K\r')
progress_val = progress if self.show_progress else None
self.scp = SCPClient(self.client.get_transport(),
progress=progress_val,
socket_timeout=15.0)
self.sftp = SFTPClient.from_transport(
self.client.get_transport())
except AuthenticationException as error:
logger.error(f'Authentication failed: \
did you remember to create an SSH key? {error}')
raise error
return self.client
def __init__(self, host, port=22, user='******', progress=None):
self.ssh = SSHClient()
self.ssh.load_system_host_keys()
self.ssh.set_missing_host_key_policy(AutoAddPolicy())
self.ssh.connect(host, port, user)
self.sftp = SFTPClient.from_transport(self.ssh.get_transport())
self.scp = SCPClient(self.ssh.get_transport(), progress=progress)
def connect_remote(self):
# open connection to remote host
try:
self.client = SSHClient()
self.client.load_system_host_keys()
self.client.set_missing_host_key_policy(AutoAddPolicy())
self.client.connect(self.host,
port=22,
username=self.user,
password=self.password,
look_for_keys=False,
timeout=10)
self.scp = SCPClient(self.client.get_transport())
sftp = SFTPClient.from_transport(self.client.get_transport())
try:
sftp.stat(self.remote_path)
except FileNotFoundError:
raise AuthenticationException("Remote path does not exist")
try:
sftp.stat(self.index_path)
except FileNotFoundError:
raise AuthenticationException("Index path does not exist")
return self.client
except AuthenticationException as error:
logger.info(
'Authentication failed: did you enter the correct username and password?'
)
logger.error(error)
self.scp = None
return error
def execute(self, client, stage_number):
"""
Move the tar file into place, unpack and run bootstrap.sh passing any given arguments.
"""
sftp = SFTPClient.from_transport(client.get_transport())
stage_directory = 'stage-' + str(stage_number)
try:
sftp.mkdir(stage_directory)
except:
logger.exception('')
sftp.chdir(stage_directory)
# Stage complete so nothing to do.
if 'stage-complete' in sftp.listdir():
return
# upload, untar, execute bootstrap.sh
sftp.put(self.tarfile, '{0}/{1}.tar'.format(sftp.getcwd(), stage_directory))
untar = 'cd {0} && tar xf {1}.tar'.format(stage_directory, stage_directory)
untar_result = self.execute_command(untar, client)
logger.debug('Command results: {0}.'.format(untar_result))
logger.info('Executing contents of tar file: {0}.'.format(self.tarfile))
bootstrap_arguments = ' '.join(self.args)
run_bootstrap = 'cd {0} '.format(
stage_directory) + '&& sudo -u root -H bash -l -c "bash bootstrap.sh {0}" > output '.format(
bootstrap_arguments) + '&& touch stage-complete'
bootstrap_result = self.execute_command(run_bootstrap, client)
if bootstrap_result[1] != 0:
logger.error("Stage did not complete: {0}.".format(stage_number))
logger.debug('Command results: {0}.'.format(bootstrap_result[0]))
def write_redirects_to_sftp(self, from_path, to_path, cron):
try:
ssh_key_object = RSAKey(filename=app.config['SFTP_SSH_KEY_PATH'],
password=app.config['SFTP_SSH_KEY_PASSPHRASE'])
remote_server_public_key = HostKeyEntry.from_line(app.config['SFTP_REMOTE_HOST_PUBLIC_KEY']).key
# This will throw a warning, but the (string, int) tuple will automatically be parsed into a Socket object
remote_server = Transport((app.config['SFTP_REMOTE_HOST'], 22))
remote_server.connect(hostkey=remote_server_public_key, username=app.config['SFTP_USERNAME'], pkey=ssh_key_object)
sftp = SFTPClient.from_transport(remote_server)
sftp.put(from_path, to_path)
if cron:
return 'SFTP publish from %s to %s succeeded' % (from_path, to_path)
else:
return fjson.dumps({
'type': 'success',
'message': 'Redirect updates successful'
})
except:
if cron:
return 'SFTP publish from %s to %s failed' % (from_path, to_path)
else:
return fjson.dumps({
'type': 'danger',
'message': 'Redirect updates failed'
})
def run(cls, connection, wspaceFolder, stmt):
if connection.transport is None:
raise ConnectionClosedError
lootFolder = join(wspaceFolder, "loot",
str(connection.endpoint).replace(':', '-'), "")
if not exists(lootFolder):
mkdir(lootFolder)
filepath = getattr(stmt, 'file', None)
if filepath is None:
print("You must specify a path")
return False
filedest = join(lootFolder, filepath.replace('/', '_'))
#TODO err management
sftp = SFTPClient.from_transport(connection.transport)
print("Retreiving file " + filepath + "... ", end="")
sys.stdout.flush()
try:
sftp.get(filepath, filedest)
except Exception as e:
print("Error " + str(type(e)) + ": " + str(e))
return False
print("Done")
print("File saved as " + filedest)
def _get_ftp_client(self,
window_size=None,
max_packet_size=None,
buffer_size=None):
""" opens an sftp or scp channel and closes it after execution finished
Returns:
"""
window_size = window_size or settings.SFTP_SCP_WINDOW_SIZE
max_packet_size = max_packet_size or settings.SFTP_SCP_MAX_PACKET_SIZE
buffer_size = buffer_size or settings.SFTP_SCP_BUFFER_SIZE
c = None
try:
if self.file_transfer_protocol == 'sftp':
c = SFTPClient.from_transport(self.ssh_transport,
window_size=window_size,
max_packet_size=max_packet_size)
else:
c = scp.SCPClient(self.ssh_transport, buff_size=buffer_size)
yield c
finally:
try:
if c:
c.close()
except Exception:
log.exception('problems closing sftp/scp client')
def _copy_to_server(self, filename, destination=None):
self._ensure_connected()
if destination is None:
destination = self.working_directory
sftp = SFTPClient.from_transport(self.ssh.get_transport())
# tar and compress a directory before copying
if os.path.isfile(filename):
sftp.put(filename, os.path.join(destination, os.path.basename(filename)))
elif os.path.isdir(filename):
tar_filename = filename + '_' + str(uuid.uuid4()) + '.tar.gz'
self.logger.info("Compressing " + filename + '...')
subprocess.call(['tar', 'cfz', tar_filename, filename])
self.logger.info("Copying " + tar_filename + " to " + self.hostname + ':' + os.path.join(destination, os.path.basename(tar_filename)) + '...')
sftp.put(tar_filename, os.path.join(destination, os.path.basename(tar_filename)))
os.remove(tar_filename)
_, stdout, stderr = self._run_command('cd ' + quote(self.working_directory) + ' && tar xvfz ' + quote(tar_filename) + ' && rm ' + quote(tar_filename))
self.logger.debug("tar STDOUT: " + stdout.read())
self.logger.debug("tar STDERR: " + stderr.read())
else:
raise NotImplementedError("Can't SFTP put anything other than a folder or file yet.")
sftp.close()
def upload(dest, srcPath, destPath, callBack, skipIfExists = False):
try:
sftp = SFTPClient.from_transport(dest.connection)
# This is pretty awful, if the file doesn't exists it throws
# an exception, and then we should continue
# Assume true
exists = True
if skipIfExists:
try:
sftp.stat(destPath)
except FileNotFoundError as e:
exists = False
if not skipIfExists or not exists:
renameUpload(sftp, srcPath, destPath, callBack)
else:
callBack()
return True
except Exception as e:
print('*** Caught exception: %s: %s' % (e.__class__, e))
traceback.print_exc()
sys.exit(1)
finally:
try:
t.close()
except:
pass
return False
def __init__(self, host, port, username, password):
self.host = host
self.port = port
self.username = username
self.password = password
self.t = Transport((self.host, self.port))
self.t.connect(username=self.username, password=self.password) # 注意：此处需要用默认传参方式传参
self.sftp = SFTPClient.from_transport(self.t)
def scp(self, srcFile, destPath):
transport = Transport((self.host, int(self.port)))
transport.connect(username=self.user, password=self.passwd)
sftp = SFTPClient.from_transport(transport)
try:
sftp.put(srcFile, destPath)
except IOError as e:
raise e
def scp(self, src_file, dest_path):
transport = Transport((self.host, int(self.port)))
transport.connect(username=self.user, password=self.passwd)
sftp = SFTPClient.from_transport(transport)
try:
sftp.put(src_file, dest_path)
except IOError as e:
raise e
def __init__(self, user, passwd, ip):
self.ip = ip
try:
t = Transport((ip, 22))
t.connect(username=user, password=passwd)
self.sftpObject = SFTPClient.from_transport(t)
self.status = 'Success'
except:
self.status = 'Failed'
def __init__(self, host, username, password):
try:
logging.basicConfig(format='%(levelname)s - %(message)s', level=logging.INFO)
transport = Transport(sock=(host))
transport.connect(username=username, password=password)
self.connection = SFTPClient.from_transport(transport)
except:
logging.error('Can not able to conenct to server')
def _get_sftp_connection(host):
transport = host.connection.get_transport()
try:
return SFTPClient.from_transport(transport)
except SSHException as e:
raise ConnectError((
"Unable to establish SFTP connection. Check that the SFTP subsystem "
"for the SSH service at {0} is enabled.").format(host), ) from e
def sftpclient(sftpserver):
from paramiko import Transport
transport = Transport((sftpserver.host, sftpserver.port))
transport.connect(username="******", password="******")
sftpclient = SFTPClient.from_transport(transport)
yield sftpclient
sftpclient.close()
transport.close()
def copy(self, local_file, remote_file):
if self.localMachine:
command = "cp %s %s" % (local_file, remote_file)
return self.execute_command(command)
else:
if not hasattr(self, 'ssh_connection'):
self.__get_ssh_connection()
sftp = SFTPClient.from_transport(self.ssh_connection.get_transport())
obj = sftp.put(local_file, remote_file)
return (0, "Copied to %s" % obj.__str__(), "")
def __init__(self, sftp_settings):
self.transport = Transport(
(sftp_settings['HOST'], int(sftp_settings['PORT'])))
self.transport.connect(username=sftp_settings['USER'],
password=sftp_settings['PASSWORD'])
self.connection = SFTPClient.from_transport(self.transport)
logger.debug(
"SFTPUploader initiated. Sending files to {host}:{port}".format(
host=sftp_settings['HOST'], port=sftp_settings['PORT']))
def _get_sftp_connection(host):
transport = host.connection.get_transport()
try:
return SFTPClient.from_transport(transport)
except SSHException as e:
six.raise_from(
ConnectError((
'Unable to establish SFTP connection. Check that the SFTP subsystem '
'for the SSH service at {0} is enabled.').format(host)), e)
def sftp_authentication(self):
try:
transport = Transport((ip_server, 222))
privatekeyfile = os.path.expanduser('./priv_key')
mykey = RSAKey.from_private_key_file(privatekeyfile)
transport.connect(username='******', pkey=mykey)
sftp_client = SFTPClient.from_transport(transport)
return sftp_client, transport
except Exception, e:
print "SFTP Authentication Fail:" + str(e)
def transport(self):
transport = Transport((self.host, self.port))
transport.connect(username=self.username, password=self.password)
sftp = SFTPClient.from_transport(transport)
sftp.put(f'{self.project_url}/web/caweb/html.zip',
f'{self.nginx_url}/html.zip')
sftp.put(f'{self.project_url}/server/caserver.zip',
f'{self.nginx_url}/server/new_caserver.zip')
print("transport finished!")
transport.close()
def __init__(self, config):
try:
transport = Transport((config.SFTP_HOST, config.SFTP_PORT))
transport.connect(username=config.SFTP_USERNAME,
password=config.SFTP_PASSWORD)
self.sftp_client = SFTPClient.from_transport(transport)
except (SSHException, socket.error) as e:
raise SFTPError(f'SFTP connection failed due to: {e}.')
self.path = config.SFTP_PATH
self.temp_file_path = config.TEMP_FILE_PATH
def scp_remote_to_local(remote_file_path, local_file_path, ip, username,
password):
"""Copies a file from remote to local"""
if not username:
username = DEFAULT_CVM_USERNAME
if not password:
password = DEFAULT_CVM_PASSWD
transport = Transport((ip, 22))
transport.connect(username=username, password=password)
sftp = SFTPClient.from_transport(transport)
sftp.get(remote_file_path, local_file_path)
def sftpOpenConnection(self, target):
""" opens an sftp connection to the given target host;
credentials are taken from /etc/ssh/sftp_passwd """
from paramiko import Transport, SFTPClient
from paramiko.util import load_host_keys
hostname, username, password = self.getCredentials(target)
hostkeys = load_host_keys('/etc/ssh/ssh_known_hosts')
hostkeytype, hostkey = hostkeys[hostname].items()[0]
trans = Transport((hostname, 22))
trans.connect(username=username, password=password, hostkey=hostkey)
return SFTPClient.from_transport(trans)
def connect(self):
if not self.session_ and not self.transport_.is_active():
if self.key_: # Try private key, if available
self.transport_.connect(username=self.user_, pkey=self.key_)
elif len(self.pwd_) > 0: # Next, try password, if available
self.transport_.connect(username=self.user_,
password=self.pwd_)
else: # Try just the user name
self.transport_.connect(username=self.user_)
self.session_ = SFTPClient.from_transport(self.transport_)
def __enter__(self):
self.set_transport()
self.sftp = SFTPClient.from_transport(self.transport)
try:
logger.info("Try to make a directory")
self.sftp.mkdir(self.remote_dir)
except Exception as e:
logger.info(e)
logger.info(f"Change the current directory into {self.remote_dir}")
self.sftp.chdir(self.remote_dir)
return self.sftp
def _get_sftp_connection(host):
# SFTP connections aren't *required* for deploys, so we create them on-demand
if host in SFTP_CONNECTIONS:
return SFTP_CONNECTIONS[host]
transport = host.connection.get_transport()
client = SFTPClient.from_transport(transport)
SFTP_CONNECTIONS[host] = client
return client
def run_scp(host_info, src, dest): """ connect to host and send srouce file to destination """ try: ssh_client = SSHClient() ssh_client.set_missing_host_key_policy(AutoAddPolicy()) ssh_client.connect(host_info.ip, host_info.port, host_info.user, key_filename=host_info.key) sftp_client = SFTPClient.from_transport(ssh_client.get_transport()) sftp_client.put(src, dest) sftp_client.close() except Exception as e: raise e
def _connect(self):
self._conn = Transport((self._host, self._port))
self._conn.window_size = pow(2, 27)
self._conn.packetizer.REKEY_BYTES = pow(2, 32)
self._conn.packetizer.REKEY_PACKETS = pow(2, 32)
if self._auth == 'key':
pkey = RSAKey.from_private_key_file(self._key_path)
self._conn.connect(username=self._user, pkey=pkey)
else:
self._conn.connect(username=self._user, password=self._passwd)
self._client = SFTPClient.from_transport(self._conn)
def release_success(self, build_info):
def progress(filename, transferred, total):
print(
colorize('fuchsia', ' * %s transfer in progress: %02d%%.\r' % (filename, transferred * 100 / total)),
end='\r'
)
ssh = SSHClient()
try:
ssh.set_missing_host_key_policy(AutoAddPolicy())
ssh.connect(self.ssh['server'], username=self.ssh['user'], key_filename=self.ssh['pkey'])
except SSHException:
raise XUtilsError('Unable to establish a SSH connection to %s' % self.ssh['server'])
dest_dir = '/'.join([
self.ssh['base_dir'], build_info['category'], build_info['pkg_name'], build_info['version'],
build_info['arch']
])
stdin, stdout, stderr = ssh.exec_command('mkdir -p %s' % dest_dir) # pylint: disable=unused-variable
if stderr.read():
raise XUtilsError('Unable to create directory %s on server %s' % (dest_dir, self.ssh['server']))
src_dir = str()
for f in listdir(self.cfg['build']['workdir']):
if f.endswith('_root.tar.gz'):
src_dir = self.cfg['build']['workdir']
if not src_dir:
src_dir = self.cfg['release']['archive_dir']
files = [
'%s-%s_root.tar.gz' % (build_info['pkg_name'], build_info['version']),
'%s-%s_debuginfo.tar.gz' % (build_info['pkg_name'], build_info['version']),
'%s-%s_root.tar.gz.gpg' % (build_info['pkg_name'], build_info['version']),
]
try:
sftp = SFTPClient.from_transport(ssh.get_transport())
sftp.chdir(dest_dir)
except SSHException:
raise XUtilsError('Unable to negotiate a SFTP session for %s' % self.ssh['server'])
for f in files:
filepath = realpath(src_dir + '/' + f)
if not exists(filepath):
self.info(colorize('yellow', '%s not found => skip it.' % f))
continue
my_callback = partial(progress, f)
remote_attr = sftp.put(filepath, f, callback=my_callback)
if stat(filepath).st_size == remote_attr.st_size:
self.info('%s successfully copied on server %s.' % (f, self.ssh['server']))
else:
raise XUtilsError('Copy of %s on server %s is corrupted.' % (f, self.ssh['server']))
sftp.close()
ssh.close()
def _connect(self):
if self._conn is not None:
log.warn("Already connected to sftp server")
return
try:
self._conn = Transport((self._host, self._port))
self._conn.window_size = pow(2, 27)
self._conn.packetizer.REKEY_BYTES = pow(2, 32)
self._conn.packetizer.REKEY_PACKETS = pow(2, 32)
self._conn.connect(username=self._user, password=self._passwd)
self._client = SFTPClient.from_transport(self._conn)
except Exception as e:
raise IrmaSFTPError("{0}".format(e))
def sftpOpen(self, target):
""" opens an sftp connection to the given target host;
credentials are taken from /etc/ssh/sftp_passwd """
from paramiko import Transport, SFTPClient
#from paramiko.util import load_host_keys
print '>>> sftpOpen self.username: '******' self.password: '******' self.hostkey: ', self.password
print ' self.hostname: ', self.hostname
trans = Transport((self.hostname, 22))
trans.banner_timeout = 120
trans.connect(username=self.username,
password=self.password,
hostkey=self.hostkey)
return SFTPClient.from_transport(trans)
def _putFile(self, data, connection):
""" Puts the given file on the TSM host. """
sftp = SFTPClient.from_transport(connection)
try:
try:
remoteFileObject = sftp.open(self._persistenceIdentifier, "w")
block = data.read(_BLOCK_SIZE)
while len(block) > 0:
remoteFileObject.write(block)
block = data.read(_BLOCK_SIZE)
except (IOError, SSHException), sshException:
errorMessage = "Cannot transfer data to TSM host!\nReason: '%s'" % str(sshException)
raise PersistenceError(errorMessage)
finally:
data.close()
sftp.close()
def listdir(hostname, path="/var/tmp", filter="", port=1035, username="", password=""):
"""
paramiko sftp listdir wrapper, with option to filter files
"""
# Paramiko client configuration
t = Transport((hostname, port))
t.connect(username=username, password=password)
sftp = SFTPClient.from_transport(t)
try:
rex = re.compile(filter)
except:
print "Invalid regular expression: " + filter
sys.exit(1)
return [x for x in sftp.listdir(path) if rex.match(x)]
def upload_report_to_sftp(self, client_id, report_date, absolute_filename):
"""
Upload the given file, using SFTP, to the configured FTP server. The
file should be uploaded to the appropriate directory for the specified
client and the date of the report.
"""
try:
client = Clients.objects.get(id=client_id)
except Clients.DoesNotExist:
logger.exception(u'No configuration for client {0}.'.format(client_id))
raise
filename = basename(absolute_filename)
base_folder, env_folder, year_folder, month_folder = self._get_sftp_dirs(client, report_date)
try:
logger.debug(u'SFTP logging on to {0} as {1}'.format(settings.SFTP_SERVER, settings.SFTP_USERNAME))
transport = Transport((settings.SFTP_SERVER, settings.SFTP_PORT))
transport.connect(username=settings.SFTP_USERNAME, password=settings.SFTP_PASSWORD)
sftp = SFTPClient.from_transport(transport)
logger.debug(u'SFTP dir {0}/{1}/{2}/{3}'.format(base_folder, env_folder, year_folder, month_folder))
sftp.chdir(base_folder)
self._make_or_change_sftp_dir(sftp, env_folder)
self._make_or_change_sftp_dir(sftp, year_folder)
self._make_or_change_sftp_dir(sftp, month_folder)
logger.debug(u'SFTP uploading {0}'.format(filename))
sftp.put(absolute_filename, filename)
except Exception:
logger.exception(u'Unrecoverable exception during SFTP upload process.')
raise
finally:
logger.debug(u'SFTP logging off')
try:
sftp.close()
except Exception:
logger.exception(u'SFTP exception while closing SFTP session.')
try:
transport.close()
except Exception:
logger.exception(u'SFTP exception while closing SSH connection.')
def sftp_connect(config):
try:
hostname = config["hostname"]
port = int(config["port"])
username = config["username"]
pkeyfile = config["privatekey_file"]
pkeypassword = config["privatekey_passphrase"]
host_keys = load_host_keys(config["hostkeys_file"])
hostkeytype, hostkey = get_hostkeytype_and_hostkey(host_keys,
config["hostname"])
t = Transport((hostname, port))
pkey = get_privatekey_from_file(pkeyfile, pkeypassword)
t.connect(username=username, pkey=pkey, hostkey=hostkey)
return SFTPClient.from_transport(t)
except SSHException, e:
print " [*] " + str(e)
sys.exit(1)
def connect(self):
""" Get a handle to a remote connection """
# Check URL
schema = urlparse(self.url)
if schema.scheme == 'sftp':
self.transport = Transport((schema.hostname, int(schema.port)))
else:
raise SFTPError('Not a valid sftp url %s, type is %s' %(self.url, schema.scheme))
# Add authentication to transport
try:
if self.password:
self.transport.connect(username=self.user_name, password=self.password)
elif self.private_key:
self.transport.connect(username=self.user_name,
pkey=RSAKey.from_private_key(StringIO(self.private_key)))
else:
raise SFTPError("No password or private_key defined")
# Connect
self.conn = SFTPClient.from_transport(self.transport)
except (socket.gaierror,error), msg:
raise SFTPError(str(msg) + ' while establishing connection to %s' % (self.url,))
def _getFile(self, connection):
""" Transfers the given file from the TSM host to the local file system. """
try:
temporaryFileObject = tempfile.TemporaryFile()
sftp = SFTPClient.from_transport(connection)
try:
temporaryFileObject.seek(0)
remoteFileObject = sftp.open(self._persistenceIdentifier)
block = remoteFileObject.read(_BLOCK_SIZE)
while len(block) > 0:
temporaryFileObject.write(block)
block = remoteFileObject.read(_BLOCK_SIZE)
sftp.remove(self._persistenceIdentifier)
temporaryFileObject.seek(0)
return temporaryFileObject
except (IOError, SSHException), error:
errorMessage = "Cannot retrieve file from TSM host!\nReason: '%s'" % str(error)
raise PersistenceError(errorMessage)
finally:
sftp.close()
def connect(self):
""" Get a handle to a remote connection """
# Check URL
schema = urlparse(self.url)
if schema.scheme == 'sftp':
hostname = schema.hostname
port = int(schema.port)
# Socket creation code inspired from paramiko.Transport.__init__
# with added bind support.
for family, socktype, _, _, _ in getaddrinfo(
hostname, port, AF_UNSPEC, SOCK_STREAM,
):
if socktype == SOCK_STREAM:
sock = socket(family, SOCK_STREAM)
if self.bind_address:
# XXX: Expects bind address to be of same family as hostname.
# May not be easy if name resolution is involved.
# Try to reconciliate them ?
sock.bind((self.bind_address, 0))
retry_on_signal(lambda: sock.connect((hostname, port)))
break
else:
raise SFTPError('No suitable socket family found')
self.transport = Transport(sock)
else:
raise SFTPError('Not a valid sftp url %s, type is %s' %(self.url, schema.scheme))
# Add authentication to transport
try:
if self.password:
self.transport.connect(username=self.user_name, password=self.password)
elif self.private_key:
self.transport.connect(username=self.user_name,
pkey=RSAKey.from_private_key(StringIO(self.private_key)))
else:
raise SFTPError("No password or private_key defined")
# Connect
self.conn = SFTPClient.from_transport(self.transport)
except (gaierror, error), msg:
raise SFTPError(str(msg) + ' while establishing connection to %s' % (self.url,))
def get_sftp_conn(config):
"""Make a SFTP connection, returns sftp client and connection objects"""
remote = config.get('remote_location')
parts = urlparse(remote)
if ':' in parts.netloc:
hostname, port = parts.netloc.split(':')
else:
hostname = parts.netloc
port = 22
port = int(port)
username = config.get('remote_username') or getuser()
luser = get_local_user(username)
sshdir = get_ssh_dir(config, luser)
hostkey = get_host_keys(hostname, sshdir)
try:
sftp = None
keys = get_ssh_keys(sshdir)
transport = Transport((hostname, port))
while not keys.empty():
try:
key = PKey.from_private_key_file(keys.get())
transport.connect(
hostkey=hostkey,
username=username,
password=None,
pkey=key)
sftp = SFTPClient.from_transport(transport)
break
except (PasswordRequiredException, SSHException):
pass
if sftp is None:
raise SaChannelUpdateTransportError("SFTP connection failed")
return sftp, transport
except BaseException as msg:
raise SaChannelUpdateTransportError(msg)
def __call__(self):
registry = getUtility(IRegistry)
recensio_settings = registry.forInterface(IRecensioSettings)
host = recensio_settings.xml_export_server
username = recensio_settings.xml_export_username
password = recensio_settings.xml_export_password
if not host:
return 'no host configured'
log.info("Starting XML export to sftp")
exporter = getUtility(IFactory, name='chronicon_exporter')()
export_xml = exporter.get_export_obj(self.context)
if export_xml is None:
msg = "Could not get export file object: {0}".format(
exporter.export_filename)
log.error(msg)
return msg
zipstream = export_xml.getFile()
try:
transport = Transport((host, 22))
transport.connect(username=username, password=password)
sftp = SFTPClient.from_transport(transport)
attribs = sftp.putfo(zipstream.getBlob().open(), self.filename)
except (IOError, SSHException) as ioe:
msg = "Export failed, {0}: {1}".format(ioe.__class__.__name__, ioe)
log.error(msg)
return msg
if attribs.st_size == zipstream.get_size():
msg = "Export successful"
log.info(msg)
return msg
else:
msg = "Export failed, {0}/{1} bytes transferred".format(
attribs.st_size, zipstream.get_size())
log.error(msg)
return msg
def upload_report(self, filepath, file_name, report_date):
"""
>>> a = CSVgenerator('123')
>>> a.upload_report(None, 'upload.txt', '2014-05-06')
"""
# appropriate changes required for doc test to run - dev only.
# test_file_path = join(settings.REPORTS_ROOT, '16b6a354-ff32-4be4-b648-fe51fc5b1508.csv')
# # trace.info('----- {}'.format(abspath(test_file_path)))
# absolute_filename = abspath(test_file_path)
# test_filename = '16b6a354-ff32-4be4-b648-fe51fc5b1508.csv'
#TODO remove after QA testing.
if not filepath:
test_file_path = join(settings.REPORTS_ROOT, file_name)
absolute_filename = abspath(test_file_path)
else:
absolute_filename = abspath(filepath)
try:
report_date = report_date if isinstance(report_date, str) else report_date.strftime(settings.DATE_FORMAT_YMD)
except Exception:
log.exception('')
try:
#TODO use sftp credentials, once available.
#TODO get client upload location for per-client report.
# ftp_client_dir = client.ftp_client_dir
# if ftp_client_dir == '':
# ftp_client_dir = getattr(settings, 'DEFAULT_SFTP_LOCATION', 'default')
# logger.exception(u'No FTP configuration for client {} using default value.'.format(client.name))
#
year_folder, month_folder, _ = report_date.split('-')
base_folder_path = self._sftp_settings.get('path', '')
base_path = base_folder_path.split('/')
base_folders = [i for i in base_path[1:-1]]
base_folder = '/' + join(*base_folders)
env_folder = str(base_path[-1:][0])
from paramiko import Transport, SFTPClient
try:
log.info(u'SFTP logging on to {0} as {1}'.format(settings.SFTP_SERVER, settings.SFTP_USERNAME))
transport = Transport((
self._sftp_settings.get('server', ''),
self._sftp_settings.get('port', 22))
)
transport.connect(
username=self._sftp_settings.get('username', ''),
password=self._sftp_settings.get('password', '')
)
sftp = SFTPClient.from_transport(transport)
log.info(u'SFTP dir {0}/{1}/{2}/{3}'.format(base_folder, env_folder, year_folder, month_folder))
try:
sftp.chdir(base_folder)
except Exception:
log.debug('Unable to change to base folder on ftp server.')
self._make_or_change_sftp_dir(sftp, env_folder)
self._make_or_change_sftp_dir(sftp, year_folder)
self._make_or_change_sftp_dir(sftp, month_folder)
log.debug(u'SFTP uploading {0}'.format(filepath))
sftp.put(absolute_filename, file_name)
except Exception:
log.exception(u'Unrecoverable exception during SFTP upload process.')
finally:
log.debug(u'SFTP logging off')
try:
sftp.close()
except Exception:
log.exception(u'SFTP exception while closing SFTP session.')
try:
transport.close()
except Exception:
log.exception(u'SFTP exception while closing SSH connection.')
except Exception:
log.debug('Error while uploading report.')
# -*- coding: utf-8 -*-
__author__ = 'Administrator'
from paramiko import Transport,RSAKey,SFTPClient
import os
host="54.187.91.18"
port=22
private_key_file=os.path.expanduser("~/.ssh/id_rsa")
key=RSAKey.from_private_key_file(private_key_file)
t=Transport((host,port))
t.connect(username="******",pkey=key)
sftp=SFTPClient.from_transport(t)
try:
sftp.put("/data/websites/test/a.php","/data/websites/")
except Exception,e:
print(e)
sftp.close()
t.close()
def copy(self, local_file, remote_file):
if not hasattr(self, 'ssh_connection'):
self.__get_ssh_connection()
sftp = SFTPClient.from_transport(self.ssh_connection.get_transport())
sftp.put(local_file, remote_file)
def submitJobToFramework(self, **kwargs):
jobCommand = 'job'
daemonArgs = DaemonArgs(self.config)
daemonArgs.command = jobCommand
unScheduledJob = kwargs['unScheduledJob']
is_fileFeeder = False
fileFeederUploadedFile = None
del daemonArgs.param[:]
# go through all parameters
for parameter in unScheduledJob.parameters.all():
# add parameter to daemonArgs.param
if parameter.service and parameter.param_key and parameter.param_value:
# check if a file feeder is used
if parameter.service == settings.FILE_FEEDER_ID:
is_fileFeeder = True
fileFeederUploadedFile = parameter.param_value
remoteFeederFile = os.path.join(self.sftpRemotePath, parameter.param_value)
parameterString = '%s.%s=%s' % ( parameter.service, parameter.param_key, remoteFeederFile )
else:
parameterString = '%s.%s=%s' % ( parameter.service, parameter.param_key, parameter.param_value )
self.logger.debug("add parameter string: %s" % parameterString)
daemonArgs.param.append([parameterString])
# in case of a filefeeder upload file to framework server
if is_fileFeeder:
self.logger.debug("is file feeder")
sftp = None
transport = None
try:
transport = Transport((self.sftpHost, self.sftpPort))
if self.sftpPassword:
transport.connect(username=self.sftpUsername, password=self.sftpPassword)
else:
privateKey = None
if self.sftpPrivateKeyType and self.sftpPrivateKeyType.lower() == 'rsa':
privateKey = RSAKey.from_private_key_file(self.sftpPrivateKey, password=self.sftpPrivateKeyPassword )
if self.sftpPrivateKeyType and self.sftpPrivateKeyType.lower() == 'dss':
privateKey = DSSKey.from_private_key_file(self.sftpPrivateKey, password=self.sftpPrivateKeyPassword )
transport.connect(username=self.sftpUsername, pkey=privateKey)
sftp = SFTPClient.from_transport(transport)
filePath = os.path.join( settings.MEDIA_ROOT, fileFeederUploadedFile )
remotePath = os.path.join( self.sftpRemotePath, fileFeederUploadedFile )
self.logger.debug("uploading file from %s to %s on remote machine" % (filePath, remotePath))
sftp.put(filePath, remotePath)
# sftp.put(filePath, remotePath, confirm=False)
sftp.chmod( remotePath, 0644 )
self.logger.debug("put OK")
except IOError as e:
self.logger.error("IOError: %s. Will continue with next scheduled job." % e)
self.saveJob(Job.FAILED_STATUS, None, unScheduledJob)
except PasswordRequiredException as e:
self.logger.error("PasswordRequiredException: %s. Will continue with next scheduled job." % e)
self.saveJob(Job.FAILED_STATUS, None, unScheduledJob)
except SSHException as e:
self.logger.error("SSH Exception: %s. Will continue with next scheduled job." % e)
self.saveJob(Job.FAILED_STATUS, None, unScheduledJob)
except Exception as e:
self.logger.error("Unkown SFTP problem. Will continue with next scheduled job. %s" % e)
self.saveJob(Job.FAILED_STATUS, None, unScheduledJob)
finally:
if sftp is not None:
sftp.close()
if transport is not None:
transport.close()
# set job workflow
daemonArgs.jd_workflow = unScheduledJob.workflow.name
frameworkJobId = None
try:
setattr(daemonArgs, jobCommand, 'submit')
frameworkJobId = self.sendFrameworkCommand(jobCommand, daemonArgs)
self.saveJob(Job.PROCESSING_STATUS, frameworkJobId, unScheduledJob)
except WorkflowNotDeployedException:
# The workflow is not deployed in the framework. To prevent the scheduler retrying continuously
# we disable this job
unScheduledJob.status = Schedule.DEACTIVATE_STATUS
unScheduledJob.save()
except:
self.saveJob(Job.FAILED_STATUS, None, unScheduledJob)
finally:
daemonArgs.clean(jobCommand)
if unScheduledJob.scheduled_start is not None:
unScheduledJob.status = Schedule.DEACTIVATED_STATUS
unScheduledJob.save()
def Link(self,\
sftp=None,\
ssh=True):
self.sock = socket()
self.sock.settimeout(self.timeout)
self.Errors = []
self.protocol = []
self.logger.info('Trying to start link using parameters [SSH=%s,SFTP=%s]'%(ssh,sftp))
try:
self.sock.connect((self.remoteMachine, self.port))
except:
self.Unlink()
self.logger.error('Link: Could not open a socket [remoteMachine = %s, port = %s]' % (self.remoteMachine, self.port))
self.transport = Transport(self.sock)
try:
if not self.password:
self.transport.connect(username = self.username, pkey = self.pk)
self.logger.info('Trying to connect using private key..')
else:
self.transport.connect(username = self.username, password = self.password)
self.logger.info('Trying to connect using password..')
except:
if not self.password:
self.logger.error('Private Key not installed and no password specified!')
self.Unlink()
self.logger.error('Could not connect using SFTP [remoteMachine = %s, Username = %s, port = %s]' %(self.remoteMachine, self.username, self.port))
sftp=None
if ssh:
try:
self.protocol.append(mySSHClient())
self.ssh = self.protocol[-1]
self.ssh.set_missing_host_key_policy(AutoAddPolicy())
self.ssh.connect(self.remoteMachine, port = self.port, username = self.username, password = self.password)
self.logger.info ('ssh connection created..')
except:
self.logger.error('Could not connect using SSH [remoteMachine = %s, Username = %s, port = %s]'%(self.remoteMachine, self.username, self.port))
if sftp:
try:
if ssh == None:
self.logger.error ('SSH must be enabled in order to connect with SFTP')
self.protocol.append(SFTPClient.from_transport(self.transport))
self.sftp = self.protocol[-1]
self.client = self.sftp
self.logger.info('sftp connection created..')
except:
self.logger.error ('Could not bring up SFTP connection')
self.sftp=None
return ssh, sftp
def connect(self):
self._transport = Transport((self._host, self._port))
self._transport.connect(username=self._user, password=self._pass)
self._client = SFTPClient.from_transport(self._transport)
def __connect(self):
self.t = Transport((self.config['hostname'], self.config['port']))
self.t.connect(username = self.config['user'], pkey = self.auth_key)
self.client = SFTPClient.from_transport(self.t)
self.client.chdir(self.path)
def connect(self, host, port=22, user=None, passw=None, cert=None, path='/', timeout=10):
"""Method connects to server
Args:
host (str): server host
port (int): server port, default protocol port
user (str): username
passw (str): password
cert (str): path to certificate file
path (str): server path
timeout (int): timeout
Returns:
bool: result
Raises:
event: ftp_before_connect
event: ftp_after_connect
"""
try:
message = '{0}/{1}@{2}:{3}{4} cert:{5}, timeout:{6}'.format(
user, passw, host, port, path, cert, timeout)
self._mh.demsg('htk_on_debug_info', self._mh._trn.msg(
'htk_ftp_connecting', message), self._mh.fromhere())
ev = event.Event(
'ftp_before_connect', host, port, user, passw, cert, path, timeout)
if (self._mh.fire_event(ev) > 0):
host = ev.argv(0)
port = ev.argv(1)
user = ev.argv(2)
passw = ev.argv(3)
cert = ev.argv(4)
path = ev.argv(5)
timeout = ev.argv(6)
self._host = host
self._port = port
self._user = user
self._passw = passw
self._cert = cert
if (ev.will_run_default()):
setdefaulttimeout(timeout)
t = Transport((host, self._port))
if (user != None or cert != None):
pkey = RSAKey.from_private_key_file(
self._cert) if (cert != None) else None
t.connect(username=user, password=passw, pkey=pkey)
self._client = SFTPClient.from_transport(t)
self._is_connected = True
if (path != None):
self.change_dir(path)
ev = event.Event('ftp_after_connect')
self._mh.fire_event(ev)
return True
except (SSHException, NoValidConnectionsError, error) as ex:
self._mh.demsg(
'htk_on_error', 'error: {0}'.format(ex), self._mh.fromhere())
return False