def _put_sftp_file_on_s3(fname,
s3_account,
s3_directory,
sftp_account,
remove_from_sftp=False,
compress=True,
gpg_recipient=None):
sftp = rowdy.sftp.SFTPConnection(sftp_account.username,
sftp_account.password,
sftp_account.hostname)
_, tempfile = mkstemp()
sftp.open_connection()
sftp.get(fname, tempfile)
sftp.close_connection()
s3_key = os.path.join(s3_directory, os.path.basename(fname))
keys = {
tempfile: s3_key,
}
s3_keys = []
for localpath, s3_key in keys.iteritems():
s3_location = _put_on_s3(localpath, s3_key, s3_account, compress,
gpg_recipient)
os.remove(localpath)
s3_keys.append(s3_location)
if remove_from_sftp:
sftp.open_connection()
sftp.remove(fname)
sftp.close_connection()
return s3_keys
def _put_sftp_file_on_s3(fname, s3_account, s3_directory, sftp_account,
remove_from_sftp=False, compress=True,
gpg_recipient=None):
sftp = rowdy.sftp.SFTPConnection(sftp_account.username,
sftp_account.password,
sftp_account.hostname)
_, tempfile = mkstemp()
sftp.open_connection()
sftp.get(fname, tempfile)
sftp.close_connection()
s3_key = os.path.join(s3_directory, os.path.basename(fname))
keys = {
tempfile: s3_key,
}
s3_keys = []
for localpath, s3_key in keys.iteritems():
s3_location = _put_on_s3(localpath, s3_key, s3_account, compress,
gpg_recipient)
os.remove(localpath)
s3_keys.append(s3_location)
if remove_from_sftp:
sftp.open_connection()
sftp.remove(fname)
sftp.close_connection()
return s3_keys
def enqueue_sftp_files(queue_name, sftp_account, sftp_folder, file_regex):
sftp = rowdy.sftp.SFTPConnection(sftp_account.username,
sftp_account.password,
sftp_account.hostname)
sftp.open_connection()
files = sftp.listdir(sftp_folder)
files = [os.path.join(sftp_folder, f) for f in files]
keys = [f for f in files if re.match(file_regex, f)]
sftp.close_connection()
_add_keys_to_upload_queue(keys, queue_name)
def enqueue_sftp_files(queue_name, sftp_account, sftp_folder, file_regex):
sftp = rowdy.sftp.SFTPConnection(sftp_account.username,
sftp_account.password,
sftp_account.hostname)
sftp.open_connection()
files = sftp.listdir(sftp_folder)
files = [os.path.join(sftp_folder, f) for f in files]
keys = [f for f in files if re.match(file_regex, f)]
sftp.close_connection()
_add_keys_to_upload_queue(keys, queue_name)
