def _put_sftp_file_on_s3(fname, s3_account, s3_directory, sftp_account, remove_from_sftp=False, compress=True, gpg_recipient=None): sftp = rowdy.sftp.SFTPConnection(sftp_account.username, sftp_account.password, sftp_account.hostname) _, tempfile = mkstemp() sftp.open_connection() sftp.get(fname, tempfile) sftp.close_connection() s3_key = os.path.join(s3_directory, os.path.basename(fname)) keys = { tempfile: s3_key, } s3_keys = [] for localpath, s3_key in keys.iteritems(): s3_location = _put_on_s3(localpath, s3_key, s3_account, compress, gpg_recipient) os.remove(localpath) s3_keys.append(s3_location) if remove_from_sftp: sftp.open_connection() sftp.remove(fname) sftp.close_connection() return s3_keys
def enqueue_sftp_files(queue_name, sftp_account, sftp_folder, file_regex): sftp = rowdy.sftp.SFTPConnection(sftp_account.username, sftp_account.password, sftp_account.hostname) sftp.open_connection() files = sftp.listdir(sftp_folder) files = [os.path.join(sftp_folder, f) for f in files] keys = [f for f in files if re.match(file_regex, f)] sftp.close_connection() _add_keys_to_upload_queue(keys, queue_name)