def test_get_file_tenant_and_user_name(self): self.assertEqual(FileUtil.get_file_tenant_and_user_name( '/opt/edware/home/landing/arrivals/ca/ca_user1/file_drop/xyz.gz.gpg', '/opt/edware/home/landing/arrivals'), ('ca', 'ca_user1')) self.assertEqual(FileUtil.get_file_tenant_and_user_name( '/opt/edware/home/landing/arrivals/ca/ca_user1/file_drop/xyz.gz.gpg', '/opt/edware/home/landing/arrivals/ca'), (None, None)) self.assertEqual(FileUtil.get_file_tenant_and_user_name( '/opt/edware/home/landing/arrivals/ca/ca_user1/xyz.gz.gpg', '/opt/edware/home/landing/arrivals'), (None, None)) self.assertEqual(FileUtil.get_file_tenant_and_user_name( '/opt/edware/home/landing/arrivals/ca/ca_user1/file_drop/xyz.gz.gpg', '/tmp'), (None, None)) self.assertEqual(FileUtil.get_file_tenant_and_user_name( '/opt/edware/home/landing/arrivals/ca/xyz.gz.gpg', '/opt/edware/home/landing/arrivals'), (None, None))
def move_files_by_sftp(self, files_to_move): ''' move files via sftp :param files_to_move: a list of files to move ''' files_moved = 0 # sort by file length in to get all checksum files sent first before source files files_to_move.sort(key=len) for file in files_to_move: self.logger.debug('SFTPing file: ' + file) file_tenant_name, file_tenant_user_name = \ FileUtil.get_file_tenant_and_user_name(file, os.path.join(self.conf.get(WatcherConst.BASE_DIR), self.conf.get(WatcherConst.SOURCE_DIR))) try: file_transfer_status = \ SendFileUtil.remote_transfer_file( source_file=file, hostname=self.conf.get(MoverConst.LANDING_ZONE_HOSTNAME), remote_base_dir=self.conf.get(MoverConst.ARRIVALS_PATH), file_tenantname=file_tenant_name, file_username=file_tenant_user_name, sftp_username=self.conf.get(MoverConst.SFTP_USER), private_key_file=self.conf.get(MoverConst.PRIVATE_KEY_FILE)) if file_transfer_status == 0: self.logger.debug('File transfer was success for {file}'.format(file=file)) os.remove(file) files_moved += 1 self.logger.debug('Deleted source file {file}'.format(file=file)) except RemoteCopyError as e: self.logger.error('File transfer failed for {file} ' 'with error {error}'.format(file=file, error=str(e))) return files_moved
def move_files_local(self, files_to_move): ''' move files :param files_to_move: a list of files to move ''' files_moved = 0 # sort by file length in to get all checksum files sent first before source files files_to_move.sort(key=len) for file in files_to_move: self.logger.debug('moving file: ' + file) file_tenant_name, file_tenant_user_name = \ FileUtil.get_file_tenant_and_user_name(file, os.path.join(self.conf.get(WatcherConst.BASE_DIR), self.conf.get(WatcherConst.SOURCE_DIR))) try: staging_dir = self.conf.get(WatcherConst.STAGING_DIR) destination_dir = os.path.join(staging_dir, file_tenant_name, file_tenant_user_name) destination_file = os.path.basename(file) destination_partial_file = destination_file + '.partial' destination_partial_file_path = os.path.join(destination_dir, destination_partial_file) destination_file_path = os.path.join(destination_dir, destination_file) os.makedirs(os.path.dirname(destination_file_path), exist_ok=True) os.rename(file, destination_partial_file_path) os.rename(destination_partial_file_path, destination_file_path) files_moved += 1 except Exception as e: self.logger.error('File is failed to move for {file} ' 'with error {error}'.format(file=file, error=str(e))) return files_moved
def test_get_file_tenant_and_user_name(self): self.assertEqual( FileUtil.get_file_tenant_and_user_name( '/opt/edware/home/landing/arrivals/ca/ca_user1/file_drop/xyz.gz.gpg', '/opt/edware/home/landing/arrivals'), ('ca', 'ca_user1')) self.assertEqual( FileUtil.get_file_tenant_and_user_name( '/opt/edware/home/landing/arrivals/ca/ca_user1/file_drop/xyz.gz.gpg', '/opt/edware/home/landing/arrivals/ca'), (None, None)) self.assertEqual( FileUtil.get_file_tenant_and_user_name( '/opt/edware/home/landing/arrivals/ca/ca_user1/xyz.gz.gpg', '/opt/edware/home/landing/arrivals'), (None, None)) self.assertEqual( FileUtil.get_file_tenant_and_user_name( '/opt/edware/home/landing/arrivals/ca/ca_user1/file_drop/xyz.gz.gpg', '/tmp'), (None, None)) self.assertEqual( FileUtil.get_file_tenant_and_user_name( '/opt/edware/home/landing/arrivals/ca/xyz.gz.gpg', '/opt/edware/home/landing/arrivals'), (None, None))