def test_get_file_tenant_and_user_name(self):
self.assertEqual(FileUtil.get_file_tenant_and_user_name(
'/opt/edware/home/landing/arrivals/ca/ca_user1/file_drop/xyz.gz.gpg',
'/opt/edware/home/landing/arrivals'), ('ca', 'ca_user1'))
self.assertEqual(FileUtil.get_file_tenant_and_user_name(
'/opt/edware/home/landing/arrivals/ca/ca_user1/file_drop/xyz.gz.gpg',
'/opt/edware/home/landing/arrivals/ca'), (None, None))
self.assertEqual(FileUtil.get_file_tenant_and_user_name(
'/opt/edware/home/landing/arrivals/ca/ca_user1/xyz.gz.gpg',
'/opt/edware/home/landing/arrivals'), (None, None))
self.assertEqual(FileUtil.get_file_tenant_and_user_name(
'/opt/edware/home/landing/arrivals/ca/ca_user1/file_drop/xyz.gz.gpg',
'/tmp'), (None, None))
self.assertEqual(FileUtil.get_file_tenant_and_user_name(
'/opt/edware/home/landing/arrivals/ca/xyz.gz.gpg',
'/opt/edware/home/landing/arrivals'), (None, None))
def move_files_by_sftp(self, files_to_move):
'''
move files via sftp
:param files_to_move: a list of files to move
'''
files_moved = 0
# sort by file length in to get all checksum files sent first before source files
files_to_move.sort(key=len)
for file in files_to_move:
self.logger.debug('SFTPing file: ' + file)
file_tenant_name, file_tenant_user_name = \
FileUtil.get_file_tenant_and_user_name(file, os.path.join(self.conf.get(WatcherConst.BASE_DIR),
self.conf.get(WatcherConst.SOURCE_DIR)))
try:
file_transfer_status = \
SendFileUtil.remote_transfer_file(
source_file=file, hostname=self.conf.get(MoverConst.LANDING_ZONE_HOSTNAME),
remote_base_dir=self.conf.get(MoverConst.ARRIVALS_PATH), file_tenantname=file_tenant_name,
file_username=file_tenant_user_name, sftp_username=self.conf.get(MoverConst.SFTP_USER),
private_key_file=self.conf.get(MoverConst.PRIVATE_KEY_FILE))
if file_transfer_status == 0:
self.logger.debug('File transfer was success for {file}'.format(file=file))
os.remove(file)
files_moved += 1
self.logger.debug('Deleted source file {file}'.format(file=file))
except RemoteCopyError as e:
self.logger.error('File transfer failed for {file} '
'with error {error}'.format(file=file, error=str(e)))
return files_moved
def move_files_local(self, files_to_move):
'''
move files
:param files_to_move: a list of files to move
'''
files_moved = 0
# sort by file length in to get all checksum files sent first before source files
files_to_move.sort(key=len)
for file in files_to_move:
self.logger.debug('moving file: ' + file)
file_tenant_name, file_tenant_user_name = \
FileUtil.get_file_tenant_and_user_name(file, os.path.join(self.conf.get(WatcherConst.BASE_DIR),
self.conf.get(WatcherConst.SOURCE_DIR)))
try:
staging_dir = self.conf.get(WatcherConst.STAGING_DIR)
destination_dir = os.path.join(staging_dir, file_tenant_name, file_tenant_user_name)
destination_file = os.path.basename(file)
destination_partial_file = destination_file + '.partial'
destination_partial_file_path = os.path.join(destination_dir, destination_partial_file)
destination_file_path = os.path.join(destination_dir, destination_file)
os.makedirs(os.path.dirname(destination_file_path), exist_ok=True)
os.rename(file, destination_partial_file_path)
os.rename(destination_partial_file_path, destination_file_path)
files_moved += 1
except Exception as e:
self.logger.error('File is failed to move for {file} '
'with error {error}'.format(file=file, error=str(e)))
return files_moved
def test_get_file_tenant_and_user_name(self):
self.assertEqual(
FileUtil.get_file_tenant_and_user_name(
'/opt/edware/home/landing/arrivals/ca/ca_user1/file_drop/xyz.gz.gpg',
'/opt/edware/home/landing/arrivals'), ('ca', 'ca_user1'))
self.assertEqual(
FileUtil.get_file_tenant_and_user_name(
'/opt/edware/home/landing/arrivals/ca/ca_user1/file_drop/xyz.gz.gpg',
'/opt/edware/home/landing/arrivals/ca'), (None, None))
self.assertEqual(
FileUtil.get_file_tenant_and_user_name(
'/opt/edware/home/landing/arrivals/ca/ca_user1/xyz.gz.gpg',
'/opt/edware/home/landing/arrivals'), (None, None))
self.assertEqual(
FileUtil.get_file_tenant_and_user_name(
'/opt/edware/home/landing/arrivals/ca/ca_user1/file_drop/xyz.gz.gpg',
'/tmp'), (None, None))
self.assertEqual(
FileUtil.get_file_tenant_and_user_name(
'/opt/edware/home/landing/arrivals/ca/xyz.gz.gpg',
'/opt/edware/home/landing/arrivals'), (None, None))