def setUp(self): self.lftp = LFTPSync() self.output_dir = mkdtemp() self.lftp_access = { 'ftp_address': 'smuc.st-and.ac.uk', 'ftp_subdir': '/pub/bodc', 'ftp_user': '******', 'ftp_password': '******', 'ftp_exclude_dir': ['test', 'log'], 'lftp_options': '--only-newer --exclude-glob *.dmp.gz', 'output_dir': self.output_dir } self.setup_lftp_mirror_log()
def main(force_reprocess_all=False): # will sys.exit(-1) if other instance is running me = singleton.SingleInstance() wip_soop_path = os.path.join(os.environ['WIP_DIR'], 'AATAMS_SATTAG_DM') lftp_output_path = os.path.join(wip_soop_path, 'zipped') log_filepath = os.path.join(wip_soop_path, 'aatams_sattag_dm.log') logging = IMOSLogging() logger = logging.logging_start(log_filepath) if not os.path.exists(lftp_output_path): os.makedirs(lftp_output_path) lftp_access = { 'ftp_address': os.environ['IMOS_PO_CREDS_AATAMS_FTP_ADDRESS'], 'ftp_subdir': '/', 'ftp_user': os.environ['IMOS_PO_CREDS_AATAMS_FTP_USERNAME'], 'ftp_password': os.environ['IMOS_PO_CREDS_AATAMS_FTP_PASSWORD'], 'ftp_exclude_dir': '', 'lftp_options': '--only-newer --exclude-glob TDR/* --exclude-glob *_ODV.zip', 'output_dir': lftp_output_path } lftp = LFTPSync() logger.info('Download new AATAMS SATTAG DM files') lftp.lftp_sync(lftp_access) # optional function argument to force the reprocess of all ZIP files if force_reprocess_all: manifest_list = list_files_recursively(lftp_output_path, '*.zip') else: manifest_list = lftp.list_new_files_path(check_file_exist=True) fd, manifest_file = mkstemp() with open(manifest_file, 'w') as f: for zip_file in manifest_list: if not(zip_file == []): f.write('%s\n' % zip_file) os.close(fd) os.chmod(manifest_file, 0o664) # since msktemp creates 600 for security # only create manifest file for non empty files if os.stat(manifest_file).st_size > 0: logger.info('ADD manifest to INCOMING_DIR') manifest_file_inco_path = os.path.join(os.environ['INCOMING_DIR'], 'AATAMS', 'AATAMS_SATTAG_DM', 'aatams_sattag_dm_lftp.%s.manifest' % datetime.datetime.utcnow().strftime('%Y%m%d-%H%M%S')) if not os.path.exists(manifest_file_inco_path): shutil.copy(manifest_file, manifest_file_inco_path) else: logger.warning('File already exist in INCOMING_DIR') exit(1) lftp.close() logging.logging_stop()
def download_lftp_dat_files(): """ lftp download of the SOOP ASF SST. Only the files not in output_dir will be downloaded """ lftp_access = { 'ftp_address': os.environ['IMOS_PO_CREDS_BOM_FTP_ADDRESS'], 'ftp_subdir': '/register/bom404/outgoing/IMOS/SHIPS', 'ftp_user': os.environ['IMOS_PO_CREDS_BOM_FTP_USERNAME'], 'ftp_password': os.environ['IMOS_PO_CREDS_BOM_FTP_PASSWORD'], 'ftp_exclude_dir': '', 'lftp_options': '--only-newer', 'output_dir': output_data_folder } global lftp lftp = LFTPSync() if os.path.exists(os.path.join(output_data_folder, 'lftp_mirror.log')): return lftp.list_new_files_path_previous_log(lftp_access) lftp.lftp_sync(lftp_access) return lftp.list_new_files_path(check_file_exist=True)
class TestGenerateNetCDFAtt(unittest.TestCase): def setUp(self): self.lftp = LFTPSync() self.output_dir = mkdtemp() self.lftp_access = { 'ftp_address': 'smuc.st-and.ac.uk', 'ftp_subdir': '/pub/bodc', 'ftp_user': '******', 'ftp_password': '******', 'ftp_exclude_dir': ['test', 'log'], 'lftp_options': '--only-newer --exclude-glob *.dmp.gz', 'output_dir': self.output_dir } self.setup_lftp_mirror_log() def setup_lftp_mirror_log(self): self.file1_lftp = [mkstemp(dir=self.output_dir)] self.file2_lftp = [mkstemp(dir=self.output_dir)] lftp_log = textwrap.dedent(""" get -O %s ftp://:@smuc.st-and.ac.uk/pub/bodc/%s get -O %s ftp://:@smuc.st-and.ac.uk/pub/bodc/%s """ % (self.output_dir, os.path.basename(self.file1_lftp[0][1]), self.output_dir, os.path.basename(self.file2_lftp[0][1]))) self.log_file = os.path.join(self.output_dir, 'lftp_mirror.log') with open(self.log_file, 'w') as f: f.write(lftp_log) def tearDown(self): os.close(self.file1_lftp[0][0]) os.close(self.file2_lftp[0][0]) shutil.rmtree(self.output_dir) def test_list_new_files_path_previous_log(self): res = self.lftp.list_new_files_path_previous_log(self.lftp_access, check_file_exist=True) # check the 2 files were seen by the log function self.assertEqual(res[0], self.file1_lftp[0][1]) self.assertEqual(res[1], self.file2_lftp[0][1])
def download_lftp_dat_files(): """ lftp download of the GTS NRT AATAMS files. Only the files not in OUTPUT_DIR will be downloaded """ lftp_access = { 'ftp_address': 'smuc.st-and.ac.uk', 'ftp_subdir': '/pub/bodc', 'ftp_user': '', 'ftp_password': '', 'ftp_exclude_dir': ['corrections', 'exports', 'test'], 'lftp_options': '--only-newer --exclude-glob *.dmp.gz', 'output_dir': OUTPUT_DIR, } global lftp lftp = LFTPSync() if os.path.exists(os.path.join(OUTPUT_DIR, 'lftp_mirror.log')): return lftp.list_new_files_path_previous_log(lftp_access) lftp.lftp_sync(lftp_access) return lftp.list_new_files_path(check_file_exist=True)
def main(force_reprocess_all=False): # will sys.exit(-1) if other instance is running me = singleton.SingleInstance() wip_soop_path = os.path.join(os.environ['WIP_DIR'], 'SOOP', 'SOOP_XBT_ASF_SST') lftp_output_path = os.path.join(wip_soop_path, 'data_unsorted', 'XBT', 'sbddata') csv_output_path = os.path.join(wip_soop_path, 'data_sorted', 'XBT', 'sbddata') log_filepath = os.path.join(wip_soop_path, 'soop_xbt.log') logging = IMOSLogging() logger = logging.logging_start(log_filepath) lftp_access = { 'ftp_address': os.environ['IMOS_PO_CREDS_CSIRO_IT_FTP_ADDRESS'], 'ftp_subdir': '/', 'ftp_user': os.environ['IMOS_PO_CREDS_CSIRO_IT_FTP_USERNAME'], 'ftp_password': os.environ['IMOS_PO_CREDS_CSIRO_IT_FTP_PASSWORD'], 'ftp_exclude_dir': '', 'lftp_options': '--only-newer', 'output_dir': lftp_output_path } lftp = LFTPSync() logger.info('Download new SOOP XBT NRT files') lftp.lftp_sync(lftp_access) # optional function argument to force the reprocess of all sbd files if force_reprocess_all: list_new_files = list_files_recursively(lftp_output_path, '*.sbd') else: list_new_files = lftp.list_new_files_path(check_file_exist=True) logger.info('Convert SBD files to CSV') processSBD = soop_xbt_realtime_processSBD() manifest_list = [] for f in list_new_files: if f.endswith(".sbd"): try: csv_file = processSBD.handle_sbd_file(f, csv_output_path) if csv_file not in manifest_list: manifest_list.append(csv_file) except Exception as err: logger.error(str(err)) pass fd, manifest_file = mkstemp() for csv_file in manifest_list: if not (csv_file == []): os.write(fd, '%s\n' % csv_file) os.close(fd) os.chmod(manifest_file, 0o664) # since msktemp creates 600 for security logger.info('ADD manifest to INCOMING_DIR') manifest_file_inco_path = os.path.join( os.environ['INCOMING_DIR'], 'SOOP', 'XBT', 'NRT', 'IMOS_SOOP-XBT_NRT_fileList.manifest') if not os.path.exists(manifest_file_inco_path): shutil.copy(manifest_file, manifest_file_inco_path) else: logger.warning('File already exist in INCOMING_DIR') exit(1) lftp.close() logging.logging_stop()