def setUp(self):
self.lftp = LFTPSync()
self.output_dir = mkdtemp()
self.lftp_access = {
'ftp_address': 'smuc.st-and.ac.uk',
'ftp_subdir': '/pub/bodc',
'ftp_user': '******',
'ftp_password': '******',
'ftp_exclude_dir': ['test', 'log'],
'lftp_options': '--only-newer --exclude-glob *.dmp.gz',
'output_dir': self.output_dir
}
self.setup_lftp_mirror_log()
def main(force_reprocess_all=False):
# will sys.exit(-1) if other instance is running
me = singleton.SingleInstance()
wip_soop_path = os.path.join(os.environ['WIP_DIR'], 'AATAMS_SATTAG_DM')
lftp_output_path = os.path.join(wip_soop_path, 'zipped')
log_filepath = os.path.join(wip_soop_path, 'aatams_sattag_dm.log')
logging = IMOSLogging()
logger = logging.logging_start(log_filepath)
if not os.path.exists(lftp_output_path):
os.makedirs(lftp_output_path)
lftp_access = {
'ftp_address': os.environ['IMOS_PO_CREDS_AATAMS_FTP_ADDRESS'],
'ftp_subdir': '/',
'ftp_user': os.environ['IMOS_PO_CREDS_AATAMS_FTP_USERNAME'],
'ftp_password': os.environ['IMOS_PO_CREDS_AATAMS_FTP_PASSWORD'],
'ftp_exclude_dir': '',
'lftp_options': '--only-newer --exclude-glob TDR/* --exclude-glob *_ODV.zip',
'output_dir': lftp_output_path
}
lftp = LFTPSync()
logger.info('Download new AATAMS SATTAG DM files')
lftp.lftp_sync(lftp_access)
# optional function argument to force the reprocess of all ZIP files
if force_reprocess_all:
manifest_list = list_files_recursively(lftp_output_path, '*.zip')
else:
manifest_list = lftp.list_new_files_path(check_file_exist=True)
fd, manifest_file = mkstemp()
with open(manifest_file, 'w') as f:
for zip_file in manifest_list:
if not(zip_file == []):
f.write('%s\n' % zip_file)
os.close(fd)
os.chmod(manifest_file, 0o664) # since msktemp creates 600 for security
# only create manifest file for non empty files
if os.stat(manifest_file).st_size > 0:
logger.info('ADD manifest to INCOMING_DIR')
manifest_file_inco_path = os.path.join(os.environ['INCOMING_DIR'], 'AATAMS',
'AATAMS_SATTAG_DM',
'aatams_sattag_dm_lftp.%s.manifest' % datetime.datetime.utcnow().strftime('%Y%m%d-%H%M%S'))
if not os.path.exists(manifest_file_inco_path):
shutil.copy(manifest_file, manifest_file_inco_path)
else:
logger.warning('File already exist in INCOMING_DIR')
exit(1)
lftp.close()
logging.logging_stop()
def download_lftp_dat_files():
"""
lftp download of the SOOP ASF SST. Only the files not in
output_dir will be downloaded
"""
lftp_access = {
'ftp_address': os.environ['IMOS_PO_CREDS_BOM_FTP_ADDRESS'],
'ftp_subdir': '/register/bom404/outgoing/IMOS/SHIPS',
'ftp_user': os.environ['IMOS_PO_CREDS_BOM_FTP_USERNAME'],
'ftp_password': os.environ['IMOS_PO_CREDS_BOM_FTP_PASSWORD'],
'ftp_exclude_dir': '',
'lftp_options': '--only-newer',
'output_dir': output_data_folder
}
global lftp
lftp = LFTPSync()
if os.path.exists(os.path.join(output_data_folder, 'lftp_mirror.log')):
return lftp.list_new_files_path_previous_log(lftp_access)
lftp.lftp_sync(lftp_access)
return lftp.list_new_files_path(check_file_exist=True)
class TestGenerateNetCDFAtt(unittest.TestCase):
def setUp(self):
self.lftp = LFTPSync()
self.output_dir = mkdtemp()
self.lftp_access = {
'ftp_address': 'smuc.st-and.ac.uk',
'ftp_subdir': '/pub/bodc',
'ftp_user': '******',
'ftp_password': '******',
'ftp_exclude_dir': ['test', 'log'],
'lftp_options': '--only-newer --exclude-glob *.dmp.gz',
'output_dir': self.output_dir
}
self.setup_lftp_mirror_log()
def setup_lftp_mirror_log(self):
self.file1_lftp = [mkstemp(dir=self.output_dir)]
self.file2_lftp = [mkstemp(dir=self.output_dir)]
lftp_log = textwrap.dedent("""
get -O %s ftp://:@smuc.st-and.ac.uk/pub/bodc/%s
get -O %s ftp://:@smuc.st-and.ac.uk/pub/bodc/%s
""" % (self.output_dir, os.path.basename(self.file1_lftp[0][1]),
self.output_dir, os.path.basename(self.file2_lftp[0][1])))
self.log_file = os.path.join(self.output_dir, 'lftp_mirror.log')
with open(self.log_file, 'w') as f:
f.write(lftp_log)
def tearDown(self):
os.close(self.file1_lftp[0][0])
os.close(self.file2_lftp[0][0])
shutil.rmtree(self.output_dir)
def test_list_new_files_path_previous_log(self):
res = self.lftp.list_new_files_path_previous_log(self.lftp_access,
check_file_exist=True)
# check the 2 files were seen by the log function
self.assertEqual(res[0], self.file1_lftp[0][1])
self.assertEqual(res[1], self.file2_lftp[0][1])
def download_lftp_dat_files():
"""
lftp download of the SOOP ASF SST. Only the files not in
output_dir will be downloaded
"""
lftp_access = {
'ftp_address': os.environ['IMOS_PO_CREDS_BOM_FTP_ADDRESS'],
'ftp_subdir': '/register/bom404/outgoing/IMOS/SHIPS',
'ftp_user': os.environ['IMOS_PO_CREDS_BOM_FTP_USERNAME'],
'ftp_password': os.environ['IMOS_PO_CREDS_BOM_FTP_PASSWORD'],
'ftp_exclude_dir': '',
'lftp_options': '--only-newer',
'output_dir': output_data_folder
}
global lftp
lftp = LFTPSync()
if os.path.exists(os.path.join(output_data_folder, 'lftp_mirror.log')):
return lftp.list_new_files_path_previous_log(lftp_access)
lftp.lftp_sync(lftp_access)
return lftp.list_new_files_path(check_file_exist=True)
def download_lftp_dat_files():
"""
lftp download of the GTS NRT AATAMS files. Only the files not in
OUTPUT_DIR will be downloaded
"""
lftp_access = {
'ftp_address': 'smuc.st-and.ac.uk',
'ftp_subdir': '/pub/bodc',
'ftp_user': '',
'ftp_password': '',
'ftp_exclude_dir': ['corrections', 'exports', 'test'],
'lftp_options': '--only-newer --exclude-glob *.dmp.gz',
'output_dir': OUTPUT_DIR,
}
global lftp
lftp = LFTPSync()
if os.path.exists(os.path.join(OUTPUT_DIR, 'lftp_mirror.log')):
return lftp.list_new_files_path_previous_log(lftp_access)
lftp.lftp_sync(lftp_access)
return lftp.list_new_files_path(check_file_exist=True)
def main(force_reprocess_all=False):
# will sys.exit(-1) if other instance is running
me = singleton.SingleInstance()
wip_soop_path = os.path.join(os.environ['WIP_DIR'], 'SOOP',
'SOOP_XBT_ASF_SST')
lftp_output_path = os.path.join(wip_soop_path, 'data_unsorted', 'XBT',
'sbddata')
csv_output_path = os.path.join(wip_soop_path, 'data_sorted', 'XBT',
'sbddata')
log_filepath = os.path.join(wip_soop_path, 'soop_xbt.log')
logging = IMOSLogging()
logger = logging.logging_start(log_filepath)
lftp_access = {
'ftp_address': os.environ['IMOS_PO_CREDS_CSIRO_IT_FTP_ADDRESS'],
'ftp_subdir': '/',
'ftp_user': os.environ['IMOS_PO_CREDS_CSIRO_IT_FTP_USERNAME'],
'ftp_password': os.environ['IMOS_PO_CREDS_CSIRO_IT_FTP_PASSWORD'],
'ftp_exclude_dir': '',
'lftp_options': '--only-newer',
'output_dir': lftp_output_path
}
lftp = LFTPSync()
logger.info('Download new SOOP XBT NRT files')
lftp.lftp_sync(lftp_access)
# optional function argument to force the reprocess of all sbd files
if force_reprocess_all:
list_new_files = list_files_recursively(lftp_output_path, '*.sbd')
else:
list_new_files = lftp.list_new_files_path(check_file_exist=True)
logger.info('Convert SBD files to CSV')
processSBD = soop_xbt_realtime_processSBD()
manifest_list = []
for f in list_new_files:
if f.endswith(".sbd"):
try:
csv_file = processSBD.handle_sbd_file(f, csv_output_path)
if csv_file not in manifest_list:
manifest_list.append(csv_file)
except Exception as err:
logger.error(str(err))
pass
fd, manifest_file = mkstemp()
for csv_file in manifest_list:
if not (csv_file == []):
os.write(fd, '%s\n' % csv_file)
os.close(fd)
os.chmod(manifest_file, 0o664) # since msktemp creates 600 for security
logger.info('ADD manifest to INCOMING_DIR')
manifest_file_inco_path = os.path.join(
os.environ['INCOMING_DIR'], 'SOOP', 'XBT', 'NRT',
'IMOS_SOOP-XBT_NRT_fileList.manifest')
if not os.path.exists(manifest_file_inco_path):
shutil.copy(manifest_file, manifest_file_inco_path)
else:
logger.warning('File already exist in INCOMING_DIR')
exit(1)
lftp.close()
logging.logging_stop()