def setUp(self):
self.logfile = [mkstemp()]
self.logging = IMOSLogging()
self.logger = self.logging.logging_start(self.logfile[0][1])
self.logger.info('info')
self.logger.warning('warning')
self.logger.error('error')
def main(incoming_file_path, deployment_code, output_dir):
global logger
global temporal_res_in_minutes
global vertical_res_in_metres
temporal_res_in_minutes = 60.0
vertical_res_in_metres = 1 # has to be an integer since used in range() later
fv02_nc_path = None
logging = IMOSLogging()
logger = logging.logging_start(
os.path.join(output_dir, '%s.log' % deployment_code))
list_fv01_url = wfs_request_matching_file_pattern('anmn_ts_timeseries_map',
'%%_FV01_%s%%' %
deployment_code,
s3_bucket_url=True,
url_column='file_url')
previous_fv02_url = wfs_request_matching_file_pattern(
'anmn_all_map',
'%%Temperature/gridded/%%_FV02_%s_%%gridded%%' % deployment_code)
if len(previous_fv02_url) == 1:
previous_fv02_url = previous_fv02_url[0]
logger.info("Downloading files:\n%s" % "\n".join(
map(str, [os.path.basename(fv01_url) for fv01_url in list_fv01_url])))
fv01_list_dir = download_list_urls(list_fv01_url)
nc_fv01_list = get_usable_fv01_list(fv01_list_dir)
if len(nc_fv01_list) < 2:
logger.error('Not enough FV01 files to create product.')
else:
fv02_nc_path = create_fv02_product(nc_fv01_list, output_dir)
return fv02_nc_path, previous_fv02_url, fv01_list_dir
class TestImosLogging(unittest.TestCase):
def setUp(self):
self.logfile = [mkstemp()]
self.logging = IMOSLogging()
self.logger = self.logging.logging_start(self.logfile[0][1])
self.logger.info('info')
self.logger.warning('warning')
self.logger.error('error')
def tearDown(self):
self.logging.logging_stop()
os.close(self.logfile[0][0])
os.remove(self.logfile[0][1])
def test_imos_logging_msg(self):
with open(self.logfile[0][1], 'r') as f:
for line in f:
# the assert val is done is non conform way as the entire string
# can be checked because of the time information added by the
# logger
if 'imos_logging - INFO - info' in line:
self.assertEqual(0, 0)
elif 'imos_logging - WARNING - warning' in line:
self.assertEqual(0, 0)
elif 'imos_logging - ERROR - error' in line:
self.assertEqual(0, 0)
else:
self.assertEqual(1, 0)
def main(incoming_file_path, deployment_code, output_dir):
global logger
global temporal_res_in_minutes
global vertical_res_in_metres
temporal_res_in_minutes = 60.0
vertical_res_in_metres = 1 # has to be an integer since used in range() later
fv02_nc_path = None
logging = IMOSLogging()
logger = logging.logging_start(os.path.join(output_dir, '%s.log' % deployment_code))
list_fv01_url = wfs_request_matching_file_pattern('anmn_ts_timeseries_map', '%%_FV01_%s%%' % deployment_code, s3_bucket_url=True, url_column='file_url')
previous_fv02_url = wfs_request_matching_file_pattern('anmn_all_map', '%%Temperature/gridded/%%_FV02_%s_%%gridded%%' % deployment_code)
if len(previous_fv02_url) == 1:
previous_fv02_url = previous_fv02_url[0]
logger.info("Downloading files:\n%s" % "\n".join(map(str, [os.path.basename(fv01_url) for fv01_url in list_fv01_url])))
fv01_list_dir = download_list_urls(list_fv01_url)
nc_fv01_list = get_usable_fv01_list(fv01_list_dir)
if len(nc_fv01_list) < 2:
logger.error('Not enough FV01 files to create product.')
else:
fv02_nc_path = create_fv02_product(nc_fv01_list, output_dir)
return fv02_nc_path, previous_fv02_url, fv01_list_dir
class TestImosLogging(unittest.TestCase):
def setUp(self):
self.logfile = [mkstemp()]
self.logging = IMOSLogging()
self.logger = self.logging.logging_start(self.logfile[0][1])
self.logger.info('info')
self.logger.warning('warning')
self.logger.error('error')
def tearDown(self):
self.logging.logging_stop()
os.close(self.logfile[0][0])
os.remove(self.logfile[0][1])
def test_imos_logging_msg(self):
with open(self.logfile[0][1], 'r') as f:
for line in f:
# the assert val is done is non conform way as the entire string
# can be checked because of the time information added by the
# logger
if 'imos_logging - INFO - info' in line:
self.assertEqual(0, 0)
elif 'imos_logging - WARNING - warning' in line:
self.assertEqual(0, 0)
elif 'imos_logging - ERROR - error' in line:
self.assertEqual(0, 0)
else:
self.assertEqual(1, 0)
def main(force_reprocess_all=False, manifest=True):
if not os.path.exists(OUTPUT_DIR):
os.makedirs(OUTPUT_DIR)
# setup logging
log_filepath = os.path.join(OUTPUT_DIR, 'aatams_nrt.log')
logging = IMOSLogging()
global logger
logger = logging.logging_start(log_filepath)
logger.info('Process AATAMS NRT')
# force the recreation of all netcdf file and push to incoming
if force_reprocess_all:
list_new_dat_gz_files = list_files_recursively(OUTPUT_DIR, '*.dat.gz')
else:
list_new_dat_gz_files = download_lftp_dat_files()
lftp.close()
dat_files = extract_dat_gz_files(list_new_dat_gz_files)
australian_tag_list = parse_australian_tags_file()
netcdf_file_path_set = set()
for dat_file in dat_files:
logger.info('Processing %s' % dat_file)
dat_file_parsed = parse_dat_file(dat_file)
index_profiles_start = separate_individual_profiles_from_dat(
dat_file_parsed)
for idx, profile in enumerate(index_profiles_start[:-1]):
profile_data = individual_profile_data(
dat_file_parsed, profile, index_profiles_start[idx + 1])
if is_profile_australian(profile_data, australian_tag_list):
extra_atts = get_extra_profile_att(profile_data,
australian_tag_list)
netcdf_file_path = create_netcdf_profile(
profile_data, extra_atts)
netcdf_file_path_set.add(netcdf_file_path)
else:
logger.warning(
("%s wmo is not an Australian tag/is not in "
"aatams_sattag_metadata.csv") % profile_data[0])
os.remove(dat_file)
# moves manifest_file or netcdf files to incoming. default is netcdf file
if not manifest:
for file in netcdf_file_path_set:
move_to_incoming(file)
else:
manifest_file = os.path.join(OUTPUT_DIR, 'manifest')
with open(manifest_file, 'w') as f:
for file in netcdf_file_path_set:
f.write("%s\n" % file)
move_to_incoming(manifest_file)
logging.logging_stop()
def main(force_reprocess_all=False):
# will sys.exit(-1) if other instance is running
me = singleton.SingleInstance()
wip_soop_path = os.path.join(os.environ['WIP_DIR'], 'AATAMS_SATTAG_DM')
lftp_output_path = os.path.join(wip_soop_path, 'zipped')
log_filepath = os.path.join(wip_soop_path, 'aatams_sattag_dm.log')
logging = IMOSLogging()
logger = logging.logging_start(log_filepath)
if not os.path.exists(lftp_output_path):
os.makedirs(lftp_output_path)
lftp_access = {
'ftp_address': os.environ['IMOS_PO_CREDS_AATAMS_FTP_ADDRESS'],
'ftp_subdir': '/',
'ftp_user': os.environ['IMOS_PO_CREDS_AATAMS_FTP_USERNAME'],
'ftp_password': os.environ['IMOS_PO_CREDS_AATAMS_FTP_PASSWORD'],
'ftp_exclude_dir': '',
'lftp_options': '--only-newer --exclude-glob TDR/* --exclude-glob *_ODV.zip',
'output_dir': lftp_output_path
}
lftp = LFTPSync()
logger.info('Download new AATAMS SATTAG DM files')
lftp.lftp_sync(lftp_access)
# optional function argument to force the reprocess of all ZIP files
if force_reprocess_all:
manifest_list = list_files_recursively(lftp_output_path, '*.zip')
else:
manifest_list = lftp.list_new_files_path(check_file_exist=True)
fd, manifest_file = mkstemp()
with open(manifest_file, 'w') as f:
for zip_file in manifest_list:
if not(zip_file == []):
f.write('%s\n' % zip_file)
os.close(fd)
os.chmod(manifest_file, 0o664) # since msktemp creates 600 for security
# only create manifest file for non empty files
if os.stat(manifest_file).st_size > 0:
logger.info('ADD manifest to INCOMING_DIR')
manifest_file_inco_path = os.path.join(os.environ['INCOMING_DIR'], 'AATAMS',
'AATAMS_SATTAG_DM',
'aatams_sattag_dm_lftp.%s.manifest' % datetime.datetime.utcnow().strftime('%Y%m%d-%H%M%S'))
if not os.path.exists(manifest_file_inco_path):
shutil.copy(manifest_file, manifest_file_inco_path)
else:
logger.warning('File already exist in INCOMING_DIR')
exit(1)
lftp.close()
logging.logging_stop()
def global_vars(vargs):
global LOGGER
logging = IMOSLogging()
LOGGER = logging.logging_start(vargs.log_file)
global NETCDF_FILE_PATH # defined as glob to be used in exception
global SHIP_CALL_SIGN_LIST
SHIP_CALL_SIGN_LIST = ship_callsign_list() # AODN CALLSIGN vocabulary
global XBT_LINE_INFO
XBT_LINE_INFO = xbt_line_info()
def setUp(self):
self.logfile = [mkstemp()]
self.logging = IMOSLogging()
self.logger = self.logging.logging_start(self.logfile[0][1])
self.logger.info('info')
self.logger.warning('warning')
self.logger.error('error')
required=False)
vargs = parser.parse_args()
if vargs.output_path is None:
vargs.output_path = tempfile.mkdtemp()
if not os.path.exists(vargs.output_path):
raise ValueError('{path} not a valid path'.format(path=vargs.output_path))
sys.exit(1)
return vargs
if __name__ == "__main__":
"""
Processing of the full BOM WAVE DM dataset
./bom_wave_dm_process -i $ARCHIVE_DIR/AODN/BOM_WAVE_DM
"""
vargs = args()
station_ls = filter(lambda f: os.path.isdir(f), glob.glob('{dir}/*'.format(dir=vargs.dataset_path)))
global logger
logger = IMOSLogging().logging_start(os.path.join(vargs.output_path, 'process.log'))
for station_path in station_ls:
logger.info('Processing WAVES for {station_path}'.format(station_path=os.path.basename(station_path)))
process_station(station_path, vargs.output_path)
logger.info('End of processing')
if not os.path.exists(vargs.output_path):
raise ValueError('{path} not a valid path'.format(path=vargs.output_path))
sys.exit(1)
return vargs
if __name__ == "__main__":
"""
Processing of the full WAVERIDER dataset from DOT-WA
./wa_waverider_process
"""
vargs = args()
global logger
logger = IMOSLogging().logging_start(os.path.join(vargs.output_path, 'process.log'))
if not os.path.exists(WIP_DIR):
os.makedirs(WIP_DIR)
sites_info = retrieve_sites_info_waverider_kml()
for _, id in enumerate(sites_info):
site_info = sites_info[id]
logger.info('Processing WAVES for id: {id} {site_path}'.format(id=id,
site_path=site_info['site_name']))
temporary_data_path, site_info = download_site_data(site_info) # returned site_info has extra md5 info from zip
try:
if site_info['already_uptodate']:
logger.info('{site_path} already up to date'.format(site_path=site_info['site_name']))
shutil.rmtree(temporary_data_path)
continue
def parse_arg():
"""
create optional script arg
"""
parser = argparse.ArgumentParser()
parser.add_argument("-c", "--campaign-path", type=str, help='campaign path', required=True)
parser.add_argument("-n", "--no-thumbnail-creation", help="process or reprocess campaign without the creation of thumbnails", action="store_false", required=False)
parser.add_argument("-p", "--push-to-incoming", help="push output data, and ALL AUV CAMPAIGN data to incoming dir for pipeline processing", action="store_true", required=False)
args = parser.parse_args()
args.campaign_path = args.campaign_path.rstrip("//")
return args
if __name__ == '__main__':
""" example:
auv_processing.py -c /vagrant/src/PS201502 -n -> no creation of thumbnails
auv_processing.py -c /vagrant/src/PS201502 -p -> full process of campaign and push to incoming ALL data(viewer plus campaign data)
"""
os.umask(0o002)
# setup logging
log_filepath = os.path.join(AUV_WIP_DIR, 'auv.log')
logging = IMOSLogging()
global logger
logger = logging.logging_start(log_filepath)
args = parse_arg()
output_data = process_campaign(args.campaign_path,
create_thumbnail=args.no_thumbnail_creation,
push_data_to_incoming=args.push_to_incoming)
required=False)
parser.add_argument(
"-p",
"--push-to-incoming",
help=
"push output data, and ALL AUV CAMPAIGN data to incoming dir for pipeline processing",
action="store_true",
required=False)
args = parser.parse_args()
args.campaign_path = args.campaign_path.rstrip("//")
return args
if __name__ == '__main__':
""" example:
auv_processing.py -c /vagrant/src/PS201502 -n -> no creation of thumbnails
auv_processing.py -c /vagrant/src/PS201502 -p -> full process of campaign and push to incoming ALL data(viewer plus campaign data)
"""
os.umask(0o002)
# setup logging
log_filepath = os.path.join(AUV_WIP_DIR, 'auv.log')
logging = IMOSLogging()
global logger
logger = logging.logging_start(log_filepath)
args = parse_arg()
process_campaign(args.campaign_path,
create_thumbnail=args.no_thumbnail_creation,
push_data_to_incoming=args.push_to_incoming)
except Exception:
raise ValueError('{path} not a valid path'.format(path=vargs.output_path))
sys.exit(1)
return vargs
if __name__ == "__main__":
"""
Processing of the full WAVERIDER dataset from DOT-WA
./wa_waverider_process
"""
vargs = args()
global logger
logger = IMOSLogging().logging_start(os.path.join(vargs.output_path, 'process.log'))
if not os.path.exists(WIP_DIR):
os.makedirs(WIP_DIR)
sites_info = retrieve_sites_info_waverider_kml()
for _, id in enumerate(sites_info):
site_info = sites_info[id]
logger.info('Processing WAVES for id: {id} {site_path}'.format(id=id,
site_path=site_info['site_name']))
temporary_data_path, site_info = download_site_data(site_info) # returned site_info has extra md5 info from zip
try:
if site_info['already_uptodate']:
logger.info('{site_path} already up to date'.format(site_path=site_info['site_name']))
shutil.rmtree(temporary_data_path)
continue
if not os.path.exists(vargs.output_path):
raise ValueError('{path} not a valid path'.format(path=vargs.output_path))
sys.exit(1)
return vargs
if __name__ == "__main__":
"""
Processing of the full WA dataset
./wa_awac_process
"""
vargs = args()
global logger
logger = IMOSLogging().logging_start(os.path.join(vargs.output_path, 'process.log'))
if not os.path.exists(WIP_DIR):
os.makedirs(WIP_DIR)
sites_info = retrieve_sites_info_awac_kml()
for _, site_code in enumerate(sites_info):
site_info = sites_info[site_code]
temporary_data_path, site_info = download_site_data(site_info) # returned site_info has extra md5 info
site_name = site_info['site_code']
try:
if site_info['already_uptodate']:
logger.info('{site_path} already up to date'.format(site_path=site_name))
shutil.rmtree(temporary_data_path)
continue
replaced_data = filedata.replace("N/A", "")
os.umask(0o002)
f = open(data_file, 'w')
f.write(replaced_data)
f.close()
if os.path.exists(download_dir):
shutil.move(data_file, os.path.join(download_dir, 'Lucinda.lev20'))
else:
logger.error('%s does not exists' % download_dir)
shutil.rmtree(temp_dir)
if __name__ == "__main__":
logging = IMOSLogging()
log_file = [mkstemp()]
global logger
logger = logging.logging_start(log_file[0][1])
try:
download_ljco_aeronet(sys.argv[1])
except Exception as err:
print(err)
logging.logging_stop()
os.close(log_file[0][0])
os.remove(log_file[0][1])
try:
nc_file_path.append(generate_qld_netcdf(resource_id, metadata, output_dir_path))
except Exception as err:
logger.error('Issue processing ressource_id {id}'.format(id=resource_id))
logger.error('{err}'.format(err=err))
logger.error(traceback.print_exc())
return set(nc_file_path)
if __name__ == '__main__':
vargs = args()
if not os.path.exists(WIP_DIR):
os.makedirs(WIP_DIR)
logging = IMOSLogging()
global logger
logger = logging.logging_start(os.path.join(WIP_DIR, 'process.log'))
package_names = list_package_names()
nc_wip_dir = os.path.join(WIP_DIR, 'NetCDF')
if not os.path.exists(nc_wip_dir):
os.makedirs(nc_wip_dir)
for package_name in package_names:
try:
ls_netcdf_path = process_site(package_name, nc_wip_dir)
for nc in ls_netcdf_path:
move_to_output_path(nc, vargs.output_path)
except Exception, e:
logger.error(str(e))
if vargs.output_path is None:
vargs.output_path = tempfile.mkdtemp()
if not os.path.exists(vargs.output_path):
try:
os.makedirs(vargs.output_path)
except Exception as e:
raise ValueError('{path} not a valid path'.format(path=vargs.output_path))
sys.exit(1)
return vargs
if __name__ == "__main__":
"""
Processing of the full BOM WAVE DM dataset
./bom_wave_dm_process -i $ARCHIVE_DIR/AODN/BOM_WAVE_DM
"""
vargs = args()
station_ls = filter(lambda f: os.path.isdir(f), glob.glob('{dir}/*'.format(dir=vargs.dataset_path)))
global logger
logger = IMOSLogging().logging_start(os.path.join(vargs.output_path, 'process.log'))
for station_path in station_ls:
logger.info('Processing WAVES for {station_path}'.format(station_path=os.path.basename(station_path)))
process_station(station_path, vargs.output_path)
logger.info('End of processing')
try:
nc_file_path.append(generate_qld_netcdf(resource_id, metadata, output_dir_path))
except Exception as err:
logger.error('Issue processing ressource_id {id}'.format(id=resource_id))
logger.error('{err}'.format(err=err))
logger.error(traceback.print_exc())
return set(nc_file_path)
if __name__ == '__main__':
vargs = args()
if not os.path.exists(WIP_DIR):
os.makedirs(WIP_DIR)
logging = IMOSLogging()
global logger
logger = logging.logging_start(os.path.join(WIP_DIR, 'process.log'))
package_names = list_package_names()
nc_wip_dir = os.path.join(WIP_DIR, 'NetCDF')
if not os.path.exists(nc_wip_dir):
os.makedirs(nc_wip_dir)
for package_name in package_names:
try:
ls_netcdf_path = process_site(package_name, nc_wip_dir)
for nc in ls_netcdf_path:
move_to_output_path(nc, vargs.output_path)
except Exception as e:
logger.error(str(e))
f.close()
replaced_data = filedata.replace("N/A", "")
f = open(data_file, 'w')
f.write(replaced_data)
f.close()
if os.path.exists(download_dir):
shutil.move(data_file, os.path.join(download_dir, 'Lucinda.lev20'))
else:
logger.error('%s does not exists' % download_dir)
shutil.rmtree(temp_dir)
if __name__ == "__main__":
logging = IMOSLogging()
log_file = [mkstemp()]
global logger
logger = logging.logging_start(log_file[0][1])
try:
download_ljco_aeronet(sys.argv[1])
except Exception, e:
print e
logging.logging_stop()
os.close(log_file[0][0])
os.remove(log_file[0][1])
def main(force_reprocess_all=False):
# will sys.exit(-1) if other instance is running
me = singleton.SingleInstance()
wip_soop_path = os.path.join(os.environ['WIP_DIR'], 'SOOP',
'SOOP_XBT_ASF_SST')
lftp_output_path = os.path.join(wip_soop_path, 'data_unsorted', 'XBT',
'sbddata')
csv_output_path = os.path.join(wip_soop_path, 'data_sorted', 'XBT',
'sbddata')
log_filepath = os.path.join(wip_soop_path, 'soop_xbt.log')
logging = IMOSLogging()
logger = logging.logging_start(log_filepath)
lftp_access = {
'ftp_address': os.environ['IMOS_PO_CREDS_CSIRO_IT_FTP_ADDRESS'],
'ftp_subdir': '/',
'ftp_user': os.environ['IMOS_PO_CREDS_CSIRO_IT_FTP_USERNAME'],
'ftp_password': os.environ['IMOS_PO_CREDS_CSIRO_IT_FTP_PASSWORD'],
'ftp_exclude_dir': '',
'lftp_options': '--only-newer',
'output_dir': lftp_output_path
}
lftp = LFTPSync()
logger.info('Download new SOOP XBT NRT files')
lftp.lftp_sync(lftp_access)
# optional function argument to force the reprocess of all sbd files
if force_reprocess_all:
list_new_files = list_files_recursively(lftp_output_path, '*.sbd')
else:
list_new_files = lftp.list_new_files_path(check_file_exist=True)
logger.info('Convert SBD files to CSV')
processSBD = soop_xbt_realtime_processSBD()
manifest_list = []
for f in list_new_files:
if f.endswith(".sbd"):
try:
csv_file = processSBD.handle_sbd_file(f, csv_output_path)
if csv_file not in manifest_list:
manifest_list.append(csv_file)
except Exception as err:
logger.error(str(err))
pass
fd, manifest_file = mkstemp()
for csv_file in manifest_list:
if not (csv_file == []):
os.write(fd, '%s\n' % csv_file)
os.close(fd)
os.chmod(manifest_file, 0o664) # since msktemp creates 600 for security
logger.info('ADD manifest to INCOMING_DIR')
manifest_file_inco_path = os.path.join(
os.environ['INCOMING_DIR'], 'SOOP', 'XBT', 'NRT',
'IMOS_SOOP-XBT_NRT_fileList.manifest')
if not os.path.exists(manifest_file_inco_path):
shutil.copy(manifest_file, manifest_file_inco_path)
else:
logger.warning('File already exist in INCOMING_DIR')
exit(1)
lftp.close()
logging.logging_stop()
except Exception:
raise ValueError('{path} not a valid path'.format(path=vargs.output_path))
sys.exit(1)
return vargs
if __name__ == "__main__":
"""
Processing of the full WA dataset
./wa_awac_process
"""
vargs = args()
global logger
logger = IMOSLogging().logging_start(os.path.join(vargs.output_path, 'process.log'))
if not os.path.exists(WIP_DIR):
os.makedirs(WIP_DIR)
sites_info = retrieve_sites_info_awac_kml()
for _, site_code in enumerate(sites_info):
site_info = sites_info[site_code]
temporary_data_path, site_info = download_site_data(site_info) # returned site_info has extra md5 info
site_name = site_info['site_code']
try:
if site_info['already_uptodate']:
logger.info('{site_path} already up to date'.format(site_path=site_name))
shutil.rmtree(temporary_data_path)
continue