def set_deployment_status(self, input_file, message):
"""
Write message to Harvestmission.csv file. ingested in RT pieline
Update the anfog_rt.harvest_listing table after ingestion of csv file by RT pipeline
Note that to be consistent with the available message in the production DB,
dashes need to be replaced by underscore, for ex delayed-mode =>delayed_mode
:return: Harvestmission.csv updated with deployment specific status
"""
name = os.path.basename(input_file)
deployment = AnfogFileClassifier.get_deployment_code(input_file)
platform = AnfogFileClassifier.get_platform(name)
listing_path = os.path.join(self.products_dir,
AnfogFileClassifier.MISSION_LISTING)
with open(listing_path, 'w') as f:
f.write('deployment_name, platform_type, status' + os.linesep)
row = "%s,%s,%s" % (deployment, platform, message.replace(
'-', '_').lower())
f.write(row)
product = PipelineFile(listing_path)
product.publish_type = PipelineFilePublishType.HARVEST_ONLY
product.check_type = PipelineFileCheckType.FORMAT_CHECK
product.dest_path = os.path.join(self.upload_destination,
os.path.basename(listing_path))
self.file_collection.add(product)
def delete_previous_version(self, mode, deployment_status):
"""
In RT mode: 2 cases 1) update of deployment in progress :select previous version of a file that needs to be
deleted (.nc) other files are automatically overwritten
2) cleaning RT folder :status "renamed","clear-files"
In DM mode either - new DM (deployment_status = delayed_mode): delete RT deployment files
or
- update DM (deployment_status = update):: delete previous .nc,
other files are automatically overwritten)
"""
if mode == 'DM' and deployment_status == 'delayed_mode':
# RT and DM folder hierarchy similar except that RT has additional level /REALTIME/
destination = AnfogFileClassifier.make_rt_path(
self.upload_destination)
delete_file_regex = '%s|%s|%s' % (
AnfogFileClassifier.ANFOG_RT_REGEX,
AnfogFileClassifier.RT_PNG_REGEX,
AnfogFileClassifier.RT_POSITION_SUMMARY)
elif mode == 'DM' and deployment_status == 'update':
destination = self.upload_destination
delete_file_regex = AnfogFileClassifier.ANFOG_DM_REGEX
elif mode == 'RT' and deployment_status in ['renamed', 'clear-files']:
destination = self.upload_destination
delete_file_regex = '%s|%s|%s' % (
AnfogFileClassifier.ANFOG_RT_REGEX,
AnfogFileClassifier.RT_PNG_REGEX,
AnfogFileClassifier.RT_POSITION_SUMMARY)
elif mode == 'RT' and deployment_status == 'in_progress':
destination = self.upload_destination
delete_file_regex = AnfogFileClassifier.ANFOG_RT_REGEX
else:
raise ValueError(
"Invalid combination of mode '{mode}' and status'{st}'".format(
mode=mode, st=deployment_status))
# publish_type set according to file type: netcdf unharvest_delete,
# everything else (png, txt, kml..) DELETE_ONLY
previous_file_list = self.state_query.query_storage(destination).keys()
self.logger.info("Mode '{mode}' and Status '{status}'".format(
mode=mode, status=deployment_status))
for filename in previous_file_list:
previous_file = PipelineFile(filename,
is_deletion=True,
dest_path=destination)
previous_file.dest_path = os.path.join(destination,
previous_file.name)
# set default publish type to delete only
previous_file.publish_type = PipelineFilePublishType.DELETE_ONLY
if previous_file.file_type is FileType.NETCDF:
previous_file.publish_type = PipelineFilePublishType.DELETE_UNHARVEST
if re.match(delete_file_regex, previous_file.name):
self.file_collection.add(previous_file)
if deployment_status not in [
'renamed', 'clear-files'
] and re.match(previous_file.name,
os.path.basename(self.primary_nc.src_path)):
# removing file as it will be overwritten. Note that test is invalid in 'renamed' status
# cause primary_nc not set(irrelevant)
self.file_collection.discard(previous_file)
else:
self.logger.info(
"adding deletion of previous file '{dest_path}'".
format(dest_path=previous_file.dest_path))
