def set_deployment_status(self, input_file, message): """ Write message to Harvestmission.csv file. ingested in RT pieline Update the anfog_rt.harvest_listing table after ingestion of csv file by RT pipeline Note that to be consistent with the available message in the production DB, dashes need to be replaced by underscore, for ex delayed-mode =>delayed_mode :return: Harvestmission.csv updated with deployment specific status """ name = os.path.basename(input_file) deployment = AnfogFileClassifier.get_deployment_code(input_file) platform = AnfogFileClassifier.get_platform(name) listing_path = os.path.join(self.products_dir, AnfogFileClassifier.MISSION_LISTING) with open(listing_path, 'w') as f: f.write('deployment_name, platform_type, status' + os.linesep) row = "%s,%s,%s" % (deployment, platform, message.replace( '-', '_').lower()) f.write(row) product = PipelineFile(listing_path) product.publish_type = PipelineFilePublishType.HARVEST_ONLY product.check_type = PipelineFileCheckType.FORMAT_CHECK product.dest_path = os.path.join(self.upload_destination, os.path.basename(listing_path)) self.file_collection.add(product)
def delete_previous_version(self, mode, deployment_status): """ In RT mode: 2 cases 1) update of deployment in progress :select previous version of a file that needs to be deleted (.nc) other files are automatically overwritten 2) cleaning RT folder :status "renamed","clear-files" In DM mode either - new DM (deployment_status = delayed_mode): delete RT deployment files or - update DM (deployment_status = update):: delete previous .nc, other files are automatically overwritten) """ if mode == 'DM' and deployment_status == 'delayed_mode': # RT and DM folder hierarchy similar except that RT has additional level /REALTIME/ destination = AnfogFileClassifier.make_rt_path( self.upload_destination) delete_file_regex = '%s|%s|%s' % ( AnfogFileClassifier.ANFOG_RT_REGEX, AnfogFileClassifier.RT_PNG_REGEX, AnfogFileClassifier.RT_POSITION_SUMMARY) elif mode == 'DM' and deployment_status == 'update': destination = self.upload_destination delete_file_regex = AnfogFileClassifier.ANFOG_DM_REGEX elif mode == 'RT' and deployment_status in ['renamed', 'clear-files']: destination = self.upload_destination delete_file_regex = '%s|%s|%s' % ( AnfogFileClassifier.ANFOG_RT_REGEX, AnfogFileClassifier.RT_PNG_REGEX, AnfogFileClassifier.RT_POSITION_SUMMARY) elif mode == 'RT' and deployment_status == 'in_progress': destination = self.upload_destination delete_file_regex = AnfogFileClassifier.ANFOG_RT_REGEX else: raise ValueError( "Invalid combination of mode '{mode}' and status'{st}'".format( mode=mode, st=deployment_status)) # publish_type set according to file type: netcdf unharvest_delete, # everything else (png, txt, kml..) DELETE_ONLY previous_file_list = self.state_query.query_storage(destination).keys() self.logger.info("Mode '{mode}' and Status '{status}'".format( mode=mode, status=deployment_status)) for filename in previous_file_list: previous_file = PipelineFile(filename, is_deletion=True, dest_path=destination) previous_file.dest_path = os.path.join(destination, previous_file.name) # set default publish type to delete only previous_file.publish_type = PipelineFilePublishType.DELETE_ONLY if previous_file.file_type is FileType.NETCDF: previous_file.publish_type = PipelineFilePublishType.DELETE_UNHARVEST if re.match(delete_file_regex, previous_file.name): self.file_collection.add(previous_file) if deployment_status not in [ 'renamed', 'clear-files' ] and re.match(previous_file.name, os.path.basename(self.primary_nc.src_path)): # removing file as it will be overwritten. Note that test is invalid in 'renamed' status # cause primary_nc not set(irrelevant) self.file_collection.discard(previous_file) else: self.logger.info( "adding deletion of previous file '{dest_path}'". format(dest_path=previous_file.dest_path))