def set_deployment_status(self, input_file, message): """ Write message to Harvestmission.csv file. ingested in RT pieline Update the anfog_rt.harvest_listing table after ingestion of csv file by RT pipeline Note that to be consistent with the available message in the production DB, dashes need to be replaced by underscore, for ex delayed-mode =>delayed_mode :return: Harvestmission.csv updated with deployment specific status """ name = os.path.basename(input_file) deployment = AnfogFileClassifier.get_deployment_code(input_file) platform = AnfogFileClassifier.get_platform(name) listing_path = os.path.join(self.products_dir, AnfogFileClassifier.MISSION_LISTING) with open(listing_path, 'w') as f: f.write('deployment_name, platform_type, status' + os.linesep) row = "%s,%s,%s" % (deployment, platform, message.replace( '-', '_').lower()) f.write(row) product = PipelineFile(listing_path) product.publish_type = PipelineFilePublishType.HARVEST_ONLY product.check_type = PipelineFileCheckType.FORMAT_CHECK product.dest_path = os.path.join(self.upload_destination, os.path.basename(listing_path)) self.file_collection.add(product)
def preprocess(self): """ pre processsing to handle the conversion of BUFR csv files to NetCDF :return: """ csv_file = self.file_collection.filter_by_attribute_id( 'file_type', FileType.CSV) if csv_file: csv_file = csv_file[0] csv_file.publish_type = PipelineFilePublishType.ARCHIVE_ONLY profiles = parse_bufr_file(csv_file.src_path) profiles = return_unique_profiles( profiles) # check for duplicate profiles within BUFR file for profile in profiles: profile = fzf_vessel_get_info( profile) # fuzzy search finder for vessel name profile = xbt_line_get_info( profile, self.xbt_line_vocab_url ) # get hard coded info per xbt line netcdf_filepath = netcdf_writer( profile, self.temp_dir) # convert BUFR to NetCDF # publish nc_file = PipelineFile( netcdf_filepath, file_update_callback=self._file_update_callback) nc_file.publish_type = PipelineFilePublishType.HARVEST_UPLOAD self.file_collection.add(nc_file)
def schedule_file_removal(self, remote_pipeline_file): """schedule a file to be removed.""" filename = remote_pipeline_file.name file_to_remove = PipelineFile( filename, is_deletion=True, dest_path=self.dest_path_function(filename), ) file_to_remove.publish_type = PipelineFilePublishType.DELETE_UNHARVEST self.file_collection.add(file_to_remove) logger.info( NRT_FILE_REMOVAL_MSG.format(file=file_to_remove, ptype=file_to_remove.publish_type))
def preprocess(self): """ Files to be deleted as found in 'soop_trv_duplicate_url' wfs layer """ files_to_delete = self.state_query.query_wfs_urls_for_layer( 'soop_trv_duplicate_url') for f in files_to_delete: file_to_delete = PipelineFile( os.path.basename(f), is_deletion=True, dest_path=f, file_update_callback=self._file_update_callback) file_to_delete.publish_type = PipelineFilePublishType.DELETE_UNHARVEST self.file_collection.add(file_to_delete)
def preprocess(self): """ Preprocessing for NRT and DM files - NRT: generate a NetCDF files based on input text file. Set the input file publish_type property to 'archive' - DM file collection: update the check_type and publish_type properties for non-NetCDF files. These files are not checked or harvested, but uploaded to S3 """ if self.custom_params is not None and self.custom_params.get( 'ship_callsign_ls'): self.ship_callsign_ls = self.custom_params['ship_callsign_ls'] else: self.ship_callsign_ls = ship_callsign_list() # Delayed mode file submitted as a zip archive if self.file_extension == '.zip': nc_file = self.file_collection.filter_by_attribute_id( 'file_type', FileType.NETCDF) if len(nc_file) != 1: raise InvalidInputFileError( "Expecting one netCDF file in ZIP archive '{zip}'".format( zip=os.path.basename(self.input_file))) # first process the NetCDF file to set the destination path for the file collection nc = nc_file[0] nc.dest_path = self.dest_path(nc.src_path) nc_dir_path = os.path.dirname(nc.dest_path) # SOOP-CO2 DM and FRMAP .txt,.pdf or/and .xml files. # Set check type to NONEMPTY and publish type to UPLOAD_ONLY non_nc_files = PipelineFileCollection( f for f in self.file_collection if f.file_type is not FileType.NETCDF) for non_nc in non_nc_files: non_nc.check_type = PipelineFileCheckType.FORMAT_CHECK non_nc.publish_type = PipelineFilePublishType.UPLOAD_ONLY non_nc.dest_path = os.path.join(nc_dir_path, non_nc.name) elif self.input_file.endswith('dat.txt'): # Single text file Realtime files (*dat.txt) rt_file = self.file_collection[0] rt_file.publish_type = PipelineFilePublishType.ARCHIVE_ONLY nrt_nc_file_path = soop_co2_nrt_nc_generator.process_co2_rt( rt_file, self.products_dir, self.ship_callsign_ls) nrt_nc_file = PipelineFile(nrt_nc_file_path) self.file_collection.add(nrt_nc_file) nrt_nc_file.publish_type = PipelineFilePublishType.HARVEST_UPLOAD
def _cleanup_previous_version(self, product_filename): """Identify any previously published version(s) of the given product file and mark them for deletion. Ignores cases where the previous version has exactly the same file name, as this will simply be overwritten. :param product_filename: File name of the newly generated product """ product_type = get_product_type(product_filename) for old_product_url in self.old_product_files.get(product_type, []): if os.path.basename(old_product_url) != product_filename: # Add the previous version as a "late deletion". It will be deleted during the handler's `publish` # step after (and only if) all new files have been successfully published. old_file = PipelineFile( old_product_url, dest_path=old_product_url, is_deletion=True, late_deletion=True, file_update_callback=self._file_update_callback) old_file.publish_type = PipelineFilePublishType.DELETE_UNHARVEST self.file_collection.add(old_file)
def preprocess(self): """Check that every input file is valid according to the include/exclude regex patterns. Any non-matching file will be left with publish_type UNSET after the _resolve step. If there are any netCDF files from burst-sampling instruments in the collection, create the burst-averaged version of each and add them to the collection. :return: None """ self.logger.info( "Checking for invalid files and adjusting check/publish properties." ) invalid_files = self.file_collection.filter_by_attribute_id( 'publish_type', PipelineFilePublishType.UNSET) if invalid_files: raise InvalidFileNameError( "File name(s) don't match the pattern expected for this upload location: {names}" .format(names=invalid_files.get_attribute_list('name'))) # Burst-processing for FV01 files with burst-sampling global attributes burst_files = (self.file_collection.filter_by_attribute_id( 'file_type', FileType.NETCDF).filter_by_attribute_regex('name', r'.*_FV01_')) for f in burst_files: with Dataset(f.src_path, mode='r') as D: has_interval = hasattr(D, 'instrument_burst_interval') has_duration = hasattr(D, 'instrument_burst_duration') is_adcp = ('DIST_ALONG_BEAMS' in D.dimensions or 'HEIGHT_ABOVE_SENSOR' in D.dimensions) if not (has_interval and has_duration) or is_adcp: continue self.logger.info("Burst-processing {f.name}".format(f=f)) product_path = create_burst_average_netcdf(f.src_path, self.products_dir) product_file = PipelineFile( product_path, file_update_callback=self._file_update_callback) product_file.publish_type = PipelineFilePublishType.HARVEST_UPLOAD self.file_collection.add(product_file)
with open(GETFEATURE_OLD_PRODUCTS_FILE) as f: TEST_GETFEATURE_OLD_PRODUCTS_JSON = f.read() with open(GETFEATURE_EMPTY_FILE) as f: TEST_GETFEATURE_EMPTY_JSON = f.read() # Create collection of input files for the products # These will be uploaded to the mocked equivalent of S3 (where the real input files will be) features = json.loads(TEST_GETFEATURE_JSON)['features'] INPUT_FILE_COLLECTION = PipelineFileCollection() for f in features: pf = PipelineFile( os.path.join(TEST_ROOT, os.path.basename(f['properties']['url'])), dest_path=f['properties']['url'] ) pf.publish_type = PipelineFilePublishType.UPLOAD_ONLY INPUT_FILE_COLLECTION.add(pf) class TestMooringsProductsHandler(HandlerTestCase): def setUp(self): self.handler_class = MooringsProductsHandler upload_broker = get_storage_broker(self.config.pipeline_config['global']['upload_uri']) upload_broker.upload(INPUT_FILE_COLLECTION) super().setUp() @patch('aodncore.util.wfs.WebFeatureService') def test_all_products(self, mock_webfeatureservice): mock_webfeatureservice().getfeature().getvalue.side_effect = [TEST_GETFEATURE_JSON, TEST_GETFEATURE_OLD_PRODUCTS_JSON]
def preprocess(self): # if input file is a NetCDF, create a .nc.gz and harvest upload it. # historically, files were always sent as *.nc.gz. But as of April 2021, files might be pushed as *.nc. # to be consistent, we transform this .nc into a .nz.gz if self.file_type is FileType.NETCDF: self.file_collection.set_publish_types( PipelineFilePublishType.NO_ACTION) gzip_path = os.path.join(self.temp_dir, self.file_basename + '.gz') with open(self.input_file, 'rb') as f_in, gzip.open(gzip_path, 'wb') as gz_out: gz_out.writelines(f_in) # publish self.add_to_collection( gzip_path, publish_type=PipelineFilePublishType.HARVEST_UPLOAD) if self.file_type is FileType.GZIP: # add nc_gz file to collection (not by default) self.file_collection.add(self.input_file_object) netcdf_file_gz_collection = self.file_collection.filter_by_attribute_id( 'file_type', FileType.GZIP) netcdf_file_gz = netcdf_file_gz_collection[0] netcdf_file_gz.publish_type = PipelineFilePublishType.HARVEST_UPLOAD # default # some GSLA files are gzipped, so gunzip them before checking them # if uploaded file is GZIP check that GZIP contains a NetCDF netcdf_collection = self.file_collection.filter_by_attribute_id( 'file_type', FileType.NETCDF) if len(netcdf_collection) != 1: raise InvalidInputFileError( "Expecting one netCDF file in GZIP archive '{gzip}'". format(gzip=os.path.basename(self.input_file))) netcdf_file_gz = self.file_collection.filter_by_attribute_id( 'file_type', FileType.GZIP)[0] netcdf_file = self.file_collection.filter_by_attribute_id( 'file_type', FileType.NETCDF)[0] # setting the path of the gz file with the gunzipped file netcdf_file_gz.dest_path = self.dest_path(netcdf_file.src_path) # Nothing to do with *.nc. Talend can harvest *.nc.gz. Set to NO_ACTION netcdf_file.publish_type = PipelineFilePublishType.NO_ACTION # we don't know the product type (DM00 or DM01) of the file already # on s3 in order to deduce its path. We need to get the product # type from the file in incoming result_previous_version_creation_date = self.get_previous_version_creation_date( netcdf_file.src_path) """ default values by default we push to the storage the file landed in the pipeline (ie *.nc.gz) """ push_new_file = True remove_previous_version = False # compare creation dates with file already on storage if result_previous_version_creation_date: new_file_creation_date = get_creation_date(netcdf_file.name) if result_previous_version_creation_date > new_file_creation_date: push_new_file = False elif result_previous_version_creation_date == new_file_creation_date: push_new_file = True else: remove_previous_version = True previous_file_path = self.get_previous_version_object( netcdf_file.src_path) if push_new_file: if GSLA_REGEX_YEARLY.match(netcdf_file.name): # yearly file should never be harvested netcdf_file_gz.publish_type = PipelineFilePublishType.UPLOAD_ONLY else: raise InvalidFileNameError( "file name: \"{filename}\" creation date is older than file already on " "storage".format(filename=netcdf_file_gz.name)) # deletion of the previous file if remove_previous_version: previous_file_name = os.path.basename(previous_file_path) file_to_delete = PipelineFile( previous_file_name, is_deletion=True, dest_path=previous_file_path, file_update_callback=self._file_update_callback) if GSLA_REGEX_YEARLY.match(netcdf_file.name): file_to_delete.publish_type = PipelineFilePublishType.DELETE_ONLY else: file_to_delete.publish_type = PipelineFilePublishType.DELETE_UNHARVEST self.file_collection.add(file_to_delete)
def get_previous_version(self, previous_file_list, path, input_file_name): """ Find previous version of each incoming file based on its type/extension and add them to the filecollection with the correct publish type extension can be: .inf', '.nc.png','.pitch.csv','.roll.csv',.gps.csv' inputs: previous_file_list : dictionary containing file listing(full path) and metadata from destination input_file : file basename path : full destination path """ if not previous_file_list: return files_to_delete = PipelineFileCollection() try: extension = ALLOWED_CONTENT_EXTENSIONS.match( input_file_name).groupdict()['extension'] except KeyError: raise ValueError( "unable to determine extension from file name {infile}".format( infile=input_file_name)) # get list of previous files basename to search through basenames = {os.path.basename(f) for f in previous_file_list} this_extension_pattern = re.compile( r".*\.{ext}$".format(ext=extension)) if input_file_name not in basenames: previous_file = [ f for f in previous_file_list if this_extension_pattern.match(f) ] if extension == 'nc': if len(previous_file) != 1: raise ValueError( "Expected exactly 1 previous versions of the netcdf file, found {n}. Aborting " .format(n=len(previous_file))) else: # if uploaded file name has the same name published file => no action, file will be overwritten, otherwise # sort file per wildcard and work out which one to delete ( # check previous file widcard : # can be '.inf', '.nc.png','.pitch.csv','.roll.csv',.gps.csv' if len(previous_file) > 1: raise ValueError( "Found more than one previous versions of the extension '{ext}'. Aborting" .format(ext=extension)) elif len(previous_file) == 0: return prev_file = previous_file[0] dest_path = os.path.join(path, os.path.basename(prev_file)) self.logger.info( "adding deletion of previous file '{dest_path}'".format( dest_path=dest_path)) file_to_delete = PipelineFile(prev_file, is_deletion=True, dest_path=dest_path) if extension == 'nc': file_to_delete.publish_type = PipelineFilePublishType.DELETE_UNHARVEST else: file_to_delete.publish_type = PipelineFilePublishType.DELETE_ONLY files_to_delete.add(file_to_delete) return files_to_delete
def preprocess(self): if self.custom_params is not None and self.custom_params.get( 'ship_callsign_ls'): self.ship_callsign_ls = self.custom_params['ship_callsign_ls'] else: self.ship_callsign_ls = ship_callsign_list() if SHIP_CODE not in self.ship_callsign_ls: raise RuntimeError( "Missing vessel callsign {callsign} from vocabulary.".format( callsign=SHIP_CODE)) self.soop_tmv_dir = os.path.join( 'IMOS', 'SOOP', 'SOOP-TMV', '{ship_code}_{ship_name}'.format( ship_code=SHIP_CODE, ship_name=self.ship_callsign_ls[SHIP_CODE]), 'realtime') txt_files = self.file_collection.filter_by_attribute_value( 'extension', '.txt') log_files = self.file_collection.filter_by_attribute_value( 'extension', '.log') nc_files = self.file_collection.filter_by_attribute_id( 'file_type', FileType.NETCDF) """ * 10secs zip files (*.log + *.txt [calibration]) -> *.zip is pushed to ARCHIVE_DIR (netcdf still needs to be generated to deduce path). *.log, *.txt and *.nc NOT added to the collection * 1sec zip files (*.log only) -> *.log & *.nc pushed to S3. *.zip not added to the collection """ if len(nc_files): # case where we re-push an existing NetCDF file f_nc = nc_files[0] f_nc.publish_type = PipelineFilePublishType.HARVEST_UPLOAD elif len(log_files): f_log = log_files[0] log_filename = os.path.basename(f_log.src_path) if SOOP_NRT_LOG_PATTERN.match(log_filename) is None: raise InvalidFileNameError( "SOOP TMV NRT input logfile has incorrect naming '{name}'." .format(name=log_filename)) # case to create NetCDF file from log file f_txt = None if len(txt_files): f_txt = txt_files[0] netcdf_filepath = netcdf_writer( f_log.src_path, self.temp_dir, self.ship_callsign_ls[SHIP_CODE], meta_path=f_txt.src_path) else: netcdf_filepath = netcdf_writer( f_log.src_path, self.temp_dir, self.ship_callsign_ls[SHIP_CODE]) # the path of logs and zips has to deduced within the pre-process as it needs the creation of a NetCDF to # get the correct info with Dataset(netcdf_filepath) as nc_open: measurement_frequency = nc_open.measurement_frequency product_type = nc_open.product_type year = datetime.strptime(nc_open.time_coverage_start, '%Y-%m-%dT%H:%M:%SZ').strftime("%Y") pre_path = os.path.join(self.soop_tmv_dir, product_type, measurement_frequency, year) if measurement_frequency == "1sec": f_log.publish_type = PipelineFilePublishType.UPLOAD_ONLY f_log.dest_path = os.path.join(pre_path, 'logs', f_log.name) nc_file = PipelineFile(netcdf_filepath) nc_file.publish_type = PipelineFilePublishType.HARVEST_UPLOAD self.file_collection.add(nc_file) elif measurement_frequency == "10secs": if self.input_file.endswith('zip'): self.input_file_object.publish_type = PipelineFilePublishType.ARCHIVE_ONLY self.input_file_object.archive_path = os.path.join( pre_path, 'logs', self.input_file_object.name) self.file_collection.add(self.input_file_object) f_log.publish_type = PipelineFilePublishType.NO_ACTION if f_txt: f_txt.publish_type = PipelineFilePublishType.NO_ACTION else: # case when a 10secs log file (and not a zip) is pushed to incoming f_log.publish_type = PipelineFilePublishType.ARCHIVE_ONLY f_log.archive_path = os.path.join(pre_path, 'logs', f_log.name)
def delete_previous_version(self, mode, deployment_status): """ In RT mode: 2 cases 1) update of deployment in progress :select previous version of a file that needs to be deleted (.nc) other files are automatically overwritten 2) cleaning RT folder :status "renamed","clear-files" In DM mode either - new DM (deployment_status = delayed_mode): delete RT deployment files or - update DM (deployment_status = update):: delete previous .nc, other files are automatically overwritten) """ if mode == 'DM' and deployment_status == 'delayed_mode': # RT and DM folder hierarchy similar except that RT has additional level /REALTIME/ destination = AnfogFileClassifier.make_rt_path( self.upload_destination) delete_file_regex = '%s|%s|%s' % ( AnfogFileClassifier.ANFOG_RT_REGEX, AnfogFileClassifier.RT_PNG_REGEX, AnfogFileClassifier.RT_POSITION_SUMMARY) elif mode == 'DM' and deployment_status == 'update': destination = self.upload_destination delete_file_regex = AnfogFileClassifier.ANFOG_DM_REGEX elif mode == 'RT' and deployment_status in ['renamed', 'clear-files']: destination = self.upload_destination delete_file_regex = '%s|%s|%s' % ( AnfogFileClassifier.ANFOG_RT_REGEX, AnfogFileClassifier.RT_PNG_REGEX, AnfogFileClassifier.RT_POSITION_SUMMARY) elif mode == 'RT' and deployment_status == 'in_progress': destination = self.upload_destination delete_file_regex = AnfogFileClassifier.ANFOG_RT_REGEX else: raise ValueError( "Invalid combination of mode '{mode}' and status'{st}'".format( mode=mode, st=deployment_status)) # publish_type set according to file type: netcdf unharvest_delete, # everything else (png, txt, kml..) DELETE_ONLY previous_file_list = self.state_query.query_storage(destination).keys() self.logger.info("Mode '{mode}' and Status '{status}'".format( mode=mode, status=deployment_status)) for filename in previous_file_list: previous_file = PipelineFile(filename, is_deletion=True, dest_path=destination) previous_file.dest_path = os.path.join(destination, previous_file.name) # set default publish type to delete only previous_file.publish_type = PipelineFilePublishType.DELETE_ONLY if previous_file.file_type is FileType.NETCDF: previous_file.publish_type = PipelineFilePublishType.DELETE_UNHARVEST if re.match(delete_file_regex, previous_file.name): self.file_collection.add(previous_file) if deployment_status not in [ 'renamed', 'clear-files' ] and re.match(previous_file.name, os.path.basename(self.primary_nc.src_path)): # removing file as it will be overwritten. Note that test is invalid in 'renamed' status # cause primary_nc not set(irrelevant) self.file_collection.discard(previous_file) else: self.logger.info( "adding deletion of previous file '{dest_path}'". format(dest_path=previous_file.dest_path))
def _add_to_collection(self, product_url): """Add a new product file to the file_collection to be harvested and uploaded.""" product_file = PipelineFile( product_url, file_update_callback=self._file_update_callback) product_file.publish_type = PipelineFilePublishType.HARVEST_UPLOAD self.file_collection.add(product_file)
def preprocess(self): jpg_output_path, jpg_output_dest_path = create_plot(self.file_collection[0].src_path, self.temp_dir) jpg_file = PipelineFile(jpg_output_path, dest_path=jpg_output_dest_path) jpg_file.publish_type = PipelineFilePublishType.UPLOAD_ONLY self.file_collection.add(jpg_file)