def test_clear_rt_deployment(self): # TEST 'clear-files' status. process and results identical to status 'renamed' preexisting_files = PipelineFileCollection() existing_file1 = PipelineFile(PREV_NC_RT, dest_path=os.path.join( 'IMOS/ANFOG/REALTIME/slocum_glider/TwoRocks20180503a/', os.path.basename(PREV_NC_RT))) existing_file2 = PipelineFile(PREV_PNG_TRANSECT, dest_path=os.path.join( 'IMOS/ANFOG/REALTIME/slocum_glider/TwoRocks20180503a/', os.path.basename(PREV_PNG_TRANSECT))) preexisting_files.update([existing_file1, existing_file2]) # set the files to UPLOAD_ONLY preexisting_files.set_publish_types(PipelineFilePublishType.UPLOAD_ONLY) # upload the 'preexisting_files' collection to the unit test's temporary upload location broker = get_storage_broker(self.config.pipeline_config['global']['upload_uri']) broker.upload(preexisting_files) handler = self.run_handler(MISSION_STATUS_CLR) # Process should resuls in : input file unhandled , preexisting file should be deleted nc = handler.file_collection.filter_by_attribute_id('file_type', FileType.NETCDF) self.assertEqual(nc[0].publish_type, PipelineFilePublishType.DELETE_UNHARVEST) self.assertTrue(nc[0].is_deleted) png = handler.file_collection.filter_by_attribute_id('file_type', FileType.PNG) self.assertEqual(png[0].publish_type, PipelineFilePublishType.DELETE_ONLY) self.assertTrue(png[0].is_deleted)
def setUp(self): self.logger = get_pipeline_logger('unittest') self.dummy_input_file = 'dummy.input_file' incoming_file_path = os.path.join( self.config.pipeline_config['watch']['incoming_dir'], os.path.basename(self.temp_nc_file)) safe_copy_file(self.temp_nc_file, incoming_file_path) celery_request = type('DummyRequest', (object, ), {'id': 'NO_REQUEST_ID'})() self.state_manager = IncomingFileStateManager( incoming_file_path, pipeline_name='UNITTEST', config=self.config, logger=self.logger, celery_request=celery_request) self.state_manager.handler = MagicMock( file_basename=self.dummy_input_file, error_cleanup_regexes=[r'test.*']) previous_file_same_name = PipelineFile( self.temp_nc_file, dest_path='dummy.input_file.40c4ec0d-c9db-498d-84f9-01011330086e') nc = PipelineFile(GOOD_NC, dest_path=os.path.basename(GOOD_NC)) png = PipelineFile(INVALID_PNG, dest_path=os.path.basename(INVALID_PNG)) ico = PipelineFile(TEST_ICO, dest_path=os.path.basename(TEST_ICO)) unknown = PipelineFile(UNKNOWN_FILE_TYPE, dest_path=os.path.basename(UNKNOWN_FILE_TYPE)) existing_collection = PipelineFileCollection( [previous_file_same_name, nc, png, ico, unknown]) self.state_manager.error_broker.upload(existing_collection)
def get_harvest_collection(delete=False, late_deletion=False, with_store=False, already_stored=False): pf_bad = PipelineFile(BAD_NC, is_deletion=delete, late_deletion=late_deletion) pf_empty = PipelineFile(EMPTY_NC, is_deletion=delete, late_deletion=late_deletion) pf_good = PipelineFile(GOOD_NC, is_deletion=delete, late_deletion=late_deletion) collection = PipelineFileCollection([pf_bad, pf_empty, pf_good]) if with_store: publish_type = PipelineFilePublishType.DELETE_UNHARVEST if delete else PipelineFilePublishType.HARVEST_UPLOAD else: publish_type = PipelineFilePublishType.UNHARVEST_ONLY if delete else PipelineFilePublishType.HARVEST_ONLY for pipeline_file in collection: pipeline_file.is_stored = already_stored pipeline_file.dest_path = os.path.join( 'DUMMY', os.path.basename(pipeline_file.src_path)) pipeline_file.publish_type = publish_type return collection
def test_cleanup(self): nc = PipelineFile(GOOD_NC, dest_path=os.path.basename(GOOD_NC)) png = PipelineFile(INVALID_PNG, dest_path=os.path.basename(INVALID_PNG)) ico = PipelineFile(TEST_ICO, dest_path=os.path.basename(TEST_ICO)) unknown = PipelineFile(UNKNOWN_FILE_TYPE, dest_path=os.path.basename(UNKNOWN_FILE_TYPE)) existing_collection = PipelineFileCollection([nc, png, ico, unknown]) self.state_manager.error_broker.upload(existing_collection) self.state_manager.move_to_processing() actual_error_files_before_cleanup = [ v.dest_path for v in self.state_manager.error_broker.query() ] expected_error_files_before_cleanup = [ 'good.nc', 'test.unknown_file_extension', 'test.ico', 'invalid.png' ] self.assertCountEqual(expected_error_files_before_cleanup, actual_error_files_before_cleanup) self.state_manager.success_exit_policies.append( ExitPolicy.DELETE_CUSTOM_REGEXES_FROM_ERROR_STORE) self.state_manager.move_to_success() actual_error_files_after_cleanup = [ v.dest_path for v in self.state_manager.error_broker.query() ] expected_error_files_after_cleanup = ['good.nc', 'invalid.png'] self.assertCountEqual(expected_error_files_after_cleanup, actual_error_files_after_cleanup)
def test_rt_update(self): """ test the update of realtime mission: update consits in : - deletion of previous netCDF - deletion of transect png files - harvest of new netCDF - overwriting of other files """ # create some PipelineFiles to represent the existing files on 'S3' preexisting_files = PipelineFileCollection() existing_file1 = PipelineFile(PREV_NC_RT, dest_path=os.path.join( 'IMOS/ANFOG/REALTIME/slocum_glider/TwoRocks20180503a/', os.path.basename(PREV_NC_RT))) existing_file2 = PipelineFile(PREV_PNG_TRANSECT, dest_path=os.path.join( 'IMOS/ANFOG/REALTIME/slocum_glider/TwoRocks20180503a/', os.path.basename(PREV_PNG_TRANSECT))) existing_file3 = PipelineFile(PREV_PNG_MISSION, dest_path=os.path.join( 'IMOS/ANFOG/REALTIME/slocum_glider/TwoRocks20180503a/', os.path.basename(PREV_PNG_MISSION))) preexisting_files.update([existing_file1, existing_file2, existing_file3]) # set the files to UPLOAD_ONLY preexisting_files.set_publish_types(PipelineFilePublishType.UPLOAD_ONLY) # upload the 'preexisting_files' collection to the unit test's temporary upload location broker = get_storage_broker(self.config.pipeline_config['global']['upload_uri']) broker.upload(preexisting_files) # run the handler handler = self.run_handler(GOOD_ZIP_RT) nc = handler.file_collection.filter_by_attribute_id('file_type', FileType.NETCDF) for n in nc: if n.name == os.path.basename(PREV_NC_RT): self.assertEqual(n.publish_type, PipelineFilePublishType.DELETE_UNHARVEST) self.assertTrue(n.is_deleted) else: self.assertEqual(n.publish_type, PipelineFilePublishType.HARVEST_UPLOAD) self.assertTrue(n.is_harvested) self.assertTrue(n.is_stored) pngs = handler.file_collection.filter_by_attribute_id('file_type', FileType.PNG) for png in pngs: if png.name == os.path.basename(PREV_PNG_MISSION): self.assertTrue(png.is_overwrite) else: self.assertTrue(png.is_uploaded) # no update the harvestMission List in that case csv = handler.file_collection.filter_by_attribute_id('file_type', FileType.CSV) self.assertEqual(len(csv), 0)
def get_remote_metadata_from_zip(self, remote_pfile): """Fetch the metdata.csv file from a RemotePipelineFile zip and wrapping it on a PipelineFile.""" # TODO: remove the first try statement when pipeline remote_collection = RemotePipelineFileCollection(remote_pfile) try: download = self.state_query.download except AttributeError: download = self.state_query._storage_broker.download download(remote_collection, self.temp_dir) dest_folder = os.path.join(self.temp_dir, os.path.dirname(remote_pfile.dest_path)) local_zipfile = os.path.join(self.temp_dir, remote_pfile.dest_path) extract_zip(local_zipfile, dest_folder) local_metadata_file = [ os.path.join(dest_folder, x) for x in os.listdir(dest_folder) if "metadata" in x ] try: return PipelineFile(local_metadata_file[0]) except IndexError: return None
def test_setup_upload_location_push_older_file(self): """ Test case: Check creation date of incoming *.nc.gz is older that one already on storage NO_ACTION *nc.gz """ # create some PipelineFiles to represent the existing files on 'S3' preexisting_files = PipelineFileCollection() existing_file = PipelineFile(PREV_NC_GZ_STORAGE, dest_path=os.path.join( 'IMOS/OceanCurrent/GSLA/DM00/2018/', os.path.basename(PREV_NC_GZ_STORAGE))) preexisting_files.update([existing_file]) # set the files to UPLOAD_ONLY preexisting_files.set_publish_types( PipelineFilePublishType.UPLOAD_ONLY) # upload the 'preexisting_files' collection to the unit test's temporary upload location broker = get_storage_broker( self.config.pipeline_config['global']['upload_uri']) broker.upload(preexisting_files) # run the handler on the new file with an older creation date self.run_handler_with_exception(InvalidFileNameError, OLDER_CREATION_DATE_NC_GZ)
def test_setup_upload_location_push_newer_file_bad_prefix(self): """ Test case: Check creation date of incoming *.nc.gz is newer that one already on storage HARVEST_UPLOAD the content of the *nc.gz BUT check THAT We don't delete files not starting with a good value of GSLA_PREFIX_PATH. In our case we patch this global variable to empty to check this """ # create some PipelineFiles to represent the existing files on 'S3' preexisting_files = PipelineFileCollection() existing_file = PipelineFile(PREV_NC_GZ_STORAGE, dest_path=os.path.join( 'IMOS/OceanCurrent/GSLA/DM00/2018/', os.path.basename(PREV_NC_GZ_STORAGE))) preexisting_files.update([existing_file]) # set the files to UPLOAD_ONLY preexisting_files.set_publish_types( PipelineFilePublishType.UPLOAD_ONLY) # upload the 'preexisting_files' collection to the unit test's temporary upload location broker = get_storage_broker( self.config.pipeline_config['global']['upload_uri']) broker.upload(preexisting_files) # run the handler self.run_handler_with_exception( AttributeValidationError, NEWER_CREATION_DATE_NC_GZ, allowed_dest_path_regexes=["IMOS/OceanCurrent/GSLA"])
def preprocess(self): """ pre processsing to handle the conversion of BUFR csv files to NetCDF :return: """ csv_file = self.file_collection.filter_by_attribute_id( 'file_type', FileType.CSV) if csv_file: csv_file = csv_file[0] csv_file.publish_type = PipelineFilePublishType.ARCHIVE_ONLY profiles = parse_bufr_file(csv_file.src_path) profiles = return_unique_profiles( profiles) # check for duplicate profiles within BUFR file for profile in profiles: profile = fzf_vessel_get_info( profile) # fuzzy search finder for vessel name profile = xbt_line_get_info( profile, self.xbt_line_vocab_url ) # get hard coded info per xbt line netcdf_filepath = netcdf_writer( profile, self.temp_dir) # convert BUFR to NetCDF # publish nc_file = PipelineFile( netcdf_filepath, file_update_callback=self._file_update_callback) nc_file.publish_type = PipelineFilePublishType.HARVEST_UPLOAD self.file_collection.add(nc_file)
def set_deployment_status(self, input_file, message): """ Write message to Harvestmission.csv file. ingested in RT pieline Update the anfog_rt.harvest_listing table after ingestion of csv file by RT pipeline Note that to be consistent with the available message in the production DB, dashes need to be replaced by underscore, for ex delayed-mode =>delayed_mode :return: Harvestmission.csv updated with deployment specific status """ name = os.path.basename(input_file) deployment = AnfogFileClassifier.get_deployment_code(input_file) platform = AnfogFileClassifier.get_platform(name) listing_path = os.path.join(self.products_dir, AnfogFileClassifier.MISSION_LISTING) with open(listing_path, 'w') as f: f.write('deployment_name, platform_type, status' + os.linesep) row = "%s,%s,%s" % (deployment, platform, message.replace( '-', '_').lower()) f.write(row) product = PipelineFile(listing_path) product.publish_type = PipelineFilePublishType.HARVEST_ONLY product.check_type = PipelineFileCheckType.FORMAT_CHECK product.dest_path = os.path.join(self.upload_destination, os.path.basename(listing_path)) self.file_collection.add(product)
def test_good_dm_file_with_compliance_check(self): # this is tested as an update to avoid raising invalid input file error cause of missing ancillary material preexisting_file = PipelineFileCollection() existing_file = PipelineFile(GOOD_NC, dest_path=os.path.join( 'IMOS/ANFOG/slocum_glider/TwoRocks20180503a/', os.path.basename(GOOD_NC))) preexisting_file.update([existing_file]) # set the files to UPLOAD_ONLY preexisting_file.set_publish_types(PipelineFilePublishType.UPLOAD_ONLY) # upload the 'preexisting_files' collection to the unit test's temporary upload location broker = get_storage_broker(self.config.pipeline_config['global']['upload_uri']) broker.upload(preexisting_file) handler = self.run_handler(GOOD_NC, check_params={'checks': ['cf', 'imos:1.4']}) f = handler.file_collection.filter_by_attribute_id('file_type', FileType.NETCDF) # self.assertEqual(f[0].check_type, PipelineFileCheckType.NC_COMPLIANCE_CHECK) self.assertEqual(f[0].publish_type, PipelineFilePublishType.HARVEST_UPLOAD) self.assertEqual(f[0].dest_path, 'IMOS/ANFOG/slocum_glider/TwoRocks20180503a/' 'IMOS_ANFOG_BCEOPSTUV_20180503T080042Z_SL210_FV01_timeseries_END-20180505T054942Z.nc') self.assertTrue(f[0].is_checked) self.assertTrue(f[0].is_stored) self.assertTrue(f[0].is_harvested)
def test_download_remotepipelinefilecollection(self): state_query = StateQuery(storage_broker=self.storage_broker, wfs_broker=self.wfs_broker) pipeline_file = PipelineFile(GOOD_NC, dest_path='dest/path/1.nc') self.storage_broker.upload(pipeline_file) remote_file = RemotePipelineFile.from_pipelinefile(pipeline_file) state_query.download(remote_file, local_path=self.temp_dir) self.assertTrue(os.path.exists(remote_file.local_path))
def test_deletion_rt_after_dm_upload(self): """test deletion of RT mission at upload of related DM version""" preexisting_files = PipelineFileCollection() existing_file1 = PipelineFile(PREV_NC_RT, dest_path=os.path.join( 'IMOS/ANFOG/REALTIME/slocum_glider/TwoRocks20180503a/', os.path.basename(PREV_NC_RT))) existing_file2 = PipelineFile(PREV_PNG_TRANSECT, dest_path=os.path.join( 'IMOS/ANFOG/REALTIME/slocum_glider/TwoRocks20180503a/', os.path.basename(PREV_PNG_TRANSECT))) existing_file3 = PipelineFile(PREV_PNG_MISSION, dest_path=os.path.join( 'IMOS/ANFOG/REALTIME/slocum_glider/TwoRocks20180503a/', os.path.basename(PREV_PNG_MISSION))) preexisting_files.update([existing_file1, existing_file2, existing_file3]) # set the files to UPLOAD_ONLY preexisting_files.set_publish_types(PipelineFilePublishType.UPLOAD_ONLY) # upload the 'preexisting_files' collection to the unit test's temporary upload location broker = get_storage_broker(self.config.pipeline_config['global']['upload_uri']) broker.upload(preexisting_files) # run the handler handler = self.run_handler(GOOD_ZIP_DM, check_params={'checks': ['cf', 'imos:1.4']}) nc = handler.file_collection.filter_by_attribute_id('file_type', FileType.NETCDF) for n in nc: if n.name == os.path.basename(PREV_NC_RT): self.assertEqual(n.publish_type, PipelineFilePublishType.DELETE_UNHARVEST) self.assertTrue(n.is_deleted) elif re.match(AnfogFileClassifier.DM_REGEX, n.name): self.assertEqual(n.publish_type, PipelineFilePublishType.HARVEST_UPLOAD) self.assertTrue(n.is_harvested) self.assertTrue(n.is_stored) else: self.assertEqual(n.publish_type, PipelineFilePublishType.ARCHIVE_ONLY) self.assertTrue(n.is_archived) pngs = handler.file_collection.filter_by_attribute_id('file_type', FileType.PNG) for png in pngs: self.assertTrue(png.is_deleted)
def schedule_file_removal(self, remote_pipeline_file): """schedule a file to be removed.""" filename = remote_pipeline_file.name file_to_remove = PipelineFile( filename, is_deletion=True, dest_path=self.dest_path_function(filename), ) file_to_remove.publish_type = PipelineFilePublishType.DELETE_UNHARVEST self.file_collection.add(file_to_remove) logger.info( NRT_FILE_REMOVAL_MSG.format(file=file_to_remove, ptype=file_to_remove.publish_type))
def test_overwrite_same_file(self, mock_callsign): # check that files with same name are overwritten preexisting_files = PipelineFileCollection() existing_file1 = PipelineFile( GOOD_NC, dest_path=os.path.join( 'IMOS/SOOP/SOOP-BA/VKAD_Antarctic-Discovery/Antarctic-Discovery_20160116-20160129/', os.path.basename(GOOD_NC))) existing_file2 = PipelineFile( CSV, dest_path=os.path.join( 'IMOS/SOOP/SOOP-BA/VKAD_Antarctic-Discovery/Antarctic-Discovery_20160116-20160129/', os.path.basename(CSV))) preexisting_files.update([existing_file1, existing_file2]) # upload the 'preexisting_files' collection to the unit test's temporary upload location broker = get_storage_broker( self.config.pipeline_config['global']['upload_uri']) broker.upload(preexisting_files) # run the handler handler = self.run_handler(GOOD_ZIP) nc = handler.file_collection.filter_by_attribute_id( 'file_type', FileType.NETCDF) self.assertEqual(nc[0].publish_type, PipelineFilePublishType.HARVEST_UPLOAD) self.assertEqual(nc[0].is_deleted, False) csvs = handler.file_collection.filter_by_attribute_id( 'file_type', FileType.CSV) for csv in csvs: if csv.name == os.path.basename(CSV): self.assertEqual(csv.publish_type, PipelineFilePublishType.UPLOAD_ONLY) self.assertEqual(csv.is_deleted, False)
def preprocess(self): """ Files to be deleted as found in 'soop_trv_duplicate_url' wfs layer """ files_to_delete = self.state_query.query_wfs_urls_for_layer( 'soop_trv_duplicate_url') for f in files_to_delete: file_to_delete = PipelineFile( os.path.basename(f), is_deletion=True, dest_path=f, file_update_callback=self._file_update_callback) file_to_delete.publish_type = PipelineFilePublishType.DELETE_UNHARVEST self.file_collection.add(file_to_delete)
def test_renamed_rt_deployment(self): # test deletion of RT files when deployment renamed or when cleaning files on S3 preexisting_files = PipelineFileCollection() existing_file1 = PipelineFile(PREV_NC_RT, dest_path=os.path.join( 'IMOS/ANFOG/REALTIME/slocum_glider/TwoRocks20180503a/', os.path.basename(PREV_NC_RT))) existing_file2 = PipelineFile(PREV_PNG_TRANSECT, dest_path=os.path.join( 'IMOS/ANFOG/REALTIME/slocum_glider/TwoRocks20180503a/', os.path.basename(PREV_PNG_TRANSECT))) existing_file3 = PipelineFile(PREV_PNG_MISSION, dest_path=os.path.join( 'IMOS/ANFOG/REALTIME/slocum_glider/TwoRocks20180503a/', os.path.basename(PREV_PNG_MISSION))) preexisting_files.update([existing_file1, existing_file2, existing_file3]) # set the files to UPLOAD_ONLY preexisting_files.set_publish_types(PipelineFilePublishType.UPLOAD_ONLY) # upload the 'preexisting_files' collection to the unit test's temporary upload location broker = get_storage_broker(self.config.pipeline_config['global']['upload_uri']) broker.upload(preexisting_files) handler = self.run_handler(MISSION_STATUS_RENAMED) # Process should resuls in : input file unhandled , preexisting file should be deleted, cvs file harvested csv = handler.file_collection.filter_by_attribute_id('file_type', FileType.CSV) self.assertEqual(csv[0].publish_type, PipelineFilePublishType.HARVEST_ONLY) self.assertTrue(csv[0].is_harvested) nc = handler.file_collection.filter_by_attribute_id('file_type', FileType.NETCDF) self.assertEqual(nc[0].publish_type, PipelineFilePublishType.DELETE_UNHARVEST) self.assertTrue(nc[0].is_deleted) pngs = handler.file_collection.filter_by_attribute_id('file_type', FileType.PNG) for png in pngs: self.assertEqual(png.publish_type, PipelineFilePublishType.DELETE_ONLY) self.assertTrue(png.is_deleted)
def preprocess(self): """ Preprocessing for NRT and DM files - NRT: generate a NetCDF files based on input text file. Set the input file publish_type property to 'archive' - DM file collection: update the check_type and publish_type properties for non-NetCDF files. These files are not checked or harvested, but uploaded to S3 """ if self.custom_params is not None and self.custom_params.get( 'ship_callsign_ls'): self.ship_callsign_ls = self.custom_params['ship_callsign_ls'] else: self.ship_callsign_ls = ship_callsign_list() # Delayed mode file submitted as a zip archive if self.file_extension == '.zip': nc_file = self.file_collection.filter_by_attribute_id( 'file_type', FileType.NETCDF) if len(nc_file) != 1: raise InvalidInputFileError( "Expecting one netCDF file in ZIP archive '{zip}'".format( zip=os.path.basename(self.input_file))) # first process the NetCDF file to set the destination path for the file collection nc = nc_file[0] nc.dest_path = self.dest_path(nc.src_path) nc_dir_path = os.path.dirname(nc.dest_path) # SOOP-CO2 DM and FRMAP .txt,.pdf or/and .xml files. # Set check type to NONEMPTY and publish type to UPLOAD_ONLY non_nc_files = PipelineFileCollection( f for f in self.file_collection if f.file_type is not FileType.NETCDF) for non_nc in non_nc_files: non_nc.check_type = PipelineFileCheckType.FORMAT_CHECK non_nc.publish_type = PipelineFilePublishType.UPLOAD_ONLY non_nc.dest_path = os.path.join(nc_dir_path, non_nc.name) elif self.input_file.endswith('dat.txt'): # Single text file Realtime files (*dat.txt) rt_file = self.file_collection[0] rt_file.publish_type = PipelineFilePublishType.ARCHIVE_ONLY nrt_nc_file_path = soop_co2_nrt_nc_generator.process_co2_rt( rt_file, self.products_dir, self.ship_callsign_ls) nrt_nc_file = PipelineFile(nrt_nc_file_path) self.file_collection.add(nrt_nc_file) nrt_nc_file.publish_type = PipelineFilePublishType.HARVEST_UPLOAD
def get_notification_data(): collection = PipelineFileCollection(PipelineFile(GOOD_NC)) collection_headers, collection_data = collection.get_table_data() data = { 'input_file': 'good.nc', 'processing_result': 'HANDLER_SUCCESS', 'handler_start_time': '2017-10-23 16:05', 'checks': None, 'collection_headers': collection_headers, 'collection_data': collection_data, 'error_details': '', 'upload_dir': None } return data
def test_setup_upload_location_push_newer_yearly_file(self): """ Test case: Check creation date of incoming yearly *.nc.gz is newer that one already on storage UPLOAD_ONLY the new incoming *.nc.gz DELETE_ONLY the previous *.nc.gz NO_ACTION on the nc inside the *.nc.gz """ # create some PipelineFiles to represent the existing files on 'S3' preexisting_files = PipelineFileCollection() existing_file = PipelineFile( PREV_YEARLY_NC_GZ_STORAGE, dest_path=os.path.join( 'IMOS/OceanCurrent/GSLA/DM00/yearfiles', os.path.basename(PREV_YEARLY_NC_GZ_STORAGE))) preexisting_files.update([existing_file]) # set the files to UPLOAD_ONLY preexisting_files.set_publish_types( PipelineFilePublishType.UPLOAD_ONLY) # upload the 'preexisting_files' collection to the unit test's temporary upload location broker = get_storage_broker( self.config.pipeline_config['global']['upload_uri']) broker.upload(preexisting_files) # run the handler handler = self.run_handler(GOOD_YEARLY_FILE_DM00) nc_file = handler.file_collection.filter_by_attribute_id( 'file_type', FileType.NETCDF)[0] self.assertEqual(nc_file.publish_type, PipelineFilePublishType.NO_ACTION) nc_gz_file = handler.file_collection.filter_by_attribute_id( 'file_type', FileType.GZIP)[0] self.assertEqual(nc_gz_file.publish_type, PipelineFilePublishType.UPLOAD_ONLY) nc_gz_delete = handler.file_collection.filter_by_attribute_value( 'name', os.path.basename(PREV_YEARLY_NC_GZ_STORAGE))[0] self.assertEqual(nc_gz_delete.publish_type, PipelineFilePublishType.DELETE_ONLY)
def _cleanup_previous_version(self, product_filename): """Identify any previously published version(s) of the given product file and mark them for deletion. Ignores cases where the previous version has exactly the same file name, as this will simply be overwritten. :param product_filename: File name of the newly generated product """ product_type = get_product_type(product_filename) for old_product_url in self.old_product_files.get(product_type, []): if os.path.basename(old_product_url) != product_filename: # Add the previous version as a "late deletion". It will be deleted during the handler's `publish` # step after (and only if) all new files have been successfully published. old_file = PipelineFile( old_product_url, dest_path=old_product_url, is_deletion=True, late_deletion=True, file_update_callback=self._file_update_callback) old_file.publish_type = PipelineFilePublishType.DELETE_UNHARVEST self.file_collection.add(old_file)
def test_setup_upload_location_push_file_newer_creation_date(self): """ Test case: Check creation date of new *.nc is newer that one already on storage """ # create some PipelineFiles to represent the existing files on 'S3' preexisting_files = PipelineFileCollection() existing_file = PipelineFile( GOOD_NC_FV01, dest_path=AcornHandler.dest_path(GOOD_NC_FV01)) preexisting_files.update([existing_file]) # set the files to UPLOAD_ONLY preexisting_files.set_publish_types( PipelineFilePublishType.UPLOAD_ONLY) # upload the 'preexisting_files' collection to the unit test's temporary upload location broker = get_storage_broker( self.config.pipeline_config['global']['upload_uri']) broker.upload(preexisting_files) # create a new file based on GOOD_NC_FV01. Modify it with an newer creation date # we patch the global variable from the handler in order to use the temporary broker file nc_file_new_creation_date_path = os.path.join( self.temp_dir, os.path.basename(GOOD_NC_FV01)) shutil.copyfile(GOOD_NC_FV01, nc_file_new_creation_date_path) with Dataset(nc_file_new_creation_date_path, mode='r+') as nc_obj: delta_time = timedelta(1, 1, 1) new_time = datetime.strptime(nc_obj.date_created, '%Y-%m-%dT%H:%M:%SZ') + delta_time nc_obj.date_created = datetime.strftime(new_time, '%Y-%m-%dT%H:%M:%SZ') # run the handler on the new file with an newer creation date handler = self.handler_class(nc_file_new_creation_date_path, include_regexes=[r'IMOS_ACORN_.*\.nc']) handler.opendap_root = broker.prefix handler.run() nc_file = handler.file_collection.filter_by_attribute_id( 'file_type', FileType.NETCDF)[0] self.assertEqual(nc_file.publish_type, PipelineFilePublishType.HARVEST_UPLOAD)
def preprocess(self): """Check that every input file is valid according to the include/exclude regex patterns. Any non-matching file will be left with publish_type UNSET after the _resolve step. If there are any netCDF files from burst-sampling instruments in the collection, create the burst-averaged version of each and add them to the collection. :return: None """ self.logger.info( "Checking for invalid files and adjusting check/publish properties." ) invalid_files = self.file_collection.filter_by_attribute_id( 'publish_type', PipelineFilePublishType.UNSET) if invalid_files: raise InvalidFileNameError( "File name(s) don't match the pattern expected for this upload location: {names}" .format(names=invalid_files.get_attribute_list('name'))) # Burst-processing for FV01 files with burst-sampling global attributes burst_files = (self.file_collection.filter_by_attribute_id( 'file_type', FileType.NETCDF).filter_by_attribute_regex('name', r'.*_FV01_')) for f in burst_files: with Dataset(f.src_path, mode='r') as D: has_interval = hasattr(D, 'instrument_burst_interval') has_duration = hasattr(D, 'instrument_burst_duration') is_adcp = ('DIST_ALONG_BEAMS' in D.dimensions or 'HEIGHT_ABOVE_SENSOR' in D.dimensions) if not (has_interval and has_duration) or is_adcp: continue self.logger.info("Burst-processing {f.name}".format(f=f)) product_path = create_burst_average_netcdf(f.src_path, self.products_dir) product_file = PipelineFile( product_path, file_update_callback=self._file_update_callback) product_file.publish_type = PipelineFilePublishType.HARVEST_UPLOAD self.file_collection.add(product_file)
def test_setup_upload_location_push_file_older_creation_date(self): """ Test case: Check creation date of new *.nc is older that one already on storage """ # create some PipelineFiles to represent the existing files on 'S3' preexisting_files = PipelineFileCollection() existing_file = PipelineFile( GOOD_NC_FV01, dest_path=AcornHandler.dest_path(GOOD_NC_FV01)) preexisting_files.update([existing_file]) # set the files to UPLOAD_ONLY preexisting_files.set_publish_types( PipelineFilePublishType.UPLOAD_ONLY) # upload the 'preexisting_files' collection to the unit test's temporary upload location broker = get_storage_broker( self.config.pipeline_config['global']['upload_uri']) broker.upload(preexisting_files) # create a new file based on GOOD_NC_FV01. Modify it with an older creation date nc_file_old_creation_date_path = os.path.join( self.temp_dir, os.path.basename(GOOD_NC_FV01)) shutil.copyfile(GOOD_NC_FV01, nc_file_old_creation_date_path) with Dataset(nc_file_old_creation_date_path, mode='r+') as nc_obj: delta_time = timedelta(1, 1, 1) new_time = datetime.strptime(nc_obj.date_created, '%Y-%m-%dT%H:%M:%SZ') - delta_time nc_obj.date_created = datetime.strftime(new_time, '%Y-%m-%dT%H:%M:%SZ') # run the handler on the new file with an older creation date handler = self.handler_class(nc_file_old_creation_date_path, include_regexes=[r'IMOS_ACORN_.*\.nc']) handler.opendap_root = broker.prefix handler.run() self.assertIsInstance(handler.error, InvalidFileContentError)
def test_dstg(self): preexisting_file = PipelineFileCollection() existing_file = PipelineFile(DSTG, dest_path=os.path.join( 'Department_of_Defence/DSTG/slocum_glider/TalismanSaberB20130706/', os.path.basename(DSTG))) preexisting_file.update([existing_file]) # set the files to UPLOAD_ONLY preexisting_file.set_publish_types(PipelineFilePublishType.UPLOAD_ONLY) # upload the 'preexisting_files' collection to the unit test's temporary upload location broker = get_storage_broker(self.config.pipeline_config['global']['upload_uri']) broker.upload(preexisting_file) # test processing of DSTG and NRL NetCDF files handler = self.run_handler(DSTG) f = handler.file_collection[0] self.assertEqual(f.publish_type, PipelineFilePublishType.HARVEST_UPLOAD) self.assertEqual(f.dest_path, 'Department_of_Defence/DSTG/slocum_glider/TalismanSaberB20130706/' + f.name) self.assertTrue(f.is_stored) self.assertTrue(f.is_harvested)
def test_setup_upload_location_push_same_file(self): """ Test case: Push same file twice to $INCOMING_DIR HARVEST_UPLOAD the incoming *.nc.gz NO_ACTION on the nc inside the *.nc.gz """ # create some PipelineFiles to represent the existing files on 'S3' preexisting_files = PipelineFileCollection() existing_file = PipelineFile(PREV_NC_GZ_STORAGE, dest_path=os.path.join( 'IMOS/OceanCurrent/GSLA/DM00/2018/', os.path.basename(PREV_NC_GZ_STORAGE))) preexisting_files.update([existing_file]) # set the files to UPLOAD_ONLY preexisting_files.set_publish_types( PipelineFilePublishType.UPLOAD_ONLY) # upload the 'preexisting_files' collection to the unit test's temporary upload location broker = get_storage_broker( self.config.pipeline_config['global']['upload_uri']) broker.upload(preexisting_files) # run the handler by uploading again the same file handler = self.run_handler(PREV_NC_GZ_STORAGE) nc_file = handler.file_collection.filter_by_attribute_id( 'file_type', FileType.NETCDF)[0] self.assertEqual(nc_file.publish_type, PipelineFilePublishType.NO_ACTION) nc_gz_file = handler.file_collection.filter_by_attribute_id( 'file_type', FileType.GZIP)[0] self.assertEqual(nc_gz_file.publish_type, PipelineFilePublishType.HARVEST_UPLOAD)
with open(GETFEATURE_FILE) as f: TEST_GETFEATURE_JSON = f.read() with open(GETFEATURE_OLD_PRODUCTS_FILE) as f: TEST_GETFEATURE_OLD_PRODUCTS_JSON = f.read() with open(GETFEATURE_EMPTY_FILE) as f: TEST_GETFEATURE_EMPTY_JSON = f.read() # Create collection of input files for the products # These will be uploaded to the mocked equivalent of S3 (where the real input files will be) features = json.loads(TEST_GETFEATURE_JSON)['features'] INPUT_FILE_COLLECTION = PipelineFileCollection() for f in features: pf = PipelineFile( os.path.join(TEST_ROOT, os.path.basename(f['properties']['url'])), dest_path=f['properties']['url'] ) pf.publish_type = PipelineFilePublishType.UPLOAD_ONLY INPUT_FILE_COLLECTION.add(pf) class TestMooringsProductsHandler(HandlerTestCase): def setUp(self): self.handler_class = MooringsProductsHandler upload_broker = get_storage_broker(self.config.pipeline_config['global']['upload_uri']) upload_broker.upload(INPUT_FILE_COLLECTION) super().setUp() @patch('aodncore.util.wfs.WebFeatureService') def test_all_products(self, mock_webfeatureservice): mock_webfeatureservice().getfeature().getvalue.side_effect = [TEST_GETFEATURE_JSON,
def get_previous_version(self, previous_file_list, path, input_file_name): """ Find previous version of each incoming file based on its type/extension and add them to the filecollection with the correct publish type extension can be: .inf', '.nc.png','.pitch.csv','.roll.csv',.gps.csv' inputs: previous_file_list : dictionary containing file listing(full path) and metadata from destination input_file : file basename path : full destination path """ if not previous_file_list: return files_to_delete = PipelineFileCollection() try: extension = ALLOWED_CONTENT_EXTENSIONS.match( input_file_name).groupdict()['extension'] except KeyError: raise ValueError( "unable to determine extension from file name {infile}".format( infile=input_file_name)) # get list of previous files basename to search through basenames = {os.path.basename(f) for f in previous_file_list} this_extension_pattern = re.compile( r".*\.{ext}$".format(ext=extension)) if input_file_name not in basenames: previous_file = [ f for f in previous_file_list if this_extension_pattern.match(f) ] if extension == 'nc': if len(previous_file) != 1: raise ValueError( "Expected exactly 1 previous versions of the netcdf file, found {n}. Aborting " .format(n=len(previous_file))) else: # if uploaded file name has the same name published file => no action, file will be overwritten, otherwise # sort file per wildcard and work out which one to delete ( # check previous file widcard : # can be '.inf', '.nc.png','.pitch.csv','.roll.csv',.gps.csv' if len(previous_file) > 1: raise ValueError( "Found more than one previous versions of the extension '{ext}'. Aborting" .format(ext=extension)) elif len(previous_file) == 0: return prev_file = previous_file[0] dest_path = os.path.join(path, os.path.basename(prev_file)) self.logger.info( "adding deletion of previous file '{dest_path}'".format( dest_path=dest_path)) file_to_delete = PipelineFile(prev_file, is_deletion=True, dest_path=dest_path) if extension == 'nc': file_to_delete.publish_type = PipelineFilePublishType.DELETE_UNHARVEST else: file_to_delete.publish_type = PipelineFilePublishType.DELETE_ONLY files_to_delete.add(file_to_delete) return files_to_delete
def preprocess(self): # if input file is a NetCDF, create a .nc.gz and harvest upload it. # historically, files were always sent as *.nc.gz. But as of April 2021, files might be pushed as *.nc. # to be consistent, we transform this .nc into a .nz.gz if self.file_type is FileType.NETCDF: self.file_collection.set_publish_types( PipelineFilePublishType.NO_ACTION) gzip_path = os.path.join(self.temp_dir, self.file_basename + '.gz') with open(self.input_file, 'rb') as f_in, gzip.open(gzip_path, 'wb') as gz_out: gz_out.writelines(f_in) # publish self.add_to_collection( gzip_path, publish_type=PipelineFilePublishType.HARVEST_UPLOAD) if self.file_type is FileType.GZIP: # add nc_gz file to collection (not by default) self.file_collection.add(self.input_file_object) netcdf_file_gz_collection = self.file_collection.filter_by_attribute_id( 'file_type', FileType.GZIP) netcdf_file_gz = netcdf_file_gz_collection[0] netcdf_file_gz.publish_type = PipelineFilePublishType.HARVEST_UPLOAD # default # some GSLA files are gzipped, so gunzip them before checking them # if uploaded file is GZIP check that GZIP contains a NetCDF netcdf_collection = self.file_collection.filter_by_attribute_id( 'file_type', FileType.NETCDF) if len(netcdf_collection) != 1: raise InvalidInputFileError( "Expecting one netCDF file in GZIP archive '{gzip}'". format(gzip=os.path.basename(self.input_file))) netcdf_file_gz = self.file_collection.filter_by_attribute_id( 'file_type', FileType.GZIP)[0] netcdf_file = self.file_collection.filter_by_attribute_id( 'file_type', FileType.NETCDF)[0] # setting the path of the gz file with the gunzipped file netcdf_file_gz.dest_path = self.dest_path(netcdf_file.src_path) # Nothing to do with *.nc. Talend can harvest *.nc.gz. Set to NO_ACTION netcdf_file.publish_type = PipelineFilePublishType.NO_ACTION # we don't know the product type (DM00 or DM01) of the file already # on s3 in order to deduce its path. We need to get the product # type from the file in incoming result_previous_version_creation_date = self.get_previous_version_creation_date( netcdf_file.src_path) """ default values by default we push to the storage the file landed in the pipeline (ie *.nc.gz) """ push_new_file = True remove_previous_version = False # compare creation dates with file already on storage if result_previous_version_creation_date: new_file_creation_date = get_creation_date(netcdf_file.name) if result_previous_version_creation_date > new_file_creation_date: push_new_file = False elif result_previous_version_creation_date == new_file_creation_date: push_new_file = True else: remove_previous_version = True previous_file_path = self.get_previous_version_object( netcdf_file.src_path) if push_new_file: if GSLA_REGEX_YEARLY.match(netcdf_file.name): # yearly file should never be harvested netcdf_file_gz.publish_type = PipelineFilePublishType.UPLOAD_ONLY else: raise InvalidFileNameError( "file name: \"{filename}\" creation date is older than file already on " "storage".format(filename=netcdf_file_gz.name)) # deletion of the previous file if remove_previous_version: previous_file_name = os.path.basename(previous_file_path) file_to_delete = PipelineFile( previous_file_name, is_deletion=True, dest_path=previous_file_path, file_update_callback=self._file_update_callback) if GSLA_REGEX_YEARLY.match(netcdf_file.name): file_to_delete.publish_type = PipelineFilePublishType.DELETE_ONLY else: file_to_delete.publish_type = PipelineFilePublishType.DELETE_UNHARVEST self.file_collection.add(file_to_delete)
def test_delete_previous_file(self, mock_callsign): # create some PipelineFiles to represent the existing files on 'S3' preexisting_files = PipelineFileCollection() existing_file1 = PipelineFile( PREV_NC, dest_path=os.path.join( 'IMOS/SOOP/SOOP-BA/VKAD_Antarctic-Discovery/Antarctic-Discovery_20160116-20160129/', os.path.basename(PREV_NC))) existing_file2 = PipelineFile( CSV, dest_path=os.path.join( 'IMOS/SOOP/SOOP-BA/VKAD_Antarctic-Discovery/Antarctic-Discovery_20160116-20160129/', os.path.basename(CSV))) existing_file3 = PipelineFile( PNG, dest_path=os.path.join( 'IMOS/SOOP/SOOP-BA/VKAD_Antarctic-Discovery/Antarctic-Discovery_20160116-20160129/', os.path.basename(PNG))) preexisting_files.update( [existing_file1, existing_file2, existing_file3]) # set the files to UPLOAD_ONLY preexisting_files.set_publish_types( PipelineFilePublishType.UPLOAD_ONLY) # upload the 'preexisting_files' collection to the unit test's temporary upload location broker = get_storage_broker( self.config.pipeline_config['global']['upload_uri']) broker.upload(preexisting_files) # run the handler handler = self.run_handler(GOOD_ZIP) # add some tests to make sure the previous files were handled appropriately, e.g. # - they were added as deletions # - they were successfully deleted # - they were the *only* ones deleted nc_files = handler.file_collection.filter_by_attribute_id( 'file_type', FileType.NETCDF) for nc in nc_files: if nc.name == os.path.basename(PREV_NC): self.assertEqual(nc.publish_type, PipelineFilePublishType.DELETE_UNHARVEST) self.assertEqual(nc.is_deleted, True) else: self.assertEqual(nc.is_deleted, False) csvs = handler.file_collection.filter_by_attribute_id( 'file_type', FileType.CSV) for csv in csvs: if csv.name == os.path.basename(CSV): self.assertEqual(csv.publish_type, PipelineFilePublishType.UPLOAD_ONLY) self.assertEqual(csv.is_deleted, False) pngs = handler.file_collection.filter_by_attribute_id( 'file_type', FileType.PNG) for png in pngs: if png.name == os.path.basename(PNG): self.assertEqual(png.is_deleted, True)