TEST_GETFEATURE_OLD_PRODUCTS_JSON = f.read() with open(GETFEATURE_EMPTY_FILE) as f: TEST_GETFEATURE_EMPTY_JSON = f.read() # Create collection of input files for the products # These will be uploaded to the mocked equivalent of S3 (where the real input files will be) features = json.loads(TEST_GETFEATURE_JSON)['features'] INPUT_FILE_COLLECTION = PipelineFileCollection() for f in features: pf = PipelineFile( os.path.join(TEST_ROOT, os.path.basename(f['properties']['url'])), dest_path=f['properties']['url'] ) pf.publish_type = PipelineFilePublishType.UPLOAD_ONLY INPUT_FILE_COLLECTION.add(pf) class TestMooringsProductsHandler(HandlerTestCase): def setUp(self): self.handler_class = MooringsProductsHandler upload_broker = get_storage_broker(self.config.pipeline_config['global']['upload_uri']) upload_broker.upload(INPUT_FILE_COLLECTION) super().setUp() @patch('aodncore.util.wfs.WebFeatureService') def test_all_products(self, mock_webfeatureservice): mock_webfeatureservice().getfeature().getvalue.side_effect = [TEST_GETFEATURE_JSON, TEST_GETFEATURE_OLD_PRODUCTS_JSON] handler = self.run_handler(GOOD_MANIFEST)
def get_previous_version(self, previous_file_list, path, input_file_name): """ Find previous version of each incoming file based on its type/extension and add them to the filecollection with the correct publish type extension can be: .inf', '.nc.png','.pitch.csv','.roll.csv',.gps.csv' inputs: previous_file_list : dictionary containing file listing(full path) and metadata from destination input_file : file basename path : full destination path """ if not previous_file_list: return files_to_delete = PipelineFileCollection() try: extension = ALLOWED_CONTENT_EXTENSIONS.match( input_file_name).groupdict()['extension'] except KeyError: raise ValueError( "unable to determine extension from file name {infile}".format( infile=input_file_name)) # get list of previous files basename to search through basenames = {os.path.basename(f) for f in previous_file_list} this_extension_pattern = re.compile( r".*\.{ext}$".format(ext=extension)) if input_file_name not in basenames: previous_file = [ f for f in previous_file_list if this_extension_pattern.match(f) ] if extension == 'nc': if len(previous_file) != 1: raise ValueError( "Expected exactly 1 previous versions of the netcdf file, found {n}. Aborting " .format(n=len(previous_file))) else: # if uploaded file name has the same name published file => no action, file will be overwritten, otherwise # sort file per wildcard and work out which one to delete ( # check previous file widcard : # can be '.inf', '.nc.png','.pitch.csv','.roll.csv',.gps.csv' if len(previous_file) > 1: raise ValueError( "Found more than one previous versions of the extension '{ext}'. Aborting" .format(ext=extension)) elif len(previous_file) == 0: return prev_file = previous_file[0] dest_path = os.path.join(path, os.path.basename(prev_file)) self.logger.info( "adding deletion of previous file '{dest_path}'".format( dest_path=dest_path)) file_to_delete = PipelineFile(prev_file, is_deletion=True, dest_path=dest_path) if extension == 'nc': file_to_delete.publish_type = PipelineFilePublishType.DELETE_UNHARVEST else: file_to_delete.publish_type = PipelineFilePublishType.DELETE_ONLY files_to_delete.add(file_to_delete) return files_to_delete