def check_parameters(dataf, vessel_code, input_param, src_file): """ Checks parameters list contains all required parameter(vessel specific) Cast selected parameter data to correct type Checks that Lat/Lon are not all missing. Returns updated dataframe """ rt_input_parameters = set.union( INPUT_RT_PARAMETERS, eval(vessel_code + '_SPECIFIC_INPUT_PARAMS')) if not all(param in input_param for param in rt_input_parameters): missing_param = [] for required_param in rt_input_parameters: if required_param not in input_param: missing_param = missing_param.append(required_param) raise InvalidFileContentError( "Missing parameter(s) '{missing_param}' in file '{src_file}'.Aborting" .format(missing_param=missing_param, src_file=src_file)) else: # required_param all present . Change dtype to numeric where relevant # var TYPE conversion to string outside this function for param in rt_input_parameters: if param not in set(['Type', 'PcDate', 'PcTime']): dataf[param] = dataf[param].apply( pd.to_numeric, errors='coerce') # convert bad non numeric to NaN if all(np.isnan(dataf['GpsShipLatitude'])) or all( np.isnan(dataf['GpsShipLongitude'])): raise InvalidFileContentError( "Latitude and/or Longitude values all missing in file '{src_file}'.Aborting" .format(src_file=src_file)) return dataf
def _read_manifest(self): """Read the manifest file and extract key parameters for product""" with open(self.input_file) as f: manifest = json.load(f) try: self.product_site_code = manifest['site_code'] self.product_variables = manifest['variables'] except KeyError: raise InvalidFileContentError( "manifest file '{self.input_file}' missing information (site_code, variables)" .format(self=self)) if 'products' in manifest: invalid_products = set(manifest['products']) - self.VALID_PRODUCTS if invalid_products: raise InvalidFileContentError( "invalid product(s) {invalid_products} requested " "in manifest file '{self.input_file}'".format( invalid_products=invalid_products, self=self)) self.products_to_create = set(manifest['products']) # Even if only the gridded product is explicitly requested, we need to re-generate and publish the hourly too, # as it is the input file for the gridded. if 'gridded' in self.products_to_create: self.products_to_create.add('hourly')
def dest_path_aodn_wave_dm(filepath): file_basename = os.path.basename(filepath) with Dataset(filepath, mode='r') as nc_obj: site_name = nc_obj.site_name if BOM_WAVERIDER.match(file_basename): data_base_dir = os.path.join(BOM_DIR, WAVERIDER_DIR, DELAYED_DIR) product_dir = site_name.replace(' ', '_') elif DES_QLD_WAVERIDER.match(file_basename): data_base_dir = os.path.join(DES_QLD_DIR, WAVERIDER_DIR, DELAYED_DIR) fields = get_pattern_subgroups_from_string(file_basename, DES_QLD_WAVERIDER) product_dir = fields['site_code'] elif DOT_WA_WAVERIDER.match(file_basename): data_base_dir = os.path.join(DOT_WA_DIR, WAVERIDER_DIR, DELAYED_DIR) fields = get_pattern_subgroups_from_string(file_basename, DOT_WA_WAVERIDER) product_dir = os.path.join(site_name.replace(' ', '_'), fields['site_code']) elif MHL_WAVERIDER.match(file_basename): data_base_dir = os.path.join(MHL_DIR_BASE, MHL_DIR, MHL_WAVERIDER_DIR) product_dir = site_name.replace(' ', '_') elif DOT_WA_AWAC.match(file_basename): data_base_dir = os.path.join(DOT_WA_DIR, AWAC_DIR, DELAYED_DIR) fields = get_pattern_subgroups_from_string(file_basename, DOT_WA_AWAC) product_dir = fields['site_code'] elif DTA_NZ_WAVERIDER.match(file_basename): data_base_dir = os.path.join(DTA_NZ_DIR, WAVERIDER_DIR, DELAYED_DIR) if 'Wave Rider Buoy' not in site_name: raise InvalidFileContentError( "file name: \"{filename}\"; global attribute site_code does not contain 'Wave Rider Buoy' string to " \ "deduce path".format(filename=file_basename)) product_dir = site_name.replace('Wave Rider Buoy', '').strip().replace(' ', '_') elif NTP_WAVE.match(file_basename): data_base_dir = os.path.join(NTP_WAVE_DIR, WAVERIDER_DIR, DELAYED_DIR) if len(site_name) == 0: raise InvalidFileContentError( "file name: \"{filename}\"; global attribute site_name is empty" .format(filename=file_basename)) product_dir = site_name else: raise InvalidFileNameError( "file name: \"{filename}\" not matching regex to deduce path". format(filename=file_basename)) return os.path.join(data_base_dir, product_dir, os.path.basename(filepath))
def preprocess(self): nc_file = self.file_collection[0] # if file contains any of FILE_TYPE_NEED_INDEX, we index, otherwise publish only if any(s in get_type(nc_file.name) for s in FILE_TYPE_NEED_INDEX): nc_file.publish_type = PipelineFilePublishType.HARVEST_UPLOAD else: nc_file.publish_type = PipelineFilePublishType.UPLOAD_ONLY # check if file with same dest_path already on s3. If yes, check its date_created nc attribute to know # if we need to overwrite this object or not destination_s3 = self.dest_path(nc_file.name) storage_query_res = self.state_query.query_storage( destination_s3).keys() # creation date of the new file in the pipeline if destination_s3 in storage_query_res: creation_date_nc_pipeline = get_creation_date(nc_file.src_path) # creation date of the file already published creation_date_nc_s3 = get_creation_date( os.path.join(self.opendap_root, destination_s3)) if creation_date_nc_pipeline < creation_date_nc_s3: raise InvalidFileContentError( "file name: \"{filename}\" creation date is older than file already on storage" .format(filename=nc_file.name))
def process(self): """Handle a zip file containing images and no NetCDF files. In this case we just want to publish the zip file itself, not the individual images. If we encounter a "mixed" zip file with images and netCDF files, we're just going to give up, for now. """ images = PipelineFileCollection(f for f in self.file_collection if f.file_type.is_image_type) netcdfs = self.file_collection.filter_by_attribute_id( 'file_type', FileType.NETCDF) is_zip = self.file_type is FileType.ZIP have_images = len(images) > 0 have_netcdfs = len(netcdfs) > 0 if is_zip and have_images: if have_netcdfs: raise InvalidFileContentError( "Zip file contains both images and netCDFs. Don't know what to do!" " They are handled differently, so please upload only one at a time." ) if not DwmFileClassifier.SOTS_IMAGES_ZIP_PATTERN.match( self.file_basename): raise InvalidFileNameError( "Zip file contains images, but its name does not match pattern for images zip file " "(regular expression '{p}')".format( p=DwmFileClassifier.SOTS_IMAGES_ZIP_PATTERN.pattern)) self.logger.info( "Zip file contains images and no netCDF files. " "Publishing original zip file instead of its contents.") self.file_collection.set_publish_types( PipelineFilePublishType.NO_ACTION) self.input_file_object.publish_type = PipelineFilePublishType.HARVEST_UPLOAD self.file_collection.add(self.input_file_object)
def get_deployment_code(cls, src_path): """Depending on data mode : DM : get deployment code from netcdf global attributes directly DSTG : no attribute deployment_code, extract deployment code from title instead RT :exctract deployment code from title """ name = os.path.basename(src_path) if re.match(cls.DSTG_REGEX, name) or re.match(cls.ANFOG_RT_REGEX, name): title = cls._get_nc_att(src_path, 'title') deployment_code = title.split()[-1] if deployment_code == 'mission': raise InvalidFileContentError( "Missing deployment code in {file} ".format(file=name)) elif re.match(cls.ANFOG_NC_REGEX, name) or re.match( cls.ADAPTER_REGEX, name): deployment_code = cls._get_nc_att(src_path, 'deployment_code') elif name.endswith('.txt'): # extract deployment code from filename like SL-Yamba20180609_completed.txt field = name.split('_') deployment_code = field[0].split('-')[1] else: raise InvalidFileNameError( "Invalidfile name {file} ".format(file=name)) return deployment_code
def preprocess(self): pz = NSWOEHSurveyProcesor(self.input_file) report = pz.check_all() if len(report) > 0: raise_string = '' for heading, messages in report.items(): raise_string += "\n{heading}:\n{messages}\n".format( heading=heading, messages="\n".join([ "{0}. {1}".format(i + 1, messages) for i, messages in enumerate(messages) ])) raise InvalidFileContentError( "Zip file '{name}' failed content checks:\n{raise_string}". format(name=os.path.basename(self.input_file), raise_string=raise_string)) # STAX & LIDAR surveys - publish zip file and shapefile only if pz.survey_methods in BASIC_PACKAGED_METHODS: for f in self.file_collection: if not SHAPEFILE_PATTERN.match(f.name): f.publish_type = PipelineFilePublishType.NO_ACTION self.file_collection.add(self.input_file_object) self.input_file_object.publish_type = PipelineFilePublishType.HARVEST_UPLOAD # For multi-beam (MB) surveys the default (publish all zipfile contents) is correct self.survey_path = pz.get_dest_path()
def dest_path_anmn_nrs_realtime(filepath): """Returns the relative path a given netCDF file should be published to, based on the name and content of the file. Only works for ANMN NRS real-time files. :param filepath: full path of the file :return: relative destination path including file name """ filename = os.path.basename(filepath) # Start with base path for this sub-facility path_list = ['IMOS', 'ANMN', 'NRS', 'REAL_TIME'] # add site code with Dataset(filepath, mode='r') as f: site_code = getattr(f, 'site_code', '') if not site_code: raise InvalidFileContentError("File '{name}' has no site_code attribute!".format(name=filename)) path_list.append(site_code) # add product sub-directory if re.match('IMOS_ANMN-NRS_MT_.*-Surface-.*-MET', filename): path_list.append('Meteorology') elif re.match('IMOS_ANMN-NRS_W_.*-Surface-.*-wave', filename): path_list.append('Wave') elif re.match('IMOS_ANMN-NRS_TPSOBUE_.*-SubSurface-.*-WQM', filename): path_list.append('Biogeochem_timeseries') else: raise InvalidInputFileError( "File name '{name}' doesn't match pattern for any known NRS real-time product".format(name=filename) ) path_list.append(filename) return os.path.join(*path_list)
def fzf_vessel_get_info(profile): """ fuzzy search finder for vessel callsign. The vessel name found in the BUFR file format is slightly different from the platform code AODN vocabulary. This function finds the closest match and modifies the name of the vessel accordingly with a confidence of 0.9 :param profile: :return: profile """ callsign_list = ship_callsign_list() ship_name = profile['profile_metadata']['ship_name'] ship_name_fuzzy_search = fwprocess.extractOne(ship_name, list(callsign_list.values()), score_cutoff=90) try: ship_name = ship_name_fuzzy_search[0] except: raise InvalidFileContentError( '{ship_name} is not a valid enough value to fuzzy match it with an existing AODN ' 'vessel name'.format(ship_name=ship_name)) callsign = next(k for k, v in callsign_list.items() if v == ship_name) # find callsign from key value # special case for the Astrolabe vessel. All NRT XBT data (post 2020) is collected by the new Astrolabe vessel, # which has a new callsign. # 2 callsigns are available in the ANDS vocabulary for this vessel; We simply force it to use the new vessel only to # avoid writing complicated and unnecessary logic if callsign == 'FHZI': callsign = 'FASB' profile['profile_metadata']['ship_name'] = ship_name profile['profile_metadata']['Callsign'] = callsign return profile
def transform_count_to_real_val(df): """ 1sec files measure FLU2 and TURB in counts. Transforming to CPHL and TURB 10secs are already in CPHL and TURB Ref: https://github.com/aodn/imos-toolbox/blob/spirit/Preprocessing/spiritCountToEngPP.txt """ measurement_frequency = get_measurement_frequency(df) if measurement_frequency == 1: # transform FLU count data to CPHL df['CPHL'] = (df['CPHL'].values - CHLU_PARAMS['blank']) * CHLU_PARAMS['scale'] # transform TURB count data to TURB df['TURB'] = (df['TURB'].values - TURB_PARAMS['blank']) * TURB_PARAMS['scale'] elif measurement_frequency == 10: # Nothing to transform pass else: raise InvalidFileContentError( "SOOP NRT input logfile has incorrect delta time. '{measurement_frequency}'. Not belonging to any of " "('10 secs', '1 sec').".format( measurement_frequency=measurement_frequency)) return df
def _get_data_category(cls, input_file): """Determine the category a file belongs to (Temperature, CTD_timeseires, Biogeochem_profile, etc..) """ var_names = set(cls._get_variable_names(input_file)) if var_names.intersection(cls.VELOCITY_VAR): return 'Velocity' if var_names.intersection(cls.WAVE_VAR): return 'Wave' if var_names.intersection(cls.CO2_VAR): return 'CO2' feature_type = cls._get_nc_att(input_file, 'featureType').lower() if feature_type == 'profile': if var_names.intersection(cls.BGC_VAR) or var_names.intersection(cls.SALINITY_VAR): return 'Biogeochem_profiles' else: raise InvalidFileContentError( "Could not determine data category for '{name}'".format(name=input_file) ) if feature_type == 'timeseries': if var_names.intersection(cls.BGC_VAR): return 'Biogeochem_timeseries' if var_names.intersection(cls.SALINITY_VAR): return 'CTD_timeseries' if feature_type == 'timeseriesprofile' and 'long-timeseries' in input_file: return 'aggregated_products' if var_names.intersection(cls.TEMP_VAR): return 'Temperature' raise InvalidFileContentError("Could not determine data category for '{name}'".format(name=input_file))
def _get_old_data_category(cls, input_file): """Determine the category a file belongs to.""" var_names = set(cls._get_variable_names(input_file)) if var_names.intersection(cls.WAVE_VAR): return 'Surface_waves' if var_names.intersection(cls.FLUX_VAR): return 'Surface_fluxes' if var_names.intersection(cls.MET_VAR): return 'Surface_properties' raise InvalidFileContentError("Could not determine data category for {input_file}".format(input_file=input_file))
def dest_path(self, filepath): with Dataset(filepath, mode='r') as nc_obj: measurement_frequency = nc_obj.measurement_frequency product_type = nc_obj.product_type year = datetime.strptime(nc_obj.time_coverage_start, '%Y-%m-%dT%H:%M:%SZ').strftime("%Y") if measurement_frequency != "1sec": raise InvalidFileContentError( "SOOP TMV NRT: NetCDF with a measurement frequency of " "{measurement_frequency} aren't allowed to be harvested". format(measurement_frequency=measurement_frequency)) return os.path.join(self.soop_tmv_dir, product_type, measurement_frequency, year, os.path.basename(filepath))
def xbt_line_get_info(profile, url): """ retrieve xbt line information from ANDS vocabulary and store new values in existing profile dictionary :param profile: :param url: url of the ANDS xbt line vocabulary :return: """ helper = XbtLineVocabHelper(url) xbt_lines_info = helper.xbt_line_info() xbt_line = profile['profile_metadata']['XBT_line'] # another weird case with missing line info if not xbt_line.strip(): xbt_line = 'NOLINE' profile['profile_metadata']['XBT_line'] = xbt_line profile['profile_metadata']['XBT_line_description'] = xbt_line return profile # In some cases, the CSV file has the "pref_lab" value of the XBT_line. This complicated discovery as in our dict, # the pref_value is a value of our dict and not the key. In the next few lines, the search logic is reversed, looking # for the key according to the value for key, value in xbt_lines_info.items(): if xbt_line == value['xbt_pref_label']: xbt_line = key break # look for xbt line value from BUFR file available in ANDS XBT line vocabulary if xbt_line not in xbt_lines_info: raise InvalidFileContentError( '{xbt_code} is not a known/correct XBT line value found in the ANDS XBT line vocabulary' .format(xbt_code=xbt_line)) xbt_line_description = xbt_lines_info[xbt_line]['xbt_line_description'] xbt_line = xbt_lines_info[xbt_line]['xbt_pref_label'] # dealing with vocabulary inconsistencies between vocab/XBT_DM/XBT_NRT, forcing consistency ... somewhat if xbt_line == 'PX30': xbt_line = 'PX30-31' profile['profile_metadata']['XBT_line'] = xbt_line profile['profile_metadata']['XBT_line_description'] = xbt_line_description return profile
def _make_aggregated_timeseries(self): """For each variable, generate aggregated timeseries product and add to file_collection.""" for var in self.product_variables: # Filter input_list to the files relevant for this var input_list = self._input_list_for_variables(var) if not input_list: raise InvalidFileContentError( "No files to aggregate for {var}".format(var=var)) self.logger.info("Aggregating {var} ({n} files)".format( var=var, n=len(input_list))) product_url, errors = main_aggregator(input_list, var, self.product_site_code, **self.product_common_kwargs) self._log_excluded_files(errors) self._add_to_collection(product_url) self._cleanup_previous_version(os.path.basename(product_url))
def _make_velocity_hourly_timeseries(self): """Generate velocity hourly product for the site and add to file_collection.""" # Filter input list to just the velocity files, i.e. files with the variables # UCUR ("eastward_sea_water_velocity") or VCUR ("northward_sea_water_velocity") input_list = self._input_list_for_variables('UCUR', 'VCUR') if not input_list: raise InvalidFileContentError("No velocity files to aggregate") self.logger.info( "Creating velocity hourly products from {n} input files".format( n=len(input_list))) product_url, errors = velocity_hourly_aggregated( input_list, self.product_site_code, **self.product_common_kwargs) self._log_excluded_files(errors) self._add_to_collection(product_url) self._cleanup_previous_version(os.path.basename(product_url))
def _get_data_category(cls, input_file): """Determine the category a file belongs to (Temperature, CTD_timeseires, Velocity, etc..) """ var_names = set(cls._get_variable_names(input_file)) if var_names.intersection(cls.VELOCITY_VAR): return 'Velocity' if var_names.intersection(cls.BGC_VAR): return 'Biogeochem_timeseries' if var_names.intersection(cls.SALINITY_VAR): return 'CTD_timeseries' if var_names.intersection(cls.TEMP_VAR): return 'Temperature' raise InvalidFileContentError( "Could not determine data category for '{name}'".format(name=input_file) )
def get_data_mode(self): """ 1) Set data mode based on NetCDF product type If FV01 => DM If FV00 is ANFOG_RT => RT, then also check zip contain ancillary files, (.png or position_summary.txt file) If not present => missing RT material 2) Set format_check type specific to product type(FV00/01) and origin(ANFOG, DSTG or NRL) """ fv01 = self.file_collection.filter_by_attribute_regex( 'name', AnfogFileClassifier.DM_REGEX) adapter_dstg = '%s|%s' % (AnfogFileClassifier.ADAPTER_REGEX, AnfogFileClassifier.DSTG_REGEX) anfog_rt = self.file_collection.filter_by_attribute_regex( 'name', AnfogFileClassifier.ANFOG_RT_REGEX) if fv01: if re.match(adapter_dstg, fv01[0].name): # Adapter and DSTG file not cf and imos compliant fv01[0].check_type = PipelineFileCheckType.FORMAT_CHECK return 'DM' elif anfog_rt: # RT file not compliant anfog_rt[0].check_type = PipelineFileCheckType.FORMAT_CHECK png = self.file_collection.filter_by_attribute_regex( 'name', AnfogFileClassifier.RT_PNG_REGEX) if png: return "RT" else: raise InvalidFileContentError( "Missing ancillary files(PNGs or summary position file) in ZIP archive {name}" .format(name=os.path.basename(self.input_file))) else: raise InvalidInputFileError( "Expecting one NetCDF file in ZIP archive '{zip}'".format( zip=os.path.basename(self.input_file)))
def process_nrt(self): """Process NRT files, only allowing updates to occur if the new files for certain campaigns are more recent.""" previous_files = self.state_query.query_storage( self.dest_path_function("")) if not previous_files: return try: old_zip_file = [ x for x in previous_files if self.current_campaign in x.name and ".zip" in PurePath(x.name).suffix ][0] except IndexError: return old_metadata_file = self.get_remote_metadata_from_zip(old_zip_file) if not old_metadata_file: return new_metadata_file = self.get_file(self.file_collection, 'metadata', campaign=self.current_campaign) logger.info( NRT_TIMESTAMP_COMPARISON_MSG.format(old_zip_file.dest_path, new_metadata_file.src_path)) if not self.is_nrt_update_required(old_metadata_file.local_path): raise InvalidFileContentError( NRT_TIMESTAMP_DIFFERS_MSG.format( incoming_file=self.input_file_object.src_path, within_file=new_metadata_file.src_path, archival_file=old_zip_file.dest_path, ))