def process_file(filename, harp_commands, export_dir): # write does not work until https://github.com/tqdm/tqdm/issues/680 is solved if not exists(export_dir / filename.name.replace("L2", "L3")): # tqdm.write(f"Converting {filename}") if exists(filename): try: output_product = harp.import_product(str(filename), operations=harp_commands) export_url = export_dir / f"{filename.stem.replace('L2', 'L3')}.nc" harp.export_product(output_product, str(export_url), file_format="netcdf") # tqdm.write(f"{filename} successfully converted") except harp._harppy.NoDataError: pass # tqdm.write((f"Exception occured in {filename}: " # "Product contains no variables or variables without data")) else: pass # tqdm.write(f'File {filename} not found') else: pass # tqdm.write("File {export_path}/{name} already exists".format(export_path=export_path, # name=filename.split('/')[-1].replace('L2', 'L3'))) return None
def HarpImport(self, filename=None, operations=None, ingestionOptions=None, variableName=None): if filename is None: importDialog = HarpImportDialog(self.frame) if importDialog.ShowModal() == wx.ID_OK: filename = importDialog.GetFilename() operations = importDialog.GetOperations() ingestionOptions = importDialog.GetIngestionOptions() variableName = importDialog.GetVariableName() importDialog.Destroy() if not filename: return if variableName: self.shell.clearCommand() self.shell.interp.more = False self.shell.DocumentEnd() command = "%s = harp.import_product(r\"%s\"" % (variableName, filename) if operations: command += ", operations='%s'" % (operations,) if ingestionOptions: command += ", options='%s'" % (ingestionOptions,) command += ")\n" self.shell.write(command) self.shell.push(command) self.shell.DocumentEnd() else: try: data = harp.import_product(filename, operations, ingestionOptions) except harp.Error as ex: message = wx.MessageDialog(self.frame, "Could not import product(s).\n\nHARP error message: \"%s\"" % str(ex), "HARP error", style=wx.OK | wx.ICON_ERROR | wx.STAY_ON_TOP) message.ShowModal() return
def process(city, product, degrees, folder, folder_src): """ """ no_data_files = [] ## 1. get all files to be processed & create a folder to store data all_files = retrieve_files(city, product, folder_src) path, fail_path = create_folder_to_save(folder, city, product) ## 2. get harp operations ops_string = get_harp_operations(city, product, degrees) if DEBUG: print("######## DEBUG MODE ON") all_files = all_files[:N_debug] for i, one_file in enumerate(all_files): try: print(f'{i+1}/{len(all_files)}: ', one_file) harp_L2_L3 = harp.import_product(one_file, operations=ops_string) export_pat = '{}{}.{}'.format( path, one_file.split("/")[-1].replace('L2', 'L3').split('.')[0], 'nc') print(f"exporting {export_pat} ...\n") harp.export_product(harp_L2_L3, export_pat, file_format='netcdf') except: no_data_files.append(one_file) save_obj(no_data_files, fail_path) print("files with no data:\n", no_data_files)
def _process_file(filename): if not exists("{export_path}/{name}".format( export_path=export_path, name=filename.split('/')[-1].replace('L2', 'L3'))): print(f"Converting {filename}") if exists(filename): try: output_product = harp.import_product( filename, operations=pre_commands) export_url = "{export_path}/{name}".format( export_path=export_path, name=filename.split('/')[-1].replace('L2', 'L3')) harp.export_product(output_product, export_url, file_format='netcdf', operations=post_commands) except harp._harppy.NoDataError: print(( f"Exception occured in {filename}: " "Product contains no variables or variables without data" )) else: print(f'File {filename} not found') else: print("File {export_path}/{name} already exists".format( export_path=export_path, name=filename.split('/')[-1].replace('L2', 'L3'))) return None
def convert_to_l3_products(filenames, pre_commands='', post_commands='', export_path='L3_data'): """ Process L2 products and convert to L3 using harpconvert :param filenames: (list) List of urls of L2 products :param pre_commands: (str) Harp command used during import of L2 products :param post_commands: (str) Harp command used during export of L3 products :param export_path: (str) Url of folder for converted products """ makedirs(export_path, exist_ok=True) for filename in filenames: if not exists("{export_path}/{name}".format(export_path=export_path, name=filename.split('/')[-1].replace('L2', 'L3'))): print(f"Converting {filename}") if exists(filename): try: output_product = harp.import_product(filename, operations=pre_commands) export_url = "{export_path}/{name}".format(export_path=export_path, name=filename.split('/')[-1].replace('L2', 'L3')) harp.export_product(output_product, export_url, file_format='netcdf', operations=post_commands) except harp._harppy.NoDataError: printRed( f"Exception occured in {filename}: Product contains no variables or variables without data") else: printRed(f'File {filename} not found') else: print("File {export_path}/{name} already exists".format( export_path=export_path, name=filename.split('/')[-1].replace('L2', 'L3')))
def get_latitude(self): """ 获取纬度 :return: """ product = harp.import_product(self.in_file) data = product.latitude.data del product return data
def get_spectrum_wavenumber(self): """ 获取波数 :return: """ product = harp.import_product(self.in_file) data = product.wavenumber.data / 1e2 del product return data
def get_spectrum_radiance(self): """ 获取辐射率 :return: """ product = harp.import_product(self.in_file) data = product.wavenumber_radiance.data * 1e5 del product return data
def iasi2hdf(infile, outfile): s = datetime.now() print('<<< {}'.format(infile)) product = harp.import_product(infile) harp.export_product(product, outfile, file_format='hdf5', hdf5_compression=5) print('>>> {}'.format(outfile)) print(datetime.now() - s)
def get_timestamp_utc(self): """ 获取时间 :return: """ product = harp.import_product(self.in_file) # 1970-01-01 到 2000-01-01 的总秒数为 946684800 data = product.datetime.data + 946684800 del product return data
def mosaic_scenes(nombre_nc, d): """ Create the regridded products """ # Search for 5P products file_names = [] products = [] # array to hold the products print("Importing with HARP....") i = 0 for filename in os.listdir(os.path.dirname(os.path.abspath(__file__))): base_file, ext = os.path.splitext(filename) if ext == ".nc" and base_file.split('_')[0] == 'S5P': product_name = base_file + "_" + str(i) print(product_name) try: product_name = harp.import_product( base_file + ext, operations= "latitude > -55 [degree_north]; latitude < 80 [degree_north];tropospheric_NO2_column_number_density_validity > 75;bin_spatial(501,39.5,0.01,651,-90.5,.01)", post_operations="bin();squash(time, (latitude,longitude))") print("Product " + base_file + ext + " imported") products.append(product_name) except: print("Didn't import!") # i = i + 1 try: print("Starting to execute HARP operations....") product_bin = harp.execute_operations(products, "", "bin()") print("Completed the regridding.") harp.export_product(product_bin, "out_8_" + str(d) + "_18.nc") except: print("HARP and/or regridding did not work!") # Delete files from directory for filename in os.listdir(os.path.dirname(os.path.abspath(__file__))): base_file, ext = os.path.splitext(filename) if ext == ".nc" and base_file.split('_')[0] == 'S5P': os.remove(base_file + ext)
def process_file(file): operations = " \ tropospheric_NO2_column_number_density_validity>50; \ derive(tropospheric_NO2_column_number_density [Pmolec/cm2]); \ derive(datetime_stop {time}); \ latitude > 34.4 [degree_north] ; latitude < 47.7 [degree_north] ; \ longitude > 5.8 [degree_east] ; longitude < 28.9 [degree_east] ; \ bin_spatial(1330, 34.4, 0.01, 2310, 5.8, 0.01); \ derive(latitude {latitude}); derive(longitude {longitude});\ keep(NO2_column_number_density, tropospheric_NO2_column_number_density,\ stratospheric_NO2_column_number_density, NO2_slant_column_number_density,\ tropopause_pressure,absorbing_aerosol_index,cloud_fraction, datetime_start, longitude, latitude)" try: harp_L2_L3 = harp.import_product(file, operations=operations) export_folder = "{export_path}\{name}".format( export_path='data\L3', name=file.split('\\')[-1].replace('L2', 'L3')) harp.export_product(harp_L2_L3, export_folder, file_format='netcdf') #Some files may not be suitable for processing due to low quality data #or other reasons, so we must handle the associated exception except Exception as e: print(e)
def harpProc(pathlist, bb, res, dateSt, dateEnd, exp_dir, prod): import harp import pandas as pd pix_x = int((bb[0][1] - bb[0][0]) / res) pix_y = int((bb[1][1] - bb[1][0]) / res) min_x = bb[0][0] min_y = bb[1][0] if prod == 'L2__SO2___': ops = f'''SO2_column_number_density_validity>50;keep(latitude_bounds, longitude_bounds,SO2_column_number_density);bin_spatial({pix_y},{min_y}, {res},{pix_x},{min_x},{res});derive(SO2_column_number_density[Pmolec/cm2])''' opt = 'so2_column=7km' elif prod == 'L2__NO2___': ops = f'''tropospheric_NO2_column_number_density_validity>75;keep(latitude_bounds, longitude_bounds,tropospheric_NO2_column_number_density);bin_spatial({pix_y},{min_y}, {res},{pix_x},{min_x},{res});derive(tropospheric_NO2_column_number_density[Pmolec/cm2])''' opt = '' elif prod == 'L2__AER_AI': ops = f'''absorbing_aerosol_index_validity>80;keep(latitude_bounds, longitude_bounds,absorbing_aerosol_index);bin_spatial({pix_y},{min_y}, {res},{pix_x},{min_x},{res});derive(absorbing_aerosol_index)''' opt = '' elif prod == 'L2__CO____': ops = f'''CO_column_number_density_validity>50;keep(latitude_bounds, longitude_bounds,CO_column_number_density);bin_spatial({pix_y},{min_y}, {res},{pix_x},{min_x},{res});derive(CO_column_number_density)''' opt = '' hrp = harp.import_product(pathlist,operations = ops, options = opt, post_operations='bin()') hrp = harp.execute_operations(hrp,"squash(time, (latitude_bounds,longitude_bounds));derive(latitude {latitude});derive(longitude {longitude});exclude(longitude_bounds,latitude_bounds,longitude_bounds_weight,latitude_bounds_weight)") if dateSt != dateEnd: dt_st = dateSt.strftime('%Y%m%d') dateEnd = dateEnd.date() - pd.Timedelta(days=1) dt_end = dateEnd.strftime('%Y%m%d') dt_shrt = f'''{dt_st}-{dt_end}''' elif dateSt == dateEnd: dt_shrt = dateSt.strftime('%Y%m%d') if prod == 'L2__SO2___': dat = hrp.SO2_column_number_density.data elif prod == 'L2__NO2___': dat = hrp.tropospheric_NO2_column_number_density.data elif prod == 'L2__AER_AI': dat = hrp.absorbing_aerosol_index.data elif prod == 'L2__CO____': dat = hrp.CO_column_number_density.data #harp.export_product(aer_exp, f'''C:/Users/Administrator/Documents/aer_export_{dt_shrt}.nc''') import xarray as xr import rioxarray da = xr.DataArray(data=dat, coords={'latitude':hrp.latitude.data,'longitude':hrp.longitude.data}, dims=('time', 'latitude', 'longitude')) da.rio.to_raster(f'''{exp_dir}/{prod}_export_{dt_shrt}.tif''')