def process_file(filename, harp_commands, export_dir):
# write does not work until https://github.com/tqdm/tqdm/issues/680 is solved
if not exists(export_dir / filename.name.replace("L2", "L3")):
# tqdm.write(f"Converting {filename}")
if exists(filename):
try:
output_product = harp.import_product(str(filename),
operations=harp_commands)
export_url = export_dir / f"{filename.stem.replace('L2', 'L3')}.nc"
harp.export_product(output_product,
str(export_url),
file_format="netcdf")
# tqdm.write(f"{filename} successfully converted")
except harp._harppy.NoDataError:
pass
# tqdm.write((f"Exception occured in {filename}: "
# "Product contains no variables or variables without data"))
else:
pass
# tqdm.write(f'File {filename} not found')
else:
pass
# tqdm.write("File {export_path}/{name} already exists".format(export_path=export_path,
# name=filename.split('/')[-1].replace('L2', 'L3')))
return None
def HarpImport(self, filename=None, operations=None, ingestionOptions=None, variableName=None):
if filename is None:
importDialog = HarpImportDialog(self.frame)
if importDialog.ShowModal() == wx.ID_OK:
filename = importDialog.GetFilename()
operations = importDialog.GetOperations()
ingestionOptions = importDialog.GetIngestionOptions()
variableName = importDialog.GetVariableName()
importDialog.Destroy()
if not filename:
return
if variableName:
self.shell.clearCommand()
self.shell.interp.more = False
self.shell.DocumentEnd()
command = "%s = harp.import_product(r\"%s\"" % (variableName, filename)
if operations:
command += ", operations='%s'" % (operations,)
if ingestionOptions:
command += ", options='%s'" % (ingestionOptions,)
command += ")\n"
self.shell.write(command)
self.shell.push(command)
self.shell.DocumentEnd()
else:
try:
data = harp.import_product(filename, operations, ingestionOptions)
except harp.Error as ex:
message = wx.MessageDialog(self.frame,
"Could not import product(s).\n\nHARP error message: \"%s\"" % str(ex),
"HARP error", style=wx.OK | wx.ICON_ERROR | wx.STAY_ON_TOP)
message.ShowModal()
return
def process(city, product, degrees, folder, folder_src):
""" """
no_data_files = []
## 1. get all files to be processed & create a folder to store data
all_files = retrieve_files(city, product, folder_src)
path, fail_path = create_folder_to_save(folder, city, product)
## 2. get harp operations
ops_string = get_harp_operations(city, product, degrees)
if DEBUG:
print("######## DEBUG MODE ON")
all_files = all_files[:N_debug]
for i, one_file in enumerate(all_files):
try:
print(f'{i+1}/{len(all_files)}: ', one_file)
harp_L2_L3 = harp.import_product(one_file, operations=ops_string)
export_pat = '{}{}.{}'.format(
path,
one_file.split("/")[-1].replace('L2', 'L3').split('.')[0],
'nc')
print(f"exporting {export_pat} ...\n")
harp.export_product(harp_L2_L3, export_pat, file_format='netcdf')
except:
no_data_files.append(one_file)
save_obj(no_data_files, fail_path)
print("files with no data:\n", no_data_files)
def _process_file(filename):
if not exists("{export_path}/{name}".format(
export_path=export_path,
name=filename.split('/')[-1].replace('L2', 'L3'))):
print(f"Converting {filename}")
if exists(filename):
try:
output_product = harp.import_product(
filename, operations=pre_commands)
export_url = "{export_path}/{name}".format(
export_path=export_path,
name=filename.split('/')[-1].replace('L2', 'L3'))
harp.export_product(output_product,
export_url,
file_format='netcdf',
operations=post_commands)
except harp._harppy.NoDataError:
print((
f"Exception occured in {filename}: "
"Product contains no variables or variables without data"
))
else:
print(f'File {filename} not found')
else:
print("File {export_path}/{name} already exists".format(
export_path=export_path,
name=filename.split('/')[-1].replace('L2', 'L3')))
return None
def convert_to_l3_products(filenames, pre_commands='', post_commands='', export_path='L3_data'):
"""
Process L2 products and convert to L3 using harpconvert
:param filenames: (list) List of urls of L2 products
:param pre_commands: (str) Harp command used during import of L2 products
:param post_commands: (str) Harp command used during export of L3 products
:param export_path: (str) Url of folder for converted products
"""
makedirs(export_path, exist_ok=True)
for filename in filenames:
if not exists("{export_path}/{name}".format(export_path=export_path,
name=filename.split('/')[-1].replace('L2', 'L3'))):
print(f"Converting {filename}")
if exists(filename):
try:
output_product = harp.import_product(filename, operations=pre_commands)
export_url = "{export_path}/{name}".format(export_path=export_path,
name=filename.split('/')[-1].replace('L2', 'L3'))
harp.export_product(output_product, export_url,
file_format='netcdf', operations=post_commands)
except harp._harppy.NoDataError:
printRed(
f"Exception occured in {filename}: Product contains no variables or variables without data")
else:
printRed(f'File {filename} not found')
else:
print("File {export_path}/{name} already exists".format(
export_path=export_path, name=filename.split('/')[-1].replace('L2', 'L3')))
def get_latitude(self):
"""
获取纬度
:return:
"""
product = harp.import_product(self.in_file)
data = product.latitude.data
del product
return data
def get_spectrum_wavenumber(self):
"""
获取波数
:return:
"""
product = harp.import_product(self.in_file)
data = product.wavenumber.data / 1e2
del product
return data
def get_spectrum_radiance(self):
"""
获取辐射率
:return:
"""
product = harp.import_product(self.in_file)
data = product.wavenumber_radiance.data * 1e5
del product
return data
def iasi2hdf(infile, outfile):
s = datetime.now()
print('<<< {}'.format(infile))
product = harp.import_product(infile)
harp.export_product(product,
outfile,
file_format='hdf5',
hdf5_compression=5)
print('>>> {}'.format(outfile))
print(datetime.now() - s)
def get_timestamp_utc(self):
"""
获取时间
:return:
"""
product = harp.import_product(self.in_file)
# 1970-01-01 到 2000-01-01 的总秒数为 946684800
data = product.datetime.data + 946684800
del product
return data
def mosaic_scenes(nombre_nc, d):
"""
Create the regridded products
"""
# Search for 5P products
file_names = []
products = [] # array to hold the products
print("Importing with HARP....")
i = 0
for filename in os.listdir(os.path.dirname(os.path.abspath(__file__))):
base_file, ext = os.path.splitext(filename)
if ext == ".nc" and base_file.split('_')[0] == 'S5P':
product_name = base_file + "_" + str(i)
print(product_name)
try:
product_name = harp.import_product(
base_file + ext,
operations=
"latitude > -55 [degree_north]; latitude < 80 [degree_north];tropospheric_NO2_column_number_density_validity > 75;bin_spatial(501,39.5,0.01,651,-90.5,.01)",
post_operations="bin();squash(time, (latitude,longitude))")
print("Product " + base_file + ext + " imported")
products.append(product_name)
except:
print("Didn't import!")
#
i = i + 1
try:
print("Starting to execute HARP operations....")
product_bin = harp.execute_operations(products, "", "bin()")
print("Completed the regridding.")
harp.export_product(product_bin, "out_8_" + str(d) + "_18.nc")
except:
print("HARP and/or regridding did not work!")
# Delete files from directory
for filename in os.listdir(os.path.dirname(os.path.abspath(__file__))):
base_file, ext = os.path.splitext(filename)
if ext == ".nc" and base_file.split('_')[0] == 'S5P':
os.remove(base_file + ext)
def process_file(file):
operations = " \
tropospheric_NO2_column_number_density_validity>50; \
derive(tropospheric_NO2_column_number_density [Pmolec/cm2]); \
derive(datetime_stop {time}); \
latitude > 34.4 [degree_north] ; latitude < 47.7 [degree_north] ; \
longitude > 5.8 [degree_east] ; longitude < 28.9 [degree_east] ; \
bin_spatial(1330, 34.4, 0.01, 2310, 5.8, 0.01); \
derive(latitude {latitude}); derive(longitude {longitude});\
keep(NO2_column_number_density, tropospheric_NO2_column_number_density,\
stratospheric_NO2_column_number_density, NO2_slant_column_number_density,\
tropopause_pressure,absorbing_aerosol_index,cloud_fraction, datetime_start, longitude, latitude)"
try:
harp_L2_L3 = harp.import_product(file, operations=operations)
export_folder = "{export_path}\{name}".format(
export_path='data\L3',
name=file.split('\\')[-1].replace('L2', 'L3'))
harp.export_product(harp_L2_L3, export_folder, file_format='netcdf')
#Some files may not be suitable for processing due to low quality data
#or other reasons, so we must handle the associated exception
except Exception as e:
print(e)
def harpProc(pathlist, bb, res, dateSt, dateEnd, exp_dir, prod):
import harp
import pandas as pd
pix_x = int((bb[0][1] - bb[0][0]) / res)
pix_y = int((bb[1][1] - bb[1][0]) / res)
min_x = bb[0][0]
min_y = bb[1][0]
if prod == 'L2__SO2___':
ops = f'''SO2_column_number_density_validity>50;keep(latitude_bounds,
longitude_bounds,SO2_column_number_density);bin_spatial({pix_y},{min_y},
{res},{pix_x},{min_x},{res});derive(SO2_column_number_density[Pmolec/cm2])'''
opt = 'so2_column=7km'
elif prod == 'L2__NO2___':
ops = f'''tropospheric_NO2_column_number_density_validity>75;keep(latitude_bounds,
longitude_bounds,tropospheric_NO2_column_number_density);bin_spatial({pix_y},{min_y},
{res},{pix_x},{min_x},{res});derive(tropospheric_NO2_column_number_density[Pmolec/cm2])'''
opt = ''
elif prod == 'L2__AER_AI':
ops = f'''absorbing_aerosol_index_validity>80;keep(latitude_bounds,
longitude_bounds,absorbing_aerosol_index);bin_spatial({pix_y},{min_y},
{res},{pix_x},{min_x},{res});derive(absorbing_aerosol_index)'''
opt = ''
elif prod == 'L2__CO____':
ops = f'''CO_column_number_density_validity>50;keep(latitude_bounds,
longitude_bounds,CO_column_number_density);bin_spatial({pix_y},{min_y},
{res},{pix_x},{min_x},{res});derive(CO_column_number_density)'''
opt = ''
hrp = harp.import_product(pathlist,operations = ops, options = opt, post_operations='bin()')
hrp = harp.execute_operations(hrp,"squash(time, (latitude_bounds,longitude_bounds));derive(latitude {latitude});derive(longitude {longitude});exclude(longitude_bounds,latitude_bounds,longitude_bounds_weight,latitude_bounds_weight)")
if dateSt != dateEnd:
dt_st = dateSt.strftime('%Y%m%d')
dateEnd = dateEnd.date() - pd.Timedelta(days=1)
dt_end = dateEnd.strftime('%Y%m%d')
dt_shrt = f'''{dt_st}-{dt_end}'''
elif dateSt == dateEnd:
dt_shrt = dateSt.strftime('%Y%m%d')
if prod == 'L2__SO2___':
dat = hrp.SO2_column_number_density.data
elif prod == 'L2__NO2___':
dat = hrp.tropospheric_NO2_column_number_density.data
elif prod == 'L2__AER_AI':
dat = hrp.absorbing_aerosol_index.data
elif prod == 'L2__CO____':
dat = hrp.CO_column_number_density.data
#harp.export_product(aer_exp, f'''C:/Users/Administrator/Documents/aer_export_{dt_shrt}.nc''')
import xarray as xr
import rioxarray
da = xr.DataArray(data=dat, coords={'latitude':hrp.latitude.data,'longitude':hrp.longitude.data}, dims=('time', 'latitude', 'longitude'))
da.rio.to_raster(f'''{exp_dir}/{prod}_export_{dt_shrt}.tif''')
