def process_file(filename, harp_commands, export_dir):
# write does not work until https://github.com/tqdm/tqdm/issues/680 is solved
if not exists(export_dir / filename.name.replace("L2", "L3")):
# tqdm.write(f"Converting {filename}")
if exists(filename):
try:
output_product = harp.import_product(str(filename),
operations=harp_commands)
export_url = export_dir / f"{filename.stem.replace('L2', 'L3')}.nc"
harp.export_product(output_product,
str(export_url),
file_format="netcdf")
# tqdm.write(f"{filename} successfully converted")
except harp._harppy.NoDataError:
pass
# tqdm.write((f"Exception occured in {filename}: "
# "Product contains no variables or variables without data"))
else:
pass
# tqdm.write(f'File {filename} not found')
else:
pass
# tqdm.write("File {export_path}/{name} already exists".format(export_path=export_path,
# name=filename.split('/')[-1].replace('L2', 'L3')))
return None
def _process_file(filename):
if not exists("{export_path}/{name}".format(
export_path=export_path,
name=filename.split('/')[-1].replace('L2', 'L3'))):
print(f"Converting {filename}")
if exists(filename):
try:
output_product = harp.import_product(
filename, operations=pre_commands)
export_url = "{export_path}/{name}".format(
export_path=export_path,
name=filename.split('/')[-1].replace('L2', 'L3'))
harp.export_product(output_product,
export_url,
file_format='netcdf',
operations=post_commands)
except harp._harppy.NoDataError:
print((
f"Exception occured in {filename}: "
"Product contains no variables or variables without data"
))
else:
print(f'File {filename} not found')
else:
print("File {export_path}/{name} already exists".format(
export_path=export_path,
name=filename.split('/')[-1].replace('L2', 'L3')))
return None
def process(city, product, degrees, folder, folder_src):
""" """
no_data_files = []
## 1. get all files to be processed & create a folder to store data
all_files = retrieve_files(city, product, folder_src)
path, fail_path = create_folder_to_save(folder, city, product)
## 2. get harp operations
ops_string = get_harp_operations(city, product, degrees)
if DEBUG:
print("######## DEBUG MODE ON")
all_files = all_files[:N_debug]
for i, one_file in enumerate(all_files):
try:
print(f'{i+1}/{len(all_files)}: ', one_file)
harp_L2_L3 = harp.import_product(one_file, operations=ops_string)
export_pat = '{}{}.{}'.format(
path,
one_file.split("/")[-1].replace('L2', 'L3').split('.')[0],
'nc')
print(f"exporting {export_pat} ...\n")
harp.export_product(harp_L2_L3, export_pat, file_format='netcdf')
except:
no_data_files.append(one_file)
save_obj(no_data_files, fail_path)
print("files with no data:\n", no_data_files)
def convert_to_l3_products(filenames, pre_commands='', post_commands='', export_path='L3_data'):
"""
Process L2 products and convert to L3 using harpconvert
:param filenames: (list) List of urls of L2 products
:param pre_commands: (str) Harp command used during import of L2 products
:param post_commands: (str) Harp command used during export of L3 products
:param export_path: (str) Url of folder for converted products
"""
makedirs(export_path, exist_ok=True)
for filename in filenames:
if not exists("{export_path}/{name}".format(export_path=export_path,
name=filename.split('/')[-1].replace('L2', 'L3'))):
print(f"Converting {filename}")
if exists(filename):
try:
output_product = harp.import_product(filename, operations=pre_commands)
export_url = "{export_path}/{name}".format(export_path=export_path,
name=filename.split('/')[-1].replace('L2', 'L3'))
harp.export_product(output_product, export_url,
file_format='netcdf', operations=post_commands)
except harp._harppy.NoDataError:
printRed(
f"Exception occured in {filename}: Product contains no variables or variables without data")
else:
printRed(f'File {filename} not found')
else:
print("File {export_path}/{name} already exists".format(
export_path=export_path, name=filename.split('/')[-1].replace('L2', 'L3')))
def iasi2hdf(infile, outfile):
s = datetime.now()
print('<<< {}'.format(infile))
product = harp.import_product(infile)
harp.export_product(product,
outfile,
file_format='hdf5',
hdf5_compression=5)
print('>>> {}'.format(outfile))
print(datetime.now() - s)
def mosaic_scenes(nombre_nc, d):
"""
Create the regridded products
"""
# Search for 5P products
file_names = []
products = [] # array to hold the products
print("Importing with HARP....")
i = 0
for filename in os.listdir(os.path.dirname(os.path.abspath(__file__))):
base_file, ext = os.path.splitext(filename)
if ext == ".nc" and base_file.split('_')[0] == 'S5P':
product_name = base_file + "_" + str(i)
print(product_name)
try:
product_name = harp.import_product(
base_file + ext,
operations=
"latitude > -55 [degree_north]; latitude < 80 [degree_north];tropospheric_NO2_column_number_density_validity > 75;bin_spatial(501,39.5,0.01,651,-90.5,.01)",
post_operations="bin();squash(time, (latitude,longitude))")
print("Product " + base_file + ext + " imported")
products.append(product_name)
except:
print("Didn't import!")
#
i = i + 1
try:
print("Starting to execute HARP operations....")
product_bin = harp.execute_operations(products, "", "bin()")
print("Completed the regridding.")
harp.export_product(product_bin, "out_8_" + str(d) + "_18.nc")
except:
print("HARP and/or regridding did not work!")
# Delete files from directory
for filename in os.listdir(os.path.dirname(os.path.abspath(__file__))):
base_file, ext = os.path.splitext(filename)
if ext == ".nc" and base_file.split('_')[0] == 'S5P':
os.remove(base_file + ext)
def process_file(file):
operations = " \
tropospheric_NO2_column_number_density_validity>50; \
derive(tropospheric_NO2_column_number_density [Pmolec/cm2]); \
derive(datetime_stop {time}); \
latitude > 34.4 [degree_north] ; latitude < 47.7 [degree_north] ; \
longitude > 5.8 [degree_east] ; longitude < 28.9 [degree_east] ; \
bin_spatial(1330, 34.4, 0.01, 2310, 5.8, 0.01); \
derive(latitude {latitude}); derive(longitude {longitude});\
keep(NO2_column_number_density, tropospheric_NO2_column_number_density,\
stratospheric_NO2_column_number_density, NO2_slant_column_number_density,\
tropopause_pressure,absorbing_aerosol_index,cloud_fraction, datetime_start, longitude, latitude)"
try:
harp_L2_L3 = harp.import_product(file, operations=operations)
export_folder = "{export_path}\{name}".format(
export_path='data\L3',
name=file.split('\\')[-1].replace('L2', 'L3'))
harp.export_product(harp_L2_L3, export_folder, file_format='netcdf')
#Some files may not be suitable for processing due to low quality data
#or other reasons, so we must handle the associated exception
except Exception as e:
print(e)
