def generate_parameters_cum(): starting_files.sort() # Convert from string to in (for comparison) dekad_start = int(start_season) dekad_end = int(end_season) # Loop over all input files for file_t0 in starting_files: # Get current date (in format '19980901') date_t0 = functions.get_date_from_path_full(file_t0) # Extract from date-string the dekad/year as integer dekad_t0 = int(date_t0[4:]) year_t0 = int(date_t0[0:4]) in_season = False # Check if season goes across two years -> define year1/2 if dekad_start < dekad_end: if dekad_t0 >= dekad_start and dekad_t0 <= dekad_end: year_sos = year_t0 in_season = True else: if dekad_t0 >= dekad_start: year_sos = year_t0 in_season = True if dekad_t0 <= dekad_end: year_sos = year_t0 - 1 in_season = True # Detect the end of the season and trigger processing if in_season: # Define output filename output_file = es2_data_dir + subdir_ident_cum + date_t0 + prod_ident_cum # Get list of dates from start of season to end of season list_dates = proc_functions.get_list_dates_for_dataset( prod, starting_sprod, version, start_date=str(year_sos) + start_season, end_date=date_t0) input_files = [] missing_file = False for ldate in list_dates: # Append the file to list if it exists ... if os.path.isfile(input_dir + ldate + in_prod_ident): input_files.append(input_dir + ldate + in_prod_ident) # ... otherwise raise a warning and break else: logger.warning( 'Missing file for date {0}. Season not computed.'. format(ldate)) missing_file = True break if not missing_file: yield (input_files, output_file)
def get_filenames_range(self): all_files = glob.glob(os.path.join(self.fullpath, "*")) filenames = [] for file in all_files: str_date = functions.get_date_from_path_full(file) my_date=datetime.date(int(str_date[0:4]),int(str_date[4:6]), int(str_date[6:8])) if my_date >= self.from_date and my_date <= self.to_date: filenames.append((file)) return filenames
def compute_monthly_prod_from_10d(input_file, output_file): # ES2- 235 Do not show temporary products like composite not complete (ex monthly composite available mid month...) input_file_date = functions.get_date_from_path_full(input_file[0]) if len(input_file) == 3: if not functions.is_date_current_month(input_file_date): output_file = functions.list_to_element(output_file) functions.check_output_dir(os.path.dirname(output_file)) args = { "input_file": input_file, "output_file": output_file, "output_format": 'GTIFF', "options": "compress = lzw" } raster_image_math.do_avg_image(**args)
def generate_parameters_ndvi_linearx1(): # Look for all input files in input_dir, and sort them input_files = glob.glob(starting_files) input_files.sort() for file_t0 in input_files: # Get current date date_t0 = functions.get_date_from_path_full(file_t0) output_file = es_constants.processing_dir+subdir_linearx1+str(date_t0)+prod_ident_linearx1 # Get files at t-1 and t+1 adjac_files = functions.files_temp_ajacent(file_t0) if len(adjac_files) == 2: # Prepare and return arguments three_files_in_a_row = [adjac_files[0], file_t0, adjac_files[1]] yield (three_files_in_a_row, output_file)
def generate_parameters_ndvi_linearx1(): # Look for all input files in input_dir, and sort them input_files = glob.glob(starting_files) input_files.sort() for file_t0 in input_files: # Get current date date_t0 = functions.get_date_from_path_full(file_t0) output_file = es_constants.processing_dir + subdir_linearx1 + str( date_t0) + prod_ident_linearx1 # Get files at t-1 and t+1 adjac_files = functions.files_temp_ajacent(file_t0) if len(adjac_files) == 2: # Prepare and return arguments three_files_in_a_row = [adjac_files[0], file_t0, adjac_files[1]] yield (three_files_in_a_row, output_file)
def std_precip_1moncum(input_file, output_file): #ES2- 235 Do not show temporary products like composite not complete (ex monthly composite available mid month...) # ex: monthly RFE in the middle of the month should not be available because incomplete and lead to wrong analysis... # Check current month ---> yes ---> skip # ----> NO ---> Check No of days (10% tolerance) # acceptable ----> # Yes ---> proceed # No ----> Skip input_file_date = functions.get_date_from_path_full(input_file[0]) if len(input_file) == 3: if not functions.is_date_current_month(input_file_date): output_file = functions.list_to_element(output_file) functions.check_output_dir(os.path.dirname(output_file)) args = { "input_file": input_file, "output_file": output_file, "output_format": 'GTIFF', "options": "compress=lzw" } raster_image_math.do_cumulate(**args)
def rain_onset(input_files, output_file): output_file = functions.list_to_element(output_file) functions.check_output_dir(os.path.dirname(output_file)) # Need to define the current_dekad number, wrt begin of season current_date = functions.get_date_from_path_full(output_file) current_dekad = current_date[4:] dekad_number = functions.dekad_nbr_in_season(current_dekad, start_season) # Call the function args = { "input_file": input_files, "output_file": output_file, 'input_nodata': None, 'output_nodata': None, 'output_type': 'Int16', "output_format": 'GTIFF', "options": "compress = lzw", 'current_dekad': dekad_number } raster_image_math.do_rain_onset(**args)
def syncGeoserver(): # # Copy some 'relevant' datasets to GeoServer # Selection of datasets is done on the basis of the product.geoserver table # # Get list of all 'relevant' subproducts (see 2. above) list_active_geoserver = esTools.get_activated_geoserver() # Loop over existing sub_products for geoserver_sprod in list_active_geoserver: # Extract local variable: my_prod = geoserver_sprod.productcode my_subprod = geoserver_sprod.subproductcode my_version = geoserver_sprod.version start_date = geoserver_sprod.startdate end_date = geoserver_sprod.enddate logger.info("Working on Product/Subproduct/Version: {0}/{1}/{2}".format(my_prod, my_subprod, my_version)) # Manage dates from bigint to datetime if functions.is_date_yyyymmdd(str(start_date), silent=True): date_start = datetime.datetime.strptime(str(start_date), '%Y%m%d').date() else: date_start = None if functions.is_date_yyyymmdd(str(end_date), silent=True): date_end = datetime.datetime.strptime(str(end_date), '%Y%m%d').date() else: date_end = None # Get additional products info product_info = querydb.get_product_out_info(productcode=my_prod, subproductcode=my_subprod, version=my_version) # my_mapset = subprod.mapsetcode my_type = product_info[0].product_type my_category = product_info[0].category_id # Create a Product object (to get mapsets) my_product = products.Product(my_prod, version=my_version) my_mapsets = my_product.mapsets if len(my_mapsets) > 1: logger.info('More than 1 mapset exists. Take the first') if len(my_mapsets) == 0: logger.warning('No any mapset exists. Skip.') continue my_mapset = my_mapsets[0] # Create a Dataset object (to get file list) # If data_start is not set (e.g. for 10davg prod) create w/o dates if date_start: my_dataset = datasets.Dataset(my_prod, my_subprod, my_mapset, version=my_version, from_date=date_start, to_date=date_end) if my_dataset._frequency.dateformat == 'MMDD': logger.warning('Product of type MMDD: date specification not supported. Skip.') continue file_list = my_dataset.get_filenames_range() else: my_dataset = datasets.Dataset(my_prod, my_subprod, my_mapset, version=my_version) file_list = my_dataset.get_filenames() # Check that there is at least 1 file if len(file_list) > 0: # Check the Workspace exists, or create it my_workspace = esTools.setWorkspaceName(my_category, my_prod, my_subprod, my_version, my_mapset, nameType=geoserverREST.geoserverWorkspaceName) if not geoserverREST.isWorkspace(my_workspace): geoserverREST.createWorkspace(my_workspace) # Loop over files and upload for my_file in file_list: my_date = functions.get_date_from_path_full(my_file) # if subprod in list_active_subprods: logger.debug("Working on Product/Subproduct/Version/Mapset/Date: {0}/{1}/{2}/{3}/{4}".format( my_prod, my_subprod, my_version, my_mapset, my_date)) # Upload the file and register esTools.uploadAndRegisterRaster(my_category, my_prod, my_subprod, my_version, my_mapset, my_date, my_type, local_data_dir)
def generate_parameters_onset(): starting_files.sort() for file_t0 in starting_files: # Get current date date_t0 = functions.get_date_from_path_full(file_t0) # Check if we are in the seasonal range [start < current <= end] dekad_t0 = int(date_t0[4:]) dekad_start = int(start_season) dekad_second = int(second_dekad) dekad_end = int(end_season) # Initialize processing to 0 do_proc = 0 in_season = False # Check we are within the season -> do_proc if dekad_start < dekad_end: if dekad_t0 > dekad_start and dekad_t0 <= dekad_end: in_season = True else: if dekad_t0 > dekad_start or dekad_t0 <= dekad_end: in_season = True if in_season and (dekad_t0 == dekad_second): do_proc = 1 if in_season and (dekad_t0 != dekad_second): do_proc = 2 if do_proc: output_file = es2_data_dir + subdir_onset + str( date_t0) + prod_ident_onset # Get files at t-1 and t-2 (if they exist) previous_files = functions.previous_files(file_t0) # Check if at least one previous file has been identified if do_proc == 1: # Check at least 1 previous file exist if len(previous_files) < 1: print('Error Case 1: no any previous file') else: # Pass two arguments (t0 and t0-1) input_files = [file_t0, previous_files[0]] yield (input_files, output_file) elif do_proc == 2: error = False # Check 2 previous files exist if len(previous_files) < 2: print('Error Case 2: a previous file is missing') error = True # Look for previous output previous_outputs = functions.previous_files(output_file) if len(previous_outputs) < 1: print('Error Case 2: the previous output is missing') error = True # Pass four arguments (t0, t0-1, t0-2 and output-1) if not error: previous_output = previous_outputs[0] if os.path.isfile(previous_output): input_files = [ file_t0, previous_files[0], previous_files[1], previous_output ] yield (input_files, output_file)
def std_precip_1d_gRf(input_file, output_file): functions.check_output_dir(os.path.dirname(output_file)) my_date = functions.get_date_from_path_full(input_file) layer = my_date + in_prod_ident_noext mapset_obj = MapSet() mapset_obj.assigndb(mapset) # Prepare tmpfile output_file_tmp = tmpdir + os.path.basename(output_file) # Extract info from mapset size_x = mapset_obj.size_x size_y = mapset_obj.size_y geo_transform = mapset_obj.geo_transform pixel_shift_x = geo_transform[1] pixel_shift_y = geo_transform[5] ulx = geo_transform[0] uly = geo_transform[3] lrx = ulx + pixel_shift_x * size_x lry = uly + pixel_shift_y * size_y txe = str(ulx) + ' ' + str(lrx) tye = str(uly) + ' ' + str(lry) te = str(ulx) + ' ' + str(lry) + ' ' + str(lrx) + ' ' + str(uly) tr = str(pixel_shift_x) + ' ' + str(pixel_shift_x) outsize = str(size_x) + ' ' + str(size_y) # Interpolate at the original resolution (no outsize) command = 'gdal_grid '\ + ' -ot Float32 -of GTiff -co "compress=LZW" ' \ + ' -txe ' + txe\ + ' -tye ' + tye\ + ' -zfield precipitat '\ + ' -l '+layer \ + ' -a invdist:power=2.0:smooting=1:radius1=0.0:radius2=0.0:angle=0.0:max_points=0:min_points=0:nodata:0.0 '\ + input_file +' '+output_file_tmp try: os.system(command) except: pass # Interpolate at the original resolution command = 'gdalwarp '\ + '-t_srs "EPSG:4326" '\ + ' -of GTiff -co "compress=LZW" ' \ + ' -te ' + te\ + ' -tr ' + tr+ ' '\ + output_file_tmp +' '+output_file # + ' -ts ' + outsize \ try: print command os.system(command) except: pass try: shutil.rmtree(tmpdir) except: print('Error in removing temporary directory. Continue') raise NameError('Error in removing tmpdir')
def test_get_date_from_path_full(self): my_date = functions.get_date_from_path_full(self.fullpath) self.assertEqual(my_date, self.str_date)
from lib.python.image_proc import raster_image_math from glob import * from lib.python import es_logging as log from lib.python import functions import os logger = log.my_logger(__name__) if __name__ == '__main__': files_dir = '/data/processing/modis-sst/v2013.1/MODIS-Africa-4km/derived/monavg/' files = glob(files_dir + '*.tif') input_file_dir = '/data/processing/modis-firms/v6.0/SPOTV-Africa-1km/derived/10dcount/' for myfile in sorted(files): print('Working on file: ' + myfile) try: date = functions.get_date_from_path_full(myfile) # input_file = glob(input_file_dir+date+'*.tif') input_file = [ "/data/processing/modis-sst/v2013.1/MODIS-Africa-4km/derived/monavg/20181101_modis-sst_monavg_MODIS-Africa-4km_v2013.1.tif" ] if not os.path.isfile(input_file[0]): print('No input file found for: ' + myfile) else: raster_image_math.assign_metadata_processing( input_file, myfile) except: print('Error in processing file: ' + myfile)