def UploadEMEPDatasets_RecentYear(dict_EMEP): """ Upload all the EMEP data from the recent year to the Geoserver (there is an error on the geoserver that the recent year is represent with a date from 1900... ) """ with tempfile.TemporaryDirectory(dir=tmp_dir) as directory: os.mkdir(directory + '/old_NC') datasets = dict_EMEP['datasets'] urlEmep = URL_BASE + dict_EMEP['services']['httpserver'] polygon_path = get_shapefile_format(directory) mask = create_mask(dict_EMEP, directory, polygon_path, urlEmep) if mask == []: log_task_file("ERROR: Mask is empty!!") return for resolution in datasets: ############################################################################################################################################### # It is not downloading the hourly and daily resolution !!!!!!!!!!!!!!!!!!! if resolution == 'hour' or resolution == 'day': continue ################################################################################################################################################## year = list(datasets[resolution].keys())[0] uploadEMEPData(year, resolution, dict_EMEP, datasets, mask, urlEmep, directory, polygon_path) save_EMEP_dict(dict_EMEP)
def UploadEMEPDatasets(dict_EMEP): """ Upload all the EMEP data to the Geoserver """ with tempfile.TemporaryDirectory(dir=tmp_dir) as directory: os.mkdir(directory + '/old_NC') datasets = dict_EMEP['datasets'] urlEmep = URL_BASE + dict_EMEP['services']['httpserver'] polygon_path = get_shapefile_format(directory) mask = create_mask(dict_EMEP, directory, polygon_path, urlEmep) if mask == []: log_task_file("ERROR: Mask is empty!!") return for resolution in datasets: ############################################################################################################################################### # It is not downloading the hourly and daily resolution !!!!!!!!!!!!!!!!!!! if resolution == 'hour' or resolution == 'day': continue ################################################################################################################################################## for year in datasets[resolution]: uploadEMEPData(year, resolution, dict_EMEP, datasets, mask, urlEmep, directory, polygon_path) save_EMEP_dict(dict_EMEP)
def Upload_EMEP_DATA(): log_task_file("Start the task Upload_EMEP_DATA") now = datetime.now() year = now.strftime("%Y") month = now.strftime("%m") if int(month) < 9: year = str(int(year) - 1) dict_EMEP = EMEP_Create_dict(year, str(int(year) - 1)) UploadEMEPDatasets(dict_EMEP)
def create_mask(dict_EMEP, directory, polygon_path, urlEmep): """ For a given shapefile, it will create a mask array that makes easier to cut the netcdfs to contain only the only useful informations """ mask = [] mask_path = directory + '/mask.npy' # if not os.path.exists(mask_path): resolution = list(dict_EMEP['datasets'].keys())[0] year = list(dict_EMEP['datasets'][resolution].keys())[0] _time = 0 filename = download_FILE( urlEmep + dict_EMEP['datasets'][resolution][year]['EMEPSite'], resolution + '-' + year, directory + "/old_NC") if filename is not None: netcdf(filename, dict_EMEP, resolution, polygon_path, None, directory) with Dataset( "%s/%s" % (directory, filename)) as nc, fiona.open(polygon_path) as fc: nc_lon = nc.variables['lon'][:] nc_lat = nc.variables['lat'][:] nc_gas = nc.variables['SURF_ugN_NOX'][_time, :, :] feature = next(iter(fc)) coords = feature['geometry']['coordinates'][0] shp_lon = np.array(coords)[:, 0] shp_lat = np.array(coords)[:, 1] mask = np.zeros_like(nc_gas, dtype=bool) start = time.time() calc_mask(mask, nc_lon, nc_lat, shp_lon, shp_lat) end = time.time() log_task_file("DONE! Mask array is compute: %s time: %f" % (mask_path, (end - start))) mask.dump(mask_path) # else: # mask = np.load(mask_path, allow_pickle=True) return mask
def uploadEMEPData(year, resolution, dict_EMEP, datasets, mask, urlEmep, directory, polygon_path): """ Upload EMEP data to the GeoServer Steps: - Download the file from EMEP site; - Cut the data to only contain data from Portugal; - Upload the data to the GeoServer """ filename = download_FILE(urlEmep + datasets[resolution][year]['EMEPSite'], "%s-%s" % (resolution, year), directory + "/old_NC") netcdf(filename, dict_EMEP, resolution, polygon_path, mask, directory) uploadGeoserver(filename, resolution, year, directory) log_task_file("Upload to Geoserver %s-%s" % (resolution, year))
def uploadGeoserver(filename, resolution, year, directory): """ Upload the file to the geoserver, by verifying the existence of the workspace resolution-year. And after the upload of the file, define the style for all the layers generate from teh file """ with open("%s/%s" % (directory, filename.replace('.nc', '.zip')), 'rb') as fileobj: workspace = "%s-%s" % (resolution, year) if checkWorkspace(workspace, gs_rest_url, user, pwd): if uploadNetCDF(workspace, fileobj, gs_rest_url, user, pwd): updateStyles(workspace, global_settings.STYLE_NAME_GENERAL, gs_rest_url, user, pwd, global_settings.PATH_EMEP_STYLE) os.remove("%s/%s" % (directory, filename.replace('.nc', '.zip'))) os.remove("%s/%s" % (directory, filename)) os.remove("%s/old_NC/%s" % (directory, filename)) return log_task_file( "It was impossible to upload the file to geoserver.\nThe path file is %s" % ("%s/%s" % (directory, filename.replace('.nc', '.zip'))))
def save_EMEP_dict(dict_EMEP): with open(global_settings.PATH_EMEP_DICT_JSON, "w") as outfile: json.dump(dict_EMEP, outfile, indent=4) #, sort_keys=True) log_task_file("Save the EMEP dictionary, path: %s" % (global_settings.PATH_EMEP_DICT_JSON))