def UploadEMEPDatasets_RecentYear(dict_EMEP):
"""
Upload all the EMEP data from the recent year to the Geoserver (there is an error on the geoserver that the recent year is represent with a date from 1900... )
"""
with tempfile.TemporaryDirectory(dir=tmp_dir) as directory:
os.mkdir(directory + '/old_NC')
datasets = dict_EMEP['datasets']
urlEmep = URL_BASE + dict_EMEP['services']['httpserver']
polygon_path = get_shapefile_format(directory)
mask = create_mask(dict_EMEP, directory, polygon_path, urlEmep)
if mask == []:
log_task_file("ERROR: Mask is empty!!")
return
for resolution in datasets:
###############################################################################################################################################
# It is not downloading the hourly and daily resolution !!!!!!!!!!!!!!!!!!!
if resolution == 'hour' or resolution == 'day':
continue
##################################################################################################################################################
year = list(datasets[resolution].keys())[0]
uploadEMEPData(year, resolution, dict_EMEP, datasets, mask,
urlEmep, directory, polygon_path)
save_EMEP_dict(dict_EMEP)
def UploadEMEPDatasets(dict_EMEP):
"""
Upload all the EMEP data to the Geoserver
"""
with tempfile.TemporaryDirectory(dir=tmp_dir) as directory:
os.mkdir(directory + '/old_NC')
datasets = dict_EMEP['datasets']
urlEmep = URL_BASE + dict_EMEP['services']['httpserver']
polygon_path = get_shapefile_format(directory)
mask = create_mask(dict_EMEP, directory, polygon_path, urlEmep)
if mask == []:
log_task_file("ERROR: Mask is empty!!")
return
for resolution in datasets:
###############################################################################################################################################
# It is not downloading the hourly and daily resolution !!!!!!!!!!!!!!!!!!!
if resolution == 'hour' or resolution == 'day':
continue
##################################################################################################################################################
for year in datasets[resolution]:
uploadEMEPData(year, resolution, dict_EMEP, datasets, mask,
urlEmep, directory, polygon_path)
save_EMEP_dict(dict_EMEP)
def Upload_EMEP_DATA():
log_task_file("Start the task Upload_EMEP_DATA")
now = datetime.now()
year = now.strftime("%Y")
month = now.strftime("%m")
if int(month) < 9:
year = str(int(year) - 1)
dict_EMEP = EMEP_Create_dict(year, str(int(year) - 1))
UploadEMEPDatasets(dict_EMEP)
def create_mask(dict_EMEP, directory, polygon_path, urlEmep):
"""
For a given shapefile, it will create a mask array that makes easier to cut the netcdfs to contain only the only useful informations
"""
mask = []
mask_path = directory + '/mask.npy'
# if not os.path.exists(mask_path):
resolution = list(dict_EMEP['datasets'].keys())[0]
year = list(dict_EMEP['datasets'][resolution].keys())[0]
_time = 0
filename = download_FILE(
urlEmep + dict_EMEP['datasets'][resolution][year]['EMEPSite'],
resolution + '-' + year, directory + "/old_NC")
if filename is not None:
netcdf(filename, dict_EMEP, resolution, polygon_path, None, directory)
with Dataset(
"%s/%s" %
(directory, filename)) as nc, fiona.open(polygon_path) as fc:
nc_lon = nc.variables['lon'][:]
nc_lat = nc.variables['lat'][:]
nc_gas = nc.variables['SURF_ugN_NOX'][_time, :, :]
feature = next(iter(fc))
coords = feature['geometry']['coordinates'][0]
shp_lon = np.array(coords)[:, 0]
shp_lat = np.array(coords)[:, 1]
mask = np.zeros_like(nc_gas, dtype=bool)
start = time.time()
calc_mask(mask, nc_lon, nc_lat, shp_lon, shp_lat)
end = time.time()
log_task_file("DONE! Mask array is compute: %s time: %f" %
(mask_path, (end - start)))
mask.dump(mask_path)
# else:
# mask = np.load(mask_path, allow_pickle=True)
return mask
def uploadEMEPData(year, resolution, dict_EMEP, datasets, mask, urlEmep,
directory, polygon_path):
"""
Upload EMEP data to the GeoServer
Steps:
- Download the file from EMEP site;
- Cut the data to only contain data from Portugal;
- Upload the data to the GeoServer
"""
filename = download_FILE(urlEmep + datasets[resolution][year]['EMEPSite'],
"%s-%s" % (resolution, year),
directory + "/old_NC")
netcdf(filename, dict_EMEP, resolution, polygon_path, mask, directory)
uploadGeoserver(filename, resolution, year, directory)
log_task_file("Upload to Geoserver %s-%s" % (resolution, year))
def uploadGeoserver(filename, resolution, year, directory):
"""
Upload the file to the geoserver, by verifying the existence of the workspace resolution-year.
And after the upload of the file, define the style for all the layers generate from teh file
"""
with open("%s/%s" % (directory, filename.replace('.nc', '.zip')),
'rb') as fileobj:
workspace = "%s-%s" % (resolution, year)
if checkWorkspace(workspace, gs_rest_url, user, pwd):
if uploadNetCDF(workspace, fileobj, gs_rest_url, user, pwd):
updateStyles(workspace, global_settings.STYLE_NAME_GENERAL,
gs_rest_url, user, pwd,
global_settings.PATH_EMEP_STYLE)
os.remove("%s/%s" %
(directory, filename.replace('.nc', '.zip')))
os.remove("%s/%s" % (directory, filename))
os.remove("%s/old_NC/%s" % (directory, filename))
return
log_task_file(
"It was impossible to upload the file to geoserver.\nThe path file is %s"
% ("%s/%s" % (directory, filename.replace('.nc', '.zip'))))
def save_EMEP_dict(dict_EMEP):
with open(global_settings.PATH_EMEP_DICT_JSON, "w") as outfile:
json.dump(dict_EMEP, outfile, indent=4) #, sort_keys=True)
log_task_file("Save the EMEP dictionary, path: %s" %
(global_settings.PATH_EMEP_DICT_JSON))