def to_onedata(files, region):
#paths
onedata_path = config.datasets_path
for file in files:
try:
date_path = files[file]['path']
print (" moving {} file to onedata".format(file))
shutil.move(date_path, os.path.join(onedata_path, region))
metadata_gen.metadata_gen(file, files[file]['inidate'], files[file]['enddate'], files[file]['region'], files[file]['N'], files[file]['W'], files[file]['params'])
shutil.rmtree(date_path, ignore_errors=True)
except:
continue
def get_meteo(self):
prefix = ''
if 'Temp' in self.params:
prefix = 'temp_'
elif 'speed' in self.params:
prefix = 'wind_'
self.general_name = self.path + '/' + self.region + '/' + prefix + self.inidate.strftime(
'%Y-%m-%d') + "_" + self.enddate.strftime('%Y-%m-%d')
title = prefix + self.inidate.strftime(
'%Y-%m-%d') + "_" + self.enddate.strftime('%Y-%m-%d') + ".csv"
beginDate = self.inidate.strftime('%Y-%m-%d')
endDate = self.enddate.strftime('%Y-%m-%d')
self.station = self.find_station() #TODO add lat/lon
print("Selected station: " + self.station)
tt = self.datosEstacion()
if (config.onedata_mode == 1):
metadata_gen.metadata_gen(title, beginDate, endDate, self.region,
str(self.lat), str(self.lon),
self.params)
return {"output": self.general_name + ".csv"}
def check_dataset(ids, api_url, start_date, end_date, region, dataset_path):
"""Checks if the available datasets satisfy the dates and location req
Parameters
----------
ids : array
List of dataset ids
api_url : string
API to get dataset metadata
start_date : datetime
Start date time to search the dataset
end_date : datetime
End date time to search the dataset
location : string
Region to get the data from
Returns
-------
downloaded_datasets : array
List of downloaded datasets
"""
file_list = []
for i in ids:
headers = {'accept': 'application/json'}
#TODO Manage different types of identifiers (i.text.replace('record', 'api/records'),headers))
r = requests.get('https://doi.org/' + i.text, headers)
r = requests.get(r.url.replace('record', 'api/records'), headers)
beginDate = ''
endDate = ''
location = ''
try:
for o in r.json()['metadata']['keywords']:
if 'beginDate' in o:
beginDate = o.rsplit(":", 1)[1].replace("'", "")
print(beginDate)
if 'endDate' in o:
endDate = o.rsplit(":", 1)[1].replace("'", "")
print(endDate)
if 'location' in o:
location = o.rsplit(":", 1)[1].replace("'", "")
print(location)
except:
print("No Keywords or any other shit")
if len(beginDate) > 0 and datetime.strptime(
beginDate, "%Y-%m-%d") <= start_date and len(
endDate) > 0 and datetime.strptime(
endDate, "%Y-%m-%d") >= end_date and len(
location) > 0 and location == region:
for u in r.json()['files']:
if u['type'] == 'csv':
print(u['links']['self'])
link = u['links']['self']
file_name = dataset_path + u["key"]
with open(file_name, "wb") as f:
print("Downloading %s" % file_name)
response = requests.get(link, stream=True)
total_length = response.headers.get('content-length')
if total_length is None: # no content length header
f.write(response.content)
else:
dl = 0
total_length = int(total_length)
for data in response.iter_content(chunk_size=4096):
dl += len(data)
f.write(data)
done = int(50 * dl / total_length)
sys.stdout.write("\r[%s%s]" %
('=' * done, ' ' *
(50 - done)))
sys.stdout.flush()
file_list.append(file_name)
print("Download complete")
#Get Latitude, Longitude
coords = config.regions['regions'][region]
lon = 0
for l in coords:
lon = lon + l[0]
lon = lon / len(coords)
lat = 0
for l in coords:
lat = lat + l[1]
lat = lat / len(coords)
print("Lat: %s Lon: %s" % (lat, lon))
#Metadata attachment
#TODO add wind, prec
print("Attaching Metadata for %s" % file_name)
metadata_gen.metadata_gen(u["key"], beginDate,
endDate, region, str(lat),
str(lon), ["ID", "Date", "Temp"])
return file_list
def metadata_gen(title, dateIni, dateEnd, geographicDesc, westBounding,
eastBounding, northBounding, southBounding, params):
return metadata_gen.metadata_gen(title, dateIni, dateEnd, geographicDesc,
westBounding, eastBounding, northBounding,
southBounding, params)