def main(args):
start_time = time.time()
parameters = SimpleNamespace(**args['parameters'])
mdt_key = args['mdt_key']
mdt_filename = os.path.basename(mdt_key)
cos = COSBackend(
aws_access_key_id=args['cos']['aws_access_key_id'],
aws_secret_access_key=args['cos']['aws_secret_access_key'],
endpoint_url=args['cos']['private_endpoint'])
cos.download_file(parameters.BUCKET, mdt_key, mdt_filename)
tiff_file = os.path.splitext(mdt_filename)[0] + '.tif'
with rasterio.open(mdt_filename) as src:
profile = src.profile
# Cloud optimized GeoTiff parameters (No hace falta rio_cogeo)
profile.update(driver='GTiff')
profile.update(blockxsize=256)
profile.update(blockysize=256)
profile.update(tiled=True)
profile.update(compress='deflate')
profile.update(interleave='band')
with rasterio.open(tiff_file, "w", **profile) as dest:
dest.write(src.read())
cos.upload_file(filename=tiff_file,
bucket=parameters.BUCKET,
key='tiff/{}'.format(tiff_file))
end_time = time.time()
return {
'result': tiff_file,
'start_time': start_time,
'end_time': end_time
}
def main(args):
start_time = time.time()
parameters = SimpleNamespace(**args['parameters'])
cos = COSBackend(
aws_access_key_id=args['cos']['aws_access_key_id'],
aws_secret_access_key=args['cos']['aws_secret_access_key'],
endpoint_url=args['cos']['private_endpoint'])
tile = args['tile']
# Download shapefile
shapefile = cos.get_object(bucket=parameters.BUCKET, key='shapefile.zip')
with open('shape.zip', 'wb') as shapf:
for chunk in iter(partial(shapefile.read, 200 * 1024 * 1024), ''):
if not chunk:
break
shapf.write(chunk)
rasters = {}
for type in ['TEMPERATURE', 'HUMIDITY', 'WIND', 'EXTRAD', 'RADIANCE']:
key = '/'.join(['tmp', type, tile, 'merged.tif'])
rasters[type.lower()] = cos.get_object(bucket=parameters.BUCKET,
key=key)
filename = combine_calculations(tile=tile, **rasters)
result_key = '/'.join(['tmp', 'ETC', args['tile'] + '.tif'])
cos.upload_file(filename=filename,
bucket=parameters.BUCKET,
key=result_key)
end_time = time.time()
return {'result': filename, 'start_time': start_time, 'end_time': end_time}
def main(args):
start_time = time.time()
args.update(args['chunk'])
parameters = SimpleNamespace(**args['parameters'])
cos = COSBackend(
aws_access_key_id=args['cos']['aws_access_key_id'],
aws_secret_access_key=args['cos']['aws_secret_access_key'],
endpoint_url=args['cos']['private_endpoint'])
mdt_key = args['mdt_key']
mdt = cos.get_object(key=mdt_key, bucket=parameters.BUCKET)
siam_stream = cos.get_object(key='siam_out.csv', bucket=parameters.BUCKET)
out = map_interpolation(siam_stream=siam_stream,
mdt=mdt,
block_x=args['block_x'],
block_y=args['block_y'],
splits=parameters.SPLITS,
area_of_influence=parameters.AREA_OF_INFLUENCE)
result_key = '/'.join([
'tmp', 'WIND',
os.path.basename(mdt_key).rsplit('.')[0],
str(args['block_x']) + '_' + str(args['block_y']) + '.tif'
])
cos.upload_file(filename=out, bucket=parameters.BUCKET, key=result_key)
end_time = time.time()
return {
'result': result_key,
'start_time': start_time,
'end_time': end_time
}
def main(args):
start_time = time.time()
parameters = SimpleNamespace(**args['parameters'])
cos = COSBackend(aws_access_key_id=args['cos']['aws_access_key_id'],
aws_secret_access_key=args['cos']['aws_secret_access_key'],
endpoint_url=args['cos']['private_endpoint'])
keys = cos.list_keys_prefix(bucket=parameters.BUCKET, prefix='tmp/{}/{}'.format(args['type'], args['tile']))
chunk = cos.get_object(key=keys[0], bucket=parameters.BUCKET)
profile = obtain_meta(chunk, parameters.SPLITS)
tiles = ((cos.get_object(bucket=parameters.BUCKET, key=key), tuple(os.path.basename(key)[:3].split('_')))
for key in keys)
out = gather_blocks(tiles, profile)
result_key = '/'.join(['tmp', args['type'], args['tile'], 'merged.tif'])
cos.upload_file(filename=out, bucket=parameters.BUCKET, key=result_key)
end_time = time.time()
return {'result': result_key, 'start_time': start_time, 'end_time': end_time}
def main(args):
start_time = time.time()
args.update(args['chunk'])
parameters = SimpleNamespace(**args['parameters'])
cos = COSBackend(
aws_access_key_id=args['cos']['aws_access_key_id'],
aws_secret_access_key=args['cos']['aws_secret_access_key'],
endpoint_url=args['cos']['private_endpoint'])
mdt_key = args['mdt_key']
mdt = cos.get_object(key=mdt_key, bucket=parameters.BUCKET)
filename = map_interpolation(mdt, parameters.DAY_OF_YEAR, args['block_x'],
args['block_y'], parameters.SPLITS)
result_key = '/'.join([
'tmp', 'EXTRAD',
os.path.basename(mdt_key).rsplit('.')[0],
str(args['block_x']) + '_' + str(args['block_y']) + '.tif'
])
cos.upload_file(filename=filename,
bucket=parameters.BUCKET,
key=result_key)
result_key = '/'.join([
'tmp', 'RADIANCE',
os.path.basename(mdt_key).rsplit('.')[0],
str(args['block_x']) + '_' + str(args['block_y']) + '.tif'
])
cos.upload_file(filename='output',
bucket=parameters.BUCKET,
key=result_key)
end_time = time.time()
return {
'result': result_key,
'start_time': start_time,
'end_time': end_time
}
def main(args):
start_time = time.time()
parameters = SimpleNamespace(**args['parameters'])
cos = COSBackend(
aws_access_key_id=args['cos']['aws_access_key_id'],
aws_secret_access_key=args['cos']['aws_secret_access_key'],
endpoint_url=args['cos']['private_endpoint'])
url = args['url']
siam_data = requests.get(url)
with open('siam_data.csv', 'wb') as siam_file:
siam_file.write(siam_data.content)
cos.download_file(bucket=parameters.BUCKET,
key='siam_locations.csv',
filename='siam_locations.csv')
columns = {
'Estación': 'COD',
'Tmed <br> (ºC)': 'temp',
'Hrmed <br> (%)': 'hr',
'Vvmed <br> (m/seg)': 'v',
'Eti.': 'dir',
'Radmed <br> (w/m2)': 'rad',
'Dvmed <br> (º)': 'dir_deg'
}
siam_data = pd.read_csv('siam_data.csv',
encoding='iso-8859-1',
sep=';',
decimal=',',
thousands='.',
na_values='-')
siam_data = siam_data[columns.keys()].rename(columns=columns)
siam_locations = pd.read_csv('siam_locations.csv',
encoding='iso-8859-1',
sep=';',
decimal=',',
thousands='.')
siam = pd.merge(siam_locations, siam_data, on='COD')
siam['tdet'] = siam['temp'] + parameters.r * (parameters.zdet -
siam['Cota'].to_numpy())
siam = siam[['X', 'Y', 'Cota', 'temp', 'hr', 'tdet', 'v'] +
list(columns.values())]
# Guess wind direction of undefined values
siam['dir_deg'] = siam.apply(
lambda row: row['dir_deg']
if not math.isnan(row['dir_deg']) else guess_nearest(
row['X'], row['Y'], 'dir_deg', siam),
axis=1)
# Guess radiation of undefined values
siam['rad'] = siam.apply(lambda row: row['rad']
if not math.isnan(row['rad']) else guess_nearest(
row['X'], row['Y'], 'rad', siam),
axis=1)
siam.to_csv('siam_out.csv', index=False)
cos.upload_file(filename='siam_out.csv',
bucket=parameters.BUCKET,
key='siam_out.csv')
end_time = time.time()
return {
'result': 'siam_out.csv',
'start_time': start_time,
'end_time': end_time
}