def get_static_data(data_access_component: DataAccessComponent, roi: str,
roi_grid: str, start_time: str, stop_time: str,
emulation_directory: str, dem_directory: str):
create_dir(emulation_directory)
create_dir(dem_directory)
rg = roi_grid
if roi_grid is 'none':
rg = None
print('Retrieving emulators ...')
emu_urls = data_access_component.get_data_urls(
roi, start_time, stop_time, 'ISO_MSI_A_EMU,ISO_MSI_B_EMU', rg)
set_permissions(emu_urls)
create_sym_links(emu_urls, emulation_directory)
print('Retrieving DEM ...')
dem_urls = data_access_component.get_data_urls(roi, start_time, stop_time,
'Aster_DEM', rg)
set_permissions(dem_urls)
create_sym_links(dem_urls, dem_directory)
print('Done retrieving static data')
def orchestrate(cls, path_to_config: str) -> None:
assert path_to_config is not None, 'ERROR: Configuration file needs to be provided'
with open(path_to_config, 'r') as config_file:
yaml_config = yaml.safe_load(config_file)
config = AttributeDict(**yaml_config)
data_access_component = DataAccessComponent()
# sar_pre_processor = SARPreProcessor()
# sar_pre_processor.pre_process_step1()
# sar_pre_processor.pre_process_step2()
# sar_pre_processor.pre_process_step3()
prior_engine.get_mean_state_vector()
return None
def get_dynamic_data(data_access_component: DataAccessComponent, roi: str,
roi_grid: str, start_time: str, stop_time: str,
modis_directory: str, cams_tiff_directory: str,
s2_l1c_directory: str):
create_dir(modis_directory)
create_dir(cams_tiff_directory)
create_dir(s2_l1c_directory)
modis_delta = datetime.timedelta(days=16)
start = datetime.datetime.strptime(start_time, '%Y-%m-%d')
modis_start = start - modis_delta
modis_start_time = datetime.datetime.strftime(modis_start, '%Y-%m-%d')
end = datetime.datetime.strptime(stop_time, '%Y-%m-%d')
modis_end = end + modis_delta
modis_end_time = datetime.datetime.strftime(modis_end, '%Y-%m-%d')
rg = roi_grid
if roi_grid is 'none':
rg = None
print('Retrieving MODIS BRDF descriptors ...')
modis_urls = data_access_component.get_data_urls(roi, modis_start_time,
modis_end_time,
'MCD43A1.006', rg)
set_permissions(modis_urls)
create_sym_links(modis_urls, modis_directory)
print('Retrieving CAMS data ...')
cams_urls = data_access_component.get_data_urls(roi, start_time, stop_time,
'CAMS_TIFF', rg)
set_permissions(cams_urls)
create_sym_links(cams_urls, cams_tiff_directory)
print('Retrieving S2 L1C data ...')
s2_urls = data_access_component.get_data_urls(roi, start_time, stop_time,
'AWS_S2_L1C, S2_L1C', rg)
set_permissions(s2_urls)
create_sym_links(s2_urls, s2_l1c_directory)
print('Done retrieving dynamic data')
def orchestrate(dir: str) -> None:
"""
This function deals with orchestration it does not take any arguments and creates all input and output by
itself.
:return:
"""
roi = 'POLYGON ((48.0 11.3, 48.2 11.3, 48.1 11.1, 48.0 11.0, 48.0 11.3))'
start_time_as_string = '2017-01-01'
end_time_as_string = '2017-01-10'
data_access_component = DataAccessComponent()
data_access_component.read_data_stores(dir +
'/frascati_data_stores.yml')
# data_access_component.read_data_stores(
# 'C:/Users/tonio/Projekte/multiply/workshop/frascati-20180205/frascati_data_stores.yml')
priors_sm_dir = data_access_component.get_data_urls(
roi, start_time_as_string, end_time_as_string, 'SoilMoisture')[0]
priors_veg_dir = data_access_component.get_data_urls(
roi, start_time_as_string, end_time_as_string, 'Vegetation')[0]
config = Orchestrator._get_config(priors_sm_dir=priors_sm_dir,
priors_veg_dir=priors_veg_dir,
roi=roi,
start_time=start_time_as_string,
end_time=end_time_as_string)
config_file_name = dir + '/config.yml'
yaml.dump(config, open(config_file_name, 'w+'))
config_as_dict = AttributeDict(**config)
# mcd43_urls = data_access_component.get_data_urls(roi, start_time_as_string, end_time_as_string, 'MCD43')
# todo get SAR pre-processed data
# todo get MODIS data
# todo get S2 pre-processed data
# todo get priors
parameters = config_as_dict['Inference']['parameters']
start_time = datetime.strptime(config_as_dict['General']['start_time'],
"%Y-%m-%d")
end_time = datetime.strptime(config_as_dict['General']['end_time'],
"%Y-%m-%d")
time_interval = config_as_dict['General']['time_interval']
time_interval_unit = config_as_dict['General']['time_interval_unit']
current_time = start_time
prior_file_dicts = []
while current_time < end_time:
time_string = current_time.strftime("%Y-%m-%d")
prior_engine = PriorEngine(datestr=time_string,
variables=parameters,
config=(config_file_name))
prior_file_dicts.append(prior_engine.get_priors())
current_time = Orchestrator._increase_time_step(
current_time, time_interval, time_interval_unit)
if not os.path.exists(dir):
os.makedirs(dir)
create_dir(modis_dir)
create_dir(cams_tiff_dir)
create_dir(s2_dir)
create_dir(provided_sdrs_dir)
# read request file for parameters
with open(configuration_file) as f:
parameters = yaml.load(f)
roi = parameters['General']['roi']
dac = DataAccessComponent()
logger.info('0-33')
s2_urls = dac.get_data_urls(roi, start_date, stop_date, 'Sentinel-2')
s2_l1c_urls = []
sdr_urls = []
for s2_url in s2_urls:
if is_valid(s2_url, DataTypeConstants.S2_L1C):
s2_l1c_urls.append(s2_url)
elif is_valid(s2_url, DataTypeConstants.S2_L2):
sdr_urls.append(s2_url)
create_sym_links(s2_l1c_urls, s2_dir)
create_sym_links(sdr_urls, provided_sdrs_dir)
if len(s2_l1c_urls) > 0:
logger.info('33-67')
modis_delta = datetime.timedelta(days=16)
start = datetime.datetime.strptime(start_date, '%Y-%m-%d')
script_progress_logger.addHandler(script_progress_logging_handler)
# setup parameters
configuration_file = sys.argv[1]
start_date = sys.argv[2]
end_date = sys.argv[3]
s1_slc_dir = sys.argv[4]
with open(configuration_file) as f:
parameters = yaml.load(f)
roi = parameters['General']['roi']
temporal_filter = parameters['SAR']['speckle_filter']['multi_temporal'][
'files']
from multiply_data_access import DataAccessComponent
dac = DataAccessComponent()
before_sar_dir = os.path.join(s1_slc_dir, 'before')
if not os.path.exists(before_sar_dir):
os.makedirs(before_sar_dir)
one_day = datetime.timedelta(days=1)
start = datetime.datetime.strptime(start_date, '%Y-%m-%d')
before = start
before_date = datetime.datetime.strftime(before, '%Y-%m-%d')
previous_num_before = -1
num_before = 0
script_progress_logger.info('0-0')
while num_before < temporal_filter:
if previous_num_before != num_before:
script_progress_logger.info(
from multiply_data_access import DataAccessComponent
dac = DataAccessComponent()
dac.read_data_stores('frascati_data_stores.yml')
sm_dir = dac.get_data_urls('', '2017-01-01', '2017-01-10', 'SoilMoisture')
print(sm_dir)
veg_dir = dac.get_data_urls('', '2017-01-01', '2017-01-10', 'Vegetation')
print(veg_dir)
#!{PYTHON}
# example syntax: data_access_put_s2_l2.py /data/mx/sdrs
from multiply_data_access import DataAccessComponent
import logging
import os
import sys
logger = logging.getLogger('ScriptProgress')
logger.setLevel(logging.INFO)
# setup parameters
configuration_file = sys.argv[1]
start_date = sys.argv[2]
stop_date = sys.argv[3]
sdrs_dir = sys.argv[4]
provided_sdrs_dir = sys.argv[5]
dac = DataAccessComponent()
sdrs = os.listdir(sdrs_dir)
for i, sdr in enumerate(sdrs):
logger.info(f'{int((i/len(sdrs)) * 100)}-{int((i+1/len(sdrs)) * 100)}')
if not os.path.exists(os.path.join(provided_sdrs_dir, sdr)):
dac.put(os.path.join(sdrs_dir, sdr), 'S2L2')
logger.info('100-100')
def _set_up_data_access(self, data_stores):
data_access_component = DataAccessComponent()
data_stores_file = utils.get_data_stores_file()
data_access_component.read_data_stores(data_stores_file)
return data_access_component
logger = logging.getLogger('ScriptProgress')
logger.setLevel(logging.INFO)
# setup parameters
configuration_file = sys.argv[1]
start_date = sys.argv[2]
stop_date = sys.argv[3]
emu_dir = sys.argv[4]
dem_dir = sys.argv[5]
def create_dir(dir):
if not os.path.exists(dir):
os.makedirs(dir)
create_dir(emu_dir)
create_dir(dem_dir)
# read request file for parameters
with open(configuration_file) as f:
parameters = yaml.safe_load(f)
roi = parameters['General']['roi']
dac = DataAccessComponent()
logger.info('0-50')
emu_urls = dac.get_data_urls(roi, start_date, stop_date, 'ISO_MSI_A_EMU,ISO_MSI_B_EMU')
create_sym_links(emu_urls, emu_dir)
logger.info('50-100')
dem_urls = dac.get_data_urls(roi, start_date, stop_date, 'Aster_DEM')
create_sym_links(dem_urls, dem_dir)
logger.info('100-100')
def put_data(data_access_component: DataAccessComponent, sdrs_directory: str):
for sdr in os.listdir(sdrs_directory):
data_access_component.put(sdrs_directory + '/' + sdr, 'S2L2')
return