def orchestrate(dir: str) -> None:
"""
This function deals with orchestration it does not take any arguments and creates all input and output by
itself.
:return:
"""
roi = 'POLYGON ((48.0 11.3, 48.2 11.3, 48.1 11.1, 48.0 11.0, 48.0 11.3))'
start_time_as_string = '2017-01-01'
end_time_as_string = '2017-01-10'
data_access_component = DataAccessComponent()
data_access_component.read_data_stores(dir +
'/frascati_data_stores.yml')
# data_access_component.read_data_stores(
# 'C:/Users/tonio/Projekte/multiply/workshop/frascati-20180205/frascati_data_stores.yml')
priors_sm_dir = data_access_component.get_data_urls(
roi, start_time_as_string, end_time_as_string, 'SoilMoisture')[0]
priors_veg_dir = data_access_component.get_data_urls(
roi, start_time_as_string, end_time_as_string, 'Vegetation')[0]
config = Orchestrator._get_config(priors_sm_dir=priors_sm_dir,
priors_veg_dir=priors_veg_dir,
roi=roi,
start_time=start_time_as_string,
end_time=end_time_as_string)
config_file_name = dir + '/config.yml'
yaml.dump(config, open(config_file_name, 'w+'))
config_as_dict = AttributeDict(**config)
# mcd43_urls = data_access_component.get_data_urls(roi, start_time_as_string, end_time_as_string, 'MCD43')
# todo get SAR pre-processed data
# todo get MODIS data
# todo get S2 pre-processed data
# todo get priors
parameters = config_as_dict['Inference']['parameters']
start_time = datetime.strptime(config_as_dict['General']['start_time'],
"%Y-%m-%d")
end_time = datetime.strptime(config_as_dict['General']['end_time'],
"%Y-%m-%d")
time_interval = config_as_dict['General']['time_interval']
time_interval_unit = config_as_dict['General']['time_interval_unit']
current_time = start_time
prior_file_dicts = []
while current_time < end_time:
time_string = current_time.strftime("%Y-%m-%d")
prior_engine = PriorEngine(datestr=time_string,
variables=parameters,
config=(config_file_name))
prior_file_dicts.append(prior_engine.get_priors())
current_time = Orchestrator._increase_time_step(
current_time, time_interval, time_interval_unit)
def get_static_data(data_access_component: DataAccessComponent, roi: str,
roi_grid: str, start_time: str, stop_time: str,
emulation_directory: str, dem_directory: str):
create_dir(emulation_directory)
create_dir(dem_directory)
rg = roi_grid
if roi_grid is 'none':
rg = None
print('Retrieving emulators ...')
emu_urls = data_access_component.get_data_urls(
roi, start_time, stop_time, 'ISO_MSI_A_EMU,ISO_MSI_B_EMU', rg)
set_permissions(emu_urls)
create_sym_links(emu_urls, emulation_directory)
print('Retrieving DEM ...')
dem_urls = data_access_component.get_data_urls(roi, start_time, stop_time,
'Aster_DEM', rg)
set_permissions(dem_urls)
create_sym_links(dem_urls, dem_directory)
print('Done retrieving static data')
def get_dynamic_data(data_access_component: DataAccessComponent, roi: str,
roi_grid: str, start_time: str, stop_time: str,
modis_directory: str, cams_tiff_directory: str,
s2_l1c_directory: str):
create_dir(modis_directory)
create_dir(cams_tiff_directory)
create_dir(s2_l1c_directory)
modis_delta = datetime.timedelta(days=16)
start = datetime.datetime.strptime(start_time, '%Y-%m-%d')
modis_start = start - modis_delta
modis_start_time = datetime.datetime.strftime(modis_start, '%Y-%m-%d')
end = datetime.datetime.strptime(stop_time, '%Y-%m-%d')
modis_end = end + modis_delta
modis_end_time = datetime.datetime.strftime(modis_end, '%Y-%m-%d')
rg = roi_grid
if roi_grid is 'none':
rg = None
print('Retrieving MODIS BRDF descriptors ...')
modis_urls = data_access_component.get_data_urls(roi, modis_start_time,
modis_end_time,
'MCD43A1.006', rg)
set_permissions(modis_urls)
create_sym_links(modis_urls, modis_directory)
print('Retrieving CAMS data ...')
cams_urls = data_access_component.get_data_urls(roi, start_time, stop_time,
'CAMS_TIFF', rg)
set_permissions(cams_urls)
create_sym_links(cams_urls, cams_tiff_directory)
print('Retrieving S2 L1C data ...')
s2_urls = data_access_component.get_data_urls(roi, start_time, stop_time,
'AWS_S2_L1C, S2_L1C', rg)
set_permissions(s2_urls)
create_sym_links(s2_urls, s2_l1c_directory)
print('Done retrieving dynamic data')
create_dir(modis_dir)
create_dir(cams_tiff_dir)
create_dir(s2_dir)
create_dir(provided_sdrs_dir)
# read request file for parameters
with open(configuration_file) as f:
parameters = yaml.load(f)
roi = parameters['General']['roi']
dac = DataAccessComponent()
logger.info('0-33')
s2_urls = dac.get_data_urls(roi, start_date, stop_date, 'Sentinel-2')
s2_l1c_urls = []
sdr_urls = []
for s2_url in s2_urls:
if is_valid(s2_url, DataTypeConstants.S2_L1C):
s2_l1c_urls.append(s2_url)
elif is_valid(s2_url, DataTypeConstants.S2_L2):
sdr_urls.append(s2_url)
create_sym_links(s2_l1c_urls, s2_dir)
create_sym_links(sdr_urls, provided_sdrs_dir)
if len(s2_l1c_urls) > 0:
logger.info('33-67')
modis_delta = datetime.timedelta(days=16)
start = datetime.datetime.strptime(start_date, '%Y-%m-%d')
modis_start = start - modis_delta
modis_start_date = datetime.datetime.strftime(modis_start, '%Y-%m-%d')
one_day = datetime.timedelta(days=1)
start = datetime.datetime.strptime(start_date, '%Y-%m-%d')
before = start
before_date = datetime.datetime.strftime(before, '%Y-%m-%d')
previous_num_before = -1
num_before = 0
script_progress_logger.info('0-0')
while num_before < temporal_filter:
if previous_num_before != num_before:
script_progress_logger.info(
f'{int((num_before / temporal_filter) * 50)}-'
f'{int(((num_before + 1) / temporal_filter) * 50)}')
before -= one_day
before_date = datetime.datetime.strftime(before, '%Y-%m-%d')
data_urls_before = dac.get_data_urls(roi, before_date, start_date,
'S1_SLC')
create_sym_links(data_urls_before, before_sar_dir)
processor = SARPreProcessor(config=configuration_file,
input=before_sar_dir,
output=before_sar_dir)
list = processor.create_processing_file_list()
previous_num_before = num_before
num_before = len(list[0]) + (len(list[1]) / 2.)
after_sar_dir = os.path.join(s1_slc_dir, 'after')
if not os.path.exists(after_sar_dir):
os.makedirs(after_sar_dir)
end = datetime.datetime.strptime(end_date, '%Y-%m-%d')
after = end
after_date = datetime.datetime.strftime(after, '%Y-%m-%d')
from multiply_data_access import DataAccessComponent
dac = DataAccessComponent()
dac.read_data_stores('frascati_data_stores.yml')
sm_dir = dac.get_data_urls('', '2017-01-01', '2017-01-10', 'SoilMoisture')
print(sm_dir)
veg_dir = dac.get_data_urls('', '2017-01-01', '2017-01-10', 'Vegetation')
print(veg_dir)
logger = logging.getLogger('ScriptProgress')
logger.setLevel(logging.INFO)
# setup parameters
configuration_file = sys.argv[1]
start_date = sys.argv[2]
stop_date = sys.argv[3]
emu_dir = sys.argv[4]
dem_dir = sys.argv[5]
def create_dir(dir):
if not os.path.exists(dir):
os.makedirs(dir)
create_dir(emu_dir)
create_dir(dem_dir)
# read request file for parameters
with open(configuration_file) as f:
parameters = yaml.safe_load(f)
roi = parameters['General']['roi']
dac = DataAccessComponent()
logger.info('0-50')
emu_urls = dac.get_data_urls(roi, start_date, stop_date, 'ISO_MSI_A_EMU,ISO_MSI_B_EMU')
create_sym_links(emu_urls, emu_dir)
logger.info('50-100')
dem_urls = dac.get_data_urls(roi, start_date, stop_date, 'Aster_DEM')
create_sym_links(dem_urls, dem_dir)
logger.info('100-100')