def check_availability(inventory_gdf, download_dir, data_mount): '''This function checks if the data is already downloaded or available through a mount point on DIAS cloud ''' from ost import Sentinel1Scene # add download path, or set to None if not found inventory_gdf['download_path'] = inventory_gdf.identifier.apply( lambda row: Sentinel1Scene(row).get_path(download_dir, data_mount)) return inventory_gdf
def _to_ard_batch(inventory_df, download_dir, processing_dir, ard_parameters, subset=None, polar='VV,VH,HH,HV', max_workers=int(os.cpu_count() / 2)): # we create a processing dictionary, # where all frames are grouped into acquisitions processing_dict = _create_processing_dict(inventory_df) for track, allScenes in processing_dict.items(): for list_of_scenes in processing_dict[track]: # get acquisition date acquisition_date = Sentinel1Scene(list_of_scenes[0]).start_date # create a subdirectory baed on acq. date out_dir = opj(processing_dir, track, acquisition_date) os.makedirs(out_dir, exist_ok=True) # check if already processed if os.path.isfile(opj(out_dir, '.processed')): logger.debug('Acquisition from {} of track {}' 'already processed'.format( acquisition_date, track)) else: # get the paths to the file for i in list_of_scenes: s1_process_scene = Sentinel1Scene(i) s1_process_scene.ard_parameters = ard_parameters scene_paths = ([ Sentinel1Scene(i).get_path(download_dir) for i in list_of_scenes ]) s1_process_scene.create_ard( filelist=scene_paths, out_dir=out_dir, out_prefix=s1_process_scene.start_date.replace( '-', ''), subset=subset, polar=polar, max_workers=max_workers)
def test_s1scene_metadata(s1_id): s1 = Sentinel1Scene(s1_id) s1_info = s1.info() control = { 'Scene_Identifier': 'S1A_IW_GRDH_1SDV_20191116T170638_20191116T170703_029939_036AAB_070F', 'Satellite': 'Sentinel-1A', 'Acquisition_Mode': 'Interferometric Wide Swath', 'Processing_Level': '1', 'Product_Type': 'Ground Range Detected (GRD)', 'Acquisition_Date': '20191116', 'Start_Time': '170638', 'Stop_Time': '170703', 'Absolute_Orbit': '029939', 'Relative_Orbit': '117' } assert s1_info == control
def grd_to_ard_batch(inventory_df, config_file): # load relevant config parameters with open(config_file, 'r') as file: config_dict = json.load(file) download_dir = Path(config_dict['download_dir']) data_mount = Path(config_dict['data_mount']) # where all frames are grouped into acquisitions processing_dict = _create_processing_dict(inventory_df) processing_df = pd.DataFrame( columns=['identifier', 'outfile', 'out_ls', 'error']) iter_list = [] for _, list_of_scenes in processing_dict.items(): # get the paths to the file scene_paths = ([ Sentinel1Scene(scene).get_path(download_dir, data_mount) for scene in list_of_scenes ]) iter_list.append(scene_paths) # now we run with godale, which works also with 1 worker executor = Executor(executor=config_dict['executor_type'], max_workers=config_dict['max_workers']) for task in executor.as_completed(func=grd_to_ard.grd_to_ard, iterable=iter_list, fargs=([ str(config_file), ])): list_of_scenes, outfile, out_ls, error = task.result() # return the info of processing as dataframe temp_df = create_processed_df(inventory_df, list_of_scenes, outfile, out_ls, error) processing_df = processing_df.append(temp_df) return processing_df
def test_s1scene_metadata(s1_id): s1 = Sentinel1Scene(s1_id) control_id = 'S1A_IW_GRDH_1SDV_20141003T040550_' \ '20141003T040619_002660_002F64_EC04' control_dict = { 'Scene_Identifier': 'S1A_IW_GRDH_1SDV_20141003T040550_20141003T040619_' '002660_002F64_EC04', 'Satellite': 'Sentinel-1A', 'Acquisition_Mode': 'Interferometric Wide Swath', 'Processing_Level': '1', 'Product_Type': 'Ground Range Detected (GRD)', 'Acquisition_Date': '20141003', 'Start_Time': '040550', 'Stop_Time': '040619', 'Absolute_Orbit': '002660', 'Relative_Orbit': '138' } assert control_dict == s1.info_dict() assert s1.scene_id == control_id
def s1_grd_notnr_ost_product(s1_grd_notnr): scene_id = s1_grd_notnr.split('/')[-1] return (scene_id, Sentinel1Scene(scene_id))
def s1_slc_ost_slave(s1_slc_slave): scene_id = os.path.basename(s1_slc_slave).replace('.zip', '') return (scene_id, Sentinel1Scene(scene_id))
def s1_slc_ost_master(s1_slc_master): scene_id = os.path.basename(s1_slc_master).replace('.zip', '') return (scene_id, Sentinel1Scene(scene_id))