def get_covered_acquisitions_by_track_date(aoi, acqs, threshold_pixel, orbit_file, orbit_dir, platform, result_file, selected_track_list):
#util.print_acquisitions(aoi['id'], util.create_acqs_from_metadata(acqs))
logger.info("\nget_covered_acquisitions_by_track_date")
logger.info("PROCESSING AOI : %s : %s" %(aoi['id'], aoi['location']))
grouped_matched = util.group_acqs_by_track_date_from_metadata(acqs) #group_acqs_by_track(acqs)
logger.info("grouped_matched Done")
print_groups(grouped_matched)
matched_ids = grouped_matched["acq_info"].keys()
#logger.info("grouped_matched : %s" %grouped_matched)
logger.info("matched_ids : %s" %matched_ids)
logger.info("PLATFORM : %s" %platform)
orbit_file = os.path.basename(orbit_file)
mission = "S1A"
if platform == "Sentinel-1B":
mission = "S1B"
selected_track_acqs = {}
logger.info("Tracks to process : %s" %grouped_matched["grouped"])
for track in grouped_matched["grouped"]:
logger.info("get_covered_acquisitions_by_track_date : Processing track : %s" %track)
if len(selected_track_list)>0:
if int(track) not in selected_track_list:
logger.info("%s not in selected_track_list %s. So skipping this track" %(track, selected_track_list))
continue
selected_track_dt_acqs = {}
for track_dt in grouped_matched["grouped"][track]:
#water_mask_test1(track, orbit_or_track_dt, acq_info, grouped_matched_orbit_number, aoi_location, aoi_id, orbit_file = None)
selected, result = gtUtil.water_mask_check(track, track_dt, grouped_matched["acq_info"], grouped_matched["grouped"][track][track_dt], aoi['location'], aoi['id'], threshold_pixel, mission, orbit_file, orbit_dir)
with open(result_file, 'a') as fo:
cw = csv.writer(fo, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL)
cw.writerow([result['dt'], result['track'],result['Track_POEORB_Land'] , result['ACQ_Union_POEORB_Land'], result['acq_union_land_area'], result['res'], result['WATER_MASK_PASSED'], result['master_ipf_count'], result['slave_ipf_count'],result['matched'], result['BL_PASSED'], result['candidate_pairs'], result['Track_AOI_Intersection'], result['ACQ_POEORB_AOI_Intersection'], result['acq_union_aoi_intersection'] ])
if selected:
logger.info("SELECTED : aoi : %s track : %s track_dt : %s" %(aoi['id'], track, track_dt))
selected_acqs = []
for acq_id in grouped_matched["grouped"][track][track_dt]:
acq = grouped_matched["acq_info"][acq_id]
#acq.pv = pv #util.get_processing_version(acq.identifier)
#util.update_grq(acq_id, acq.pv)
logger.info("APPENDING : %s" %acq_id)
selected_acqs.append(acq)
selected_track_dt_acqs[track_dt] = selected_acqs
selected_track_acqs[track] = selected_track_dt_acqs
#exit (0)
logger.info("get_covered_acquisitions_by_track_date returns : %s" %selected_track_acqs)
return selected_track_acqs
def get_covered_acquisitions(aoi, acqs, orbit_file):
#util.print_acquisitions(aoi['id'], util.create_acqs_from_metadata(acqs))
logger.info("AOI : %s" %aoi['location'])
grouped_matched = util.group_acqs_by_orbit_number_from_metadata(acqs) #group_acqs_by_track(acqs)
matched_ids = grouped_matched["acq_info"].keys()
#logger.info("grouped_matched : %s" %grouped_matched)
logger.info("matched_ids : %s" %matched_ids)
selected_track_acqs = {}
for track in grouped_matched["grouped"]:
selected_orbitnumber_acqs = {}
for orbitnumber in grouped_matched["grouped"][track]:
selected = gtUtil.water_mask_check(track, orbitnumber, grouped_matched["acq_info"], grouped_matched["grouped"][track][orbitnumber], aoi['location'], aoi['id'], threshold_pixel, orbit_file)
if selected:
logger.info("SELECTED")
selected_acqs = []
for pv in grouped_matched["grouped"][track][orbitnumber]:
for acq_id in grouped_matched["grouped"][track][orbitnumber][pv]:
acq = grouped_matched["acq_info"][acq_id]
if not acq.pv:
acq.pv = pv #util.get_processing_version(acq.identifier)
#util.update_grq(acq_id, acq.pv)
logger.info("APPENDING : %s" %acq_id)
selected_acqs.append(acq)
selected_orbitnumber_acqs[orbitnumber] = selected_acqs
selected_track_acqs[track] = selected_orbitnumber_acqs
#exit (0)
return selected_track_acqs
def get_candidate_pair_list_by_orbitnumber(track, selected_track_acqs,
aoi_data, orbit_data, job_data,
aoi_blacklist, threshold_pixel):
logger.info("get_candidate_pair_list : %s Orbits" %
len(selected_track_acqs.keys()))
candidate_pair_list = []
orbit_ipf_dict = {}
min_max_count = 0
aoi_location = aoi_data['aoi_location']
logger.info("aoi_location : %s " % aoi_location)
for orbitnumber in sorted(selected_track_acqs.keys(), reverse=True):
logger.info(orbitnumber)
slaves_track = {}
slave_acqs = []
master_acqs = selected_track_acqs[orbitnumber]
master_ipf_count, master_starttime, master_endtime, master_location, master_track, direction, master_orbitnumber = util.get_union_data_from_acqs(
master_acqs)
#master_ipf_count = util.get_ipf_count(master_acqs)
#master_union_geojson = util.get_union_geojson_acqs(master_acqs)
#util.print_acquisitions(aoi_data['aoi_id'], master_acqs)
query = util.get_overlapping_slaves_query(orbit_data, aoi_location,
track, direction,
master_orbitnumber)
acqs = [
i['fields']['partial'][0] for i in util.query_es2(query, es_index)
]
logger.info("Found {} slave acqs : {}".format(
len(acqs), json.dumps([i['id'] for i in acqs], indent=2)))
#matched_acqs = util.create_acqs_from_metadata(process_query(query))
slave_acqs = util.create_acqs_from_metadata(acqs)
logger.info("\nSLAVE ACQS")
#util.print_acquisitions(aoi_id, slave_acqs)
slave_grouped_matched = util.group_acqs_by_orbit_number(slave_acqs)
#slave_grouped_matched = util.group_acqs_by_track_date(slave_acqs)
orbitnumber_pv = {}
selected_slave_acqs_by_orbitnumber = {}
logger.info("\n\n\nTRACK : %s" % track)
rejected_slave_orbitnumber = []
for slave_orbitnumber in sorted(
slave_grouped_matched["grouped"][track], reverse=True):
selected_slave_acqs = []
selected, result = gtUtil.water_mask_check(
track, slave_orbitnumber, slave_grouped_matched["acq_info"],
slave_grouped_matched["grouped"][track][slave_orbitnumber],
aoi_location, aoi, threshold_pixel)
if not selected:
logger.info(
"Removing the acquisitions of orbitnumber : %s for failing water mask test"
% slave_orbitnumber)
rejected_slave_orbitnumber.append(slave_orbitnumber)
continue
pv_list = []
for pv in slave_grouped_matched["grouped"][track][
slave_orbitnumber]:
logger.info("\tpv : %s" % pv)
pv_list.append(pv)
slave_ids = slave_grouped_matched["grouped"][track][
slave_orbitnumber][pv]
for slave_id in slave_ids:
selected_slave_acqs.append(
slave_grouped_matched["acq_info"][slave_id])
orbitnumber_pv[slave_orbitnumber] = len(list(set(pv_list)))
selected_slave_acqs_by_orbitnumber[
slave_orbitnumber] = selected_slave_acqs
for slave_orbitnumber in sorted(
selected_slave_acqs_by_orbitnumber.keys(), reverse=True):
slave_ipf_count = orbitnumber_pv[slave_orbitnumber]
slave_acqs = selected_slave_acqs_by_orbitnumber[slave_orbitnumber]
result['master_ipf_count'] = master_ipf_count
result['slave_ipf_count'] = slave_ipf_count
matched, orbit_candidate_pair = process_enumeration(
master_acqs, master_ipf_count, slave_acqs, slave_ipf_count,
direction, aoi_location, aoi_blacklist, job_data, result)
result['matched'] = matched
result['candidate_pairs'] = orbit_candidate_pair
write_result_file(result_file, result)
if matched:
for candidate_pair in orbit_candidate_pair:
publish_initiator_pair(candidate_pair, job_data,
orbit_data)
logger.info("\n\nSUCCESSFULLY PUBLISHED : %s" %
candidate_pair)
candidate_pair_list.append(orbit_candidate_pair)
min_max_count = min_max_count + 1
if min_max_count >= MIN_MATCH:
return min_max_count, candidate_pair_list
return min_max_count, candidate_pair_list
def get_covered_acquisitions_by_track_date(aoi, acqs, threshold_pixel,
orbit_file, orbit_dir, platform,
result_file, selected_track_list):
#util.print_acquisitions(aoi['id'], util.create_acqs_from_metadata(acqs))
logger.info("\nget_covered_acquisitions_by_track_date")
#logger.info(acqs)
logger.info("PROCESSING AOI : %s : \nlocation %s" %
(aoi['id'], aoi['location']))
grouped_matched = util.group_acqs_by_track_date_from_metadata(
acqs) #group_acqs_by_track(acqs)
logger.info("grouped_matched Done")
print_groups(grouped_matched)
matched_ids = grouped_matched["acq_info"].keys()
#logger.info("grouped_matched : %s" %grouped_matched)
logger.info("matched_ids : %s" % matched_ids)
logger.info("PLATFORM : %s" % platform)
orbit_type = "P"
orbit_file = os.path.basename(orbit_file)
mission = "S1A"
if platform == "Sentinel-1B":
mission = "S1B"
selected_track_acqs = {}
result_track_acqs = {}
logger.info("Tracks to process : %s" % grouped_matched["grouped"])
for track in grouped_matched["grouped"]:
logger.info(
"get_covered_acquisitions_by_track_date : Processing track : %s" %
track)
if len(selected_track_list) > 0:
if int(track) not in selected_track_list:
logger.info(
"%s not in selected_track_list %s. So skipping this track"
% (track, selected_track_list))
continue
selected_track_dt_acqs = {}
result_track_dt_acqs = {}
for track_dt in grouped_matched["grouped"][track]:
filtered_acd_ids, dropped_ids = util.filter_acq_ids(
grouped_matched["acq_info"],
grouped_matched["grouped"][track][track_dt])
logger.info("filtered_acd_ids : %s" % filtered_acd_ids)
valid_orbit = False
valid_orbit_err = ''
try:
selected, result, removed_ids = gtUtil.water_mask_check(
track, track_dt, grouped_matched["acq_info"],
filtered_acd_ids, aoi['location'], aoi['id'],
threshold_pixel, mission, orbit_type, orbit_file,
orbit_dir)
valid_orbit = True
orbit_name = orbit_file.split('.EOF')[0].strip()
if len(removed_ids) > 0:
logger.info("Removed Acquisitions by WaterMaskTest : %s" %
removed_ids)
for acq_id in removed_ids:
logger.info("removing %s from filtered_acd_ids" %
acq_id)
filtered_acd_ids.remove(acq_id)
logger.info("filtered_acd_ids : %s:" % filtered_acd_ids)
except InvalidOrbitException as err:
selected = False
valid_orbit = False
valid_orbit_err = err
result['orbit_name'] = orbit_name
result['track'] = track
result['master_dropped_ids'] = dropped_ids
result_track_dt_acqs[track_dt] = result
starttime, endtime = util.get_start_end_time2(
grouped_matched["acq_info"], filtered_acd_ids)
result['starttime'] = starttime
result['endtime'] = endtime
result['union_geojson'] = aoi['location']
#master_dt_str = util.get_time_str_with_format(track_dt, "%Y%m%dT%H%M%S")
logger.info("master_dt_str : %s" % track_dt)
result['list_master_dt'] = track_dt
result['list_slave_dt'] = track_dt
result['master_count'] = 1
result['slave_count'] = 0
if selected:
logger.info("SELECTED : aoi : %s track : %s track_dt : %s" %
(aoi['id'], track, track_dt))
selected_acqs = []
for acq_id in filtered_acd_ids:
acq = grouped_matched["acq_info"][acq_id]
#acq.pv = pv #util.get_processing_version(acq.identifier)
#util.update_grq(acq_id, acq.pv)
logger.info("APPENDING : %s" % acq_id)
selected_acqs.append(acq)
selected_track_dt_acqs[track_dt] = selected_acqs
result['orbit_quality_check_passed'] = True
else:
result['result'] = False
id_hash = '0000'
result['orbit_quality_check_passed'] = False
result['failed_orbit'] = 'reference'
publish_result(result, id_hash)
if not valid_orbit:
raise InvalidOrbitException(valid_orbit_err)
try:
with open(result_file, 'a') as fo:
cw = csv.writer(fo,
delimiter=',',
quotechar='"',
quoting=csv.QUOTE_MINIMAL)
cw.writerow([
result.get('dt', ''),
result.get('orbit_name', ''), "Primary",
result.get('track', ''),
result.get('Track_POEORB_Land', ''),
result.get('ACQ_Union_POEORB_Land', ''),
result.get('delta_area', ''),
result.get('res', ''),
result.get('area_threshold_passed', ''),
result.get('WATER_MASK_PASSED', ''),
result.get('primary_ipf_count', ''),
result.get('secondary_ipf_count', ''),
result.get('BL_PASSED', ''),
result.get('matched', ''),
result.get('candidate_pairs', ''),
result.get('fail_reason', ''),
result.get('comment', ''),
result.get('Track_AOI_Intersection', ''),
result.get('ACQ_POEORB_AOI_Intersection', '')
])
except Exception as err:
logger.info("\n\nERROR Writing to csv file : %s" % str(err))
traceback.print_exc()
selected_track_acqs[track] = selected_track_dt_acqs
logger.info("CHECK: selected_track_acqs[track] : %s" %
selected_track_acqs[track])
result_track_acqs[track] = result_track_dt_acqs
#exit (0)
logger.info("get_covered_acquisitions_by_track_date returns : %s" %
selected_track_acqs)
return selected_track_acqs, result_track_acqs
def get_candidate_pair_list(aoi, track, selected_track_acqs, aoi_data,
orbit_data, job_data, aoi_blacklist,
threshold_pixel, acquisition_version):
logger.info("get_candidate_pair_list : %s Orbits" %
len(selected_track_acqs.keys()))
candidate_pair_list = []
orbit_ipf_dict = {}
min_max_count = 0
aoi_location = aoi_data['aoi_location']
logger.info("aoi_location : %s " % aoi_location)
result_file = "RESULT_SUMMARY_%s.csv" % aoi
aoi_id = aoi_data['aoi_id']
orbitNumber = []
for track_dt in sorted(selected_track_acqs.keys(), reverse=True):
logger.info(track_dt)
slaves_track = {}
slave_acqs = []
master_acqs = selected_track_acqs[track_dt]
master_ipf_count, master_starttime, master_endtime, master_location, master_track, direction, master_orbitnumber = util.get_union_data_from_acqs(
master_acqs)
#master_ipf_count = util.get_ipf_count(master_acqs)
#master_union_geojson = util.get_union_geojson_acqs(master_acqs)
orbitNumber.append(master_orbitnumber)
#util.print_acquisitions(aoi_data['aoi_id'], master_acqs)
query = util.get_overlapping_slaves_query(
util.get_isoformat_date(master_starttime), aoi_location, track,
direction, orbit_data['platform'], master_orbitnumber,
acquisition_version)
logger.info("Slave Finding Query : %s" % query)
es_index = "grq_%s_acquisition-s1-iw_slc/acquisition-S1-IW_SLC/" % (
acquisition_version)
logger.info("es_index : %s" % es_index)
acqs = [
i['fields']['partial'][0] for i in util.query_es2(query, es_index)
]
logger.info("Found {} slave acqs : {}".format(
len(acqs), json.dumps([i['id'] for i in acqs], indent=2)))
if len(acqs) == 0:
logger.info(
"ERROR ERROR : NO SLAVE FOUND for AOI %s and track %s" %
(aoi_data['aoi_id'], track))
continue
#matched_acqs = util.create_acqs_from_metadata(process_query(query))
slave_acqs = create_acqs_from_metadata(acqs)
logger.info("\nSLAVE ACQS")
#util.print_acquisitions(aoi_id, slave_acqs)
slave_grouped_matched = util.group_acqs_by_track_date(slave_acqs)
logger.info("Priniting Slaves")
print_groups(slave_grouped_matched)
track_dt_pv = {}
selected_slave_acqs_by_track_dt = {}
logger.info("\n\n\nTRACK : %s" % track)
rejected_slave_track_dt = []
for slave_track_dt in sorted(slave_grouped_matched["grouped"][track],
reverse=True):
selected_slave_acqs = []
orbit_file = None
orbit_dt = slave_track_dt.replace(minute=0, hour=12,
second=0).isoformat()
logger.info("\n\n\nProcessing AOI: %s Track : %s orbit_dt : %s" %
(aoi, track, orbit_dt))
slave_platform = get_group_platform(
slave_grouped_matched["grouped"][track][slave_track_dt],
slave_grouped_matched["acq_info"])
logger.info("slave_platform : %s" % slave_platform)
isOrbitFile, orbit_id, orbit_url, orbit_file = util.get_orbit_file(
orbit_dt, slave_platform)
if isOrbitFile:
logger.info("orbit id : %s : orbit_url : %s" %
(orbit_id, orbit_url))
slave_orbit_file_path = os.path.basename(orbit_url)
downloaded = gtUtil.download_orbit_file(orbit_url, orbit_file)
if downloaded:
logger.info("Slave Orbiut File Downloaded")
#orbit_file = slave_orbit_file_path
orbit_dir = os.getcwd()
logger.info("orbit_file : %s\norbit_dir : %s" %
(orbit_file, orbit_dir))
orbit_dir = os.getcwd()
mission = "S1A"
if slave_platform == "Sentinel-1B":
mission = "S1B"
logger.info("slave_platform : %s" % slave_platform)
if orbit_file:
logger.info(
"Orbit File Exists, so Running water_mask_check for slave for date %s is running with orbit file : %s in %s"
% (slave_track_dt, orbit_file, orbit_dir))
selected, result = gtUtil.water_mask_check(
track, slave_track_dt, slave_grouped_matched["acq_info"],
slave_grouped_matched["grouped"][track][slave_track_dt],
aoi_location, aoi, threshold_pixel, mission, orbit_file,
orbit_dir)
if not selected:
logger.info(
"Removing the acquisitions of orbitnumber : %s for failing water mask test"
% slave_track_dt)
rejected_slave_track_dt.append(slave_track_dt)
write_result_file(result_file, result)
'''
with open(result_file, 'a') as fo:
cw = csv.writer(fo, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL)
cw.writerow([result['dt'], result['track'],result['Track_POEORB_Land'] , result['ACQ_Union_POEORB_Land'], result['acq_union_land_area'], result['res'], result['WATER_MASK_PASSED'], result['master_ipf_count'], result['slave_ipf_count'],result['matched'], result['BL_PASSED'], result['candidate_pairs'], result['Track_AOI_Intersection'], result['ACQ_POEORB_AOI_Intersection'], result['acq_union_aoi_intersection']])
'''
continue
else:
logger.info(
"Orbit File NOT Exists, so Running water_mask_check for slave for date %s is running without orbit file."
% slave_track_dt)
selected, result = gtUtil.water_mask_check(
track, slave_track_dt, slave_grouped_matched["acq_info"],
slave_grouped_matched["grouped"][track][slave_track_dt],
aoi_location, aoi, threshold_pixel)
if not selected:
logger.info(
"Removing the acquisitions of orbitnumber : %s for failing water mask test"
% slave_track_dt)
rejected_slave_track_dt.append(slave_track_dt)
continue
slave_ipf_count = util.get_ipf_count_by_acq_id(
slave_grouped_matched["grouped"][track][slave_track_dt],
slave_grouped_matched["acq_info"])
logger.info("slave_ipf_count : %s" % slave_ipf_count)
selected_slave_acqs = list()
slave_ids = slave_grouped_matched["grouped"][track][slave_track_dt]
for slave_id in slave_ids:
selected_slave_acqs.append(
slave_grouped_matched["acq_info"][slave_id])
track_dt_pv[slave_track_dt] = slave_ipf_count
selected_slave_acqs_by_track_dt[
slave_track_dt] = selected_slave_acqs
logger.info("Processing Slaves with date : %s" % slave_track_dt)
result['master_ipf_count'] = master_ipf_count
result['slave_ipf_count'] = slave_ipf_count
matched, orbit_candidate_pair, result = process_enumeration(
master_acqs, master_ipf_count, selected_slave_acqs,
slave_ipf_count, direction, aoi_location, aoi_blacklist,
job_data, result)
result['matched'] = matched
result['candidate_pairs'] = orbit_candidate_pair
write_result_file(result_file, result)
if matched:
for candidate_pair in orbit_candidate_pair:
candidate_pair["master_track_dt"] = track_dt
candidate_pair["slave_trck_dt"] = slave_track_dt
publish_initiator_pair(candidate_pair, job_data,
orbit_data, aoi_id)
candidate_pair_list.append(orbit_candidate_pair)
min_max_count = min_max_count + 1
if min_max_count >= MIN_MATCH:
return min_max_count, candidate_pair_list
return min_max_count, candidate_pair_list