def create_prov_es_json(id, url, prod_dir, prov_file):
"""Create provenance JSON file."""
# get info
csk_files = glob(os.path.join(prod_dir, "CSKS*"))
pf = "CSKS?"
dtype = "NA"
repo_dir = "?"
if len(csk_files) > 0:
match = CSK_RE.search(os.path.basename(csk_files[0]))
if match: pf, dtype = match.groups()
if dtype == "RAW":
dtype = "RAW_B"
repo_dir = "csk_rawb"
elif dtype == "SCS":
dtype = "SCS_B"
repo_dir = "csk_scsb"
platform = "eos:%s" % pf
platform_title = "COSMO-SkyMed Satellite %s" % pf[-1]
instrument = "eos:%s-SAR" % pf
instrument_title = "%s-SAR" % pf
level = "L0"
version = "v1.0"
collection = "eos:CSK-%s-%s" % (dtype, version)
collection_shortname = "CSK-%s-%s" % (dtype, version)
collection_label = "CSK %s Scenes %s" % (dtype, version)
collection_loc = "https://aria-dav.jpl.nasa.gov/repository/products/%s/%s" % (
repo_dir, version)
sensor = "eos:SAR"
sensor_title = "Synthetic-aperture radar (SAR)"
gov_org = "eos:ASI"
gov_org_title = "Agenzia Spaziale Italiana"
software_version = "2.0.0_201604"
software_title = "InSAR SCE (InSAR Scientific Computing Environment) v%s" % software_version
software = "eos:ISCE-%s" % software_version
software_location = "https://winsar.unavco.org/isce.html"
algorithm = "eos:metadata_extraction"
prod_dir = "file://%s%s" % (socket.getfqdn(), prod_dir)
# put in fake start/end times so that prov:used and prov:generated
# are properly created by the prov lib
fake_time = datetime.utcnow().isoformat() + 'Z'
job_id = "ingest-%s-%s" % (id, fake_time)
bundle_id = "bundle-ingest-%s-%s" % (id, fake_time)
doc = ProvEsDocument()
#bndl = doc.bundle("hysds:%s" % get_uuid(bundle_id))
bndl = None
input_id = "hysds:%s" % get_uuid(url)
input_ds = doc.granule(input_id,
None, [url], [instrument],
None,
level,
None,
label=os.path.basename(url),
bundle=bndl)
doc.collection(collection,
None,
collection_shortname,
collection_label, [collection_loc], [instrument],
level,
version,
label=collection_label,
bundle=bndl)
output_id = "hysds:%s" % get_uuid(prod_dir)
output_ds = doc.granule(output_id,
None, [prod_dir], [instrument],
collection,
level,
version,
label=id,
bundle=bndl)
doc.governingOrganization(gov_org, label=gov_org_title, bundle=bndl)
doc.platform(platform, [instrument], label=platform_title, bundle=bndl)
doc.instrument(instrument,
platform, [sensor], [gov_org],
label=instrument_title,
bundle=bndl)
doc.sensor(sensor, instrument, label=sensor_title, bundle=bndl)
doc.software(software, [algorithm],
software_version,
label=software_title,
location=software_location,
bundle=bndl)
doc.processStep("hysds:%s" % get_uuid(job_id),
fake_time,
fake_time, [software],
None,
None, [input_ds.identifier], [output_ds.identifier],
label=job_id,
bundle=bndl,
prov_type="hysds:ingest")
with open(prov_file, 'w') as f:
json.dump(json.loads(doc.serialize()), f, indent=2, sort_keys=True)
def create_prov_es_json(ctx_file, id, prod_dir, prov_file):
"""Create provenance JSON file."""
# get abs path
prod_dir = os.path.abspath(prod_dir)
# get context
with open(ctx_file) as f:
context = json.load(f)
# get mission char
mis_char = MISSION_RE.search(context.get('file')).group(1)
mis_char_lc = mis_char.lower()
# get input url
input_url = context.get('localize_urls', [{'url': None}])[0]['url']
# get info
s1_files = glob(os.path.join(prod_dir, "s1%s-*.tiff" % mis_char_lc))
pf = "S1%s" % mis_char
dtype = "NA"
repo_dir = "?"
if len(s1_files) > 0:
match = S1_RE.search(os.path.basename(s1_files[0]))
if match: pf, swathnum, dtype = match.groups()
if dtype == "raw":
dtype = "RAW"
repo_dir = "s1%s_raw" % mis_char_lc
elif dtype == "slc":
dtype = "SLC"
repo_dir = "s1%s_slc" % mis_char_lc
platform = "eos:%s" % pf
platform_title = "Sentinel1%s Satellite" % mis_char
instrument = "eos:%s-SAR" % pf
instrument_title = "%s-SAR" % pf
level = "L0"
version = "v1.0"
collection = "eos:S1%s-%s-%s" % (mis_char, dtype, version)
collection_shortname = "S1%s-%s-%s" % (mis_char, dtype, version)
collection_label = "S1%s %s Scenes %s" % (mis_char, dtype, version)
collection_loc = "https://aria-dst-dav.jpl.nasa.gov/repository/products/%s/%s" % (
repo_dir, version)
sensor = "eos:SAR"
sensor_title = "Synthetic-aperture radar (SAR)"
gov_org = "eos:ESA"
gov_org_title = "European Space Agency"
software_version = "2.0.0_201604"
software_title = "InSAR SCE (InSAR Scientific Computing Environment) v%s" % software_version
software = "eos:ISCE-%s" % software_version
software_location = "https://winsar.unavco.org/isce.html"
algorithm = "eos:metadata_extraction"
prod_dir = "file://%s%s" % (socket.getfqdn(), prod_dir)
# put in fake start/end times so that prov:used and prov:generated
# are properly created by the prov lib
fake_time = datetime.utcnow().isoformat() + 'Z'
job_id = "ingest-%s-%s" % (id, fake_time)
bundle_id = "bundle-ingest-%s-%s" % (id, fake_time)
doc = ProvEsDocument()
#bndl = doc.bundle("hysds:%s" % get_uuid(bundle_id))
bndl = None
input_id = "hysds:%s" % get_uuid(input_url)
input_ds = doc.granule(input_id,
None, [input_url], [instrument],
None,
level,
None,
label=os.path.basename(input_url),
bundle=bndl)
doc.collection(collection,
None,
collection_shortname,
collection_label, [collection_loc], [instrument],
level,
version,
label=collection_label,
bundle=bndl)
output_id = "hysds:%s" % get_uuid(prod_dir)
output_ds = doc.granule(output_id,
None, [prod_dir], [instrument],
collection,
level,
version,
label=id,
bundle=bndl)
doc.governingOrganization(gov_org, label=gov_org_title, bundle=bndl)
doc.platform(platform, [instrument], label=platform_title, bundle=bndl)
doc.instrument(instrument,
platform, [sensor], [gov_org],
label=instrument_title,
bundle=bndl)
doc.sensor(sensor, instrument, label=sensor_title, bundle=bndl)
doc.software(software, [algorithm],
software_version,
label=software_title,
location=software_location,
bundle=bndl)
doc.processStep("hysds:%s" % get_uuid(job_id),
fake_time,
fake_time, [software],
None,
None, [input_ds.identifier], [output_ds.identifier],
label=job_id,
bundle=bndl,
prov_type="hysds:ingest")
with open(prov_file, 'w') as f:
json.dump(json.loads(doc.serialize()), f, indent=2, sort_keys=True)
def create_prov_es_json(id, project, master_orbit_file, slave_orbit_file,
aria_dem_xml, aria_dem_file, work_dir, prov_file):
"""Create provenance JSON file."""
# get abs paths
work_dir = os.path.abspath(work_dir)
prod_dir = os.path.join(work_dir, id)
# get context
ctx_file = os.path.join(prod_dir, "%s.context.json" % id)
with open(ctx_file) as f:
context = json.load(f)
# put in fake start/end times so that prov:used and prov:generated
# are properly created by the prov lib
fake_time = datetime.utcnow().isoformat() + 'Z'
job_id = "create_interferogram-%s" % fake_time
bundle_id = "bundle-create_interferogram-%s" % fake_time
# create PROV-ES doc
doc = ProvEsDocument()
#bndl = doc.bundle("hysds:%s" % get_uuid(bundle_id))
bndl = None
# input and output identifiers
input_ids = {}
platform_ids = {}
instrument_ids = {}
# full url paths
work_url = "file://%s%s" % (socket.getfqdn(), work_dir)
prod_url = "%s/%s" % (work_url, id)
# add sentinel.ini file
ini_ent = doc.file("hysds:%s" % get_uuid("%s/sentinel.ini" % work_url),
["%s/sentinel.ini" % work_url],
label="sentinel.ini")
input_ids[ini_ent.identifier] = True
# add orbit files
master_orbit_ent = doc.file(
"hysds:%s" % get_uuid("%s/%s" % (work_url, master_orbit_file)),
["%s/%s" % (work_url, master_orbit_file)],
label=os.path.basename(master_orbit_file))
input_ids[master_orbit_ent.identifier] = True
slave_orbit_ent = doc.file(
"hysds:%s" % get_uuid("%s/%s" % (work_url, slave_orbit_file)),
["%s/%s" % (work_url, slave_orbit_file)],
label=os.path.basename(slave_orbit_file))
input_ids[slave_orbit_ent.identifier] = True
# get list of S1A urls
level = "L0"
version = "v1.0"
sensor = "eos:SAR"
sensor_title = "Synthetic-aperture radar (SAR)"
gov_org = "eos:ESA"
gov_org_title = "European Space Agency"
doc.governingOrganization(gov_org, label=gov_org_title, bundle=bndl)
instrument = ""
for i, url in enumerate(
[context.get('master_zip_url', ''),
context.get('slave_zip_url', '')]):
match = PLATFORM_RE.search(url)
if not match: continue
pf = match.group(1)
platform = "eos:%s" % pf
platform_title = "Sentinel1A Satellite"
instrument = "eos:%s-SAR" % pf
instrument_title = "%s-SAR" % pf
input_ds = doc.product("hysds:%s" % get_uuid(url),
None, [url], [instrument],
None,
level,
None,
label=os.path.basename(url),
bundle=bndl)
input_ids[input_ds.identifier] = True
if platform not in platform_ids:
doc.platform(platform, [instrument],
label=platform_title,
bundle=bndl)
platform_ids[platform] = True
if instrument not in instrument_ids:
doc.instrument(instrument,
platform, [sensor], [gov_org],
label=instrument_title,
bundle=bndl)
doc.sensor(sensor, instrument, label=sensor_title, bundle=bndl)
instrument_ids[instrument] = True
# add dem xml, file and related provenance
srtm_platform = "eos:SpaceShuttleEndeavour"
srtm_platform_title = "USS Endeavour"
srtm_instrument = "eos:SRTM"
srtm_instrument_title = "Shuttle Radar Topography Mission (SRTM)"
srtm_sensor = "eos:radar"
srtm_sensor_title = "radar"
srtm_gov_org = "eos:JPL"
srtm_gov_org_title = "Jet Propulsion Laboratory"
doc.governingOrganization(srtm_gov_org,
label=srtm_gov_org_title,
bundle=bndl)
dem_xml_ent = doc.file("hysds:%s" % get_uuid("%s/%s" %
(work_url, aria_dem_xml)),
["%s/%s" % (work_url, aria_dem_xml)],
label=os.path.basename(aria_dem_xml))
input_ids[dem_xml_ent.identifier] = True
dem_file_ent = doc.file("hysds:%s" % get_uuid("%s/%s" %
(work_url, aria_dem_file)),
["%s/%s" % (work_url, aria_dem_file)],
label=os.path.basename(aria_dem_file))
input_ids[dem_file_ent.identifier] = True
doc.platform(srtm_platform, [srtm_instrument],
label=srtm_platform_title,
bundle=bndl)
doc.instrument(srtm_instrument,
srtm_platform, [srtm_sensor], [srtm_gov_org],
label=srtm_instrument_title,
bundle=bndl)
doc.sensor(srtm_sensor,
srtm_instrument,
label=srtm_sensor_title,
bundle=bndl)
instrument_ids[srtm_instrument] = True
# software and algorithm
algorithm = "eos:interferogram_generation"
software_version = "2.0.0_201604"
software_title = "InSAR SCE (InSAR Scientific Computing Environment) v%s" % software_version
software = "eos:ISCE-%s" % software_version
software_location = "https://winsar.unavco.org/isce.html"
doc.software(software, [algorithm],
software_version,
label=software_title,
location=software_location,
bundle=bndl)
# output
int_level = "L2"
int_version = "v1.0"
int_collection = "eos:S1A-interferograms-%s" % int_version
int_collection_shortname = "S1A-interferograms-%s" % int_version
int_collection_label = "ISCE generated S1A interferograms %s" % int_version
int_collection_loc = "https://aria-dst-dav.jpl.nasa.gov/products/s1a_ifg/%s" % int_version
doc.collection(int_collection,
None,
int_collection_shortname,
int_collection_label, [int_collection_loc],
instrument_ids.keys(),
int_level,
int_version,
label=int_collection_label,
bundle=bndl)
output_ds = doc.granule("hysds:%s" % get_uuid(prod_url),
None, [prod_url],
instrument_ids.keys(),
int_collection,
int_level,
int_version,
label=id,
bundle=bndl)
# runtime context
rt_ctx_id = "hysds:runtimeContext-sentinel_ifg-%s" % project
doc.runtimeContext(rt_ctx_id, [project], label=project, bundle=bndl)
# create process
doc.processStep("hysds:%s" % get_uuid(job_id),
fake_time,
fake_time, [software],
None,
rt_ctx_id,
input_ids.keys(), [output_ds.identifier],
label=job_id,
bundle=bndl,
prov_type="hysds:create_interferogram")
# write
with open(prov_file, 'w') as f:
json.dump(json.loads(doc.serialize()), f, indent=2, sort_keys=True)
def create_prov_es_json(id, netsel_file, jobdesc_file, project, aria_dem_xml,
aria_dem_file, prod_dir, work_dir, prov_file):
"""Create provenance JSON file."""
# put in fake start/end times so that prov:used and prov:generated
# are properly created by the prov lib
fake_time = datetime.utcnow().isoformat() + 'Z'
job_id = "create_interferogram-%s" % fake_time
bundle_id = "bundle-create_interferogram-%s" % fake_time
# create PROV-ES doc
doc = ProvEsDocument()
#bndl = doc.bundle("hysds:%s" % get_uuid(bundle_id))
bndl = None
# input and output identifiers
input_ids = {}
platform_ids = {}
instrument_ids = {}
# full url paths
work_url = "file://%s%s" % (socket.getfqdn(), work_dir)
prod_url = "%s/%s" % (work_url, prod_dir)
# add network selector file
#netsel_ent = bndl.entity("hysds:%s" % get_uuid("%s/%s" % (work_url, netsel_file)),
netsel_ent = doc.file("hysds:%s" % get_uuid("%s/%s" % (work_url, netsel_file)),
["%s/%s" % (work_url, netsel_file)],
label=os.path.basename(netsel_file))
input_ids[netsel_ent.identifier] = True
# add job description file
#jobdesc_ent = bndl.entity("hysds:%s" % get_uuid("%s/%s" % (work_url, jobdesc_file)),
jobdesc_ent = doc.file("hysds:%s" % get_uuid("%s/%s" % (work_url, jobdesc_file)),
["%s/%s" % (work_url, jobdesc_file)],
label=os.path.basename(jobdesc_file))
input_ids[jobdesc_ent.identifier] = True
# get list of CSK urls
level = "L0"
version = "v1.0"
sensor = "eos:SAR"
sensor_title = "Synthetic-aperture radar (SAR)"
gov_org = "eos:ASI"
gov_org_title = "Agenzia Spaziale Italiana"
doc.governingOrganization(gov_org, label=gov_org_title, bundle=bndl)
instrument = ""
for i, url in enumerate(get_netsel_urls(netsel_file)):
match = PLATFORM_RE.search(url)
if not match: continue
pf = match.group(1)
platform = "eos:%s" % pf
platform_title = "COSMO-SkyMed Satellite %s" % pf[-1]
instrument = "eos:%s-SAR" % pf
instrument_title = "%s-SAR" % pf
input_ds = doc.product("hysds:%s" % get_uuid(url), None,
[url], [instrument], None, level, version,
label=os.path.basename(url), bundle=bndl)
input_ids[input_ds.identifier] = True
if platform not in platform_ids:
doc.platform(platform, [instrument], label=platform_title,
bundle=bndl)
platform_ids[platform] = True
if instrument not in instrument_ids:
doc.instrument(instrument, platform, [sensor], [gov_org],
label=instrument_title, bundle=bndl)
doc.sensor(sensor, instrument, label=sensor_title, bundle=bndl)
instrument_ids[instrument] = True
# add dem xml, file and related provenance
srtm_platform = "eos:SpaceShuttleEndeavour"
srtm_platform_title = "USS Endeavour"
srtm_instrument = "eos:SRTM"
srtm_instrument_title = "Shuttle Radar Topography Mission (SRTM)"
srtm_sensor = "eos:radar"
srtm_sensor_title = "radar"
srtm_gov_org = "eos:JPL"
srtm_gov_org_title = "Jet Propulsion Laboratory"
doc.governingOrganization(srtm_gov_org, label=srtm_gov_org_title, bundle=bndl)
#dem_xml_ent = bndl.entity("hysds:%s" % get_uuid("%s/%s" % (work_url, aria_dem_xml)),
dem_xml_ent = doc.file("hysds:%s" % get_uuid("%s/%s" % (work_url, aria_dem_xml)),
["%s/%s" % (work_url, aria_dem_xml)],
label=os.path.basename(aria_dem_xml))
input_ids[dem_xml_ent.identifier] = True
#dem_file_ent = bndl.entity("hysds:%s" % get_uuid("%s/%s" % (work_url, aria_dem_file)),
dem_file_ent = doc.file("hysds:%s" % get_uuid("%s/%s" % (work_url, aria_dem_file)),
["%s/%s" % (work_url, aria_dem_file)],
label=os.path.basename(aria_dem_file))
input_ids[dem_file_ent.identifier] = True
doc.platform(srtm_platform, [srtm_instrument], label=srtm_platform_title,
bundle=bndl)
doc.instrument(srtm_instrument, srtm_platform, [srtm_sensor], [srtm_gov_org],
label=srtm_instrument_title, bundle=bndl)
doc.sensor(srtm_sensor, srtm_instrument, label=srtm_sensor_title, bundle=bndl)
instrument_ids[srtm_instrument] = True
# software and algorithm
algorithm = "eos:interferogram_generation"
software_version = "2.0.0_201604"
software_title = "InSAR SCE (InSAR Scientific Computing Environment) v%s" % software_version
software = "eos:ISCE-%s" % software_version
software_location = "https://winsar.unavco.org/isce.html"
doc.software(software, [algorithm], software_version, label=software_title,
location=software_location, bundle=bndl)
# output
int_level = "L2"
int_version = "v1.0"
int_collection = "eos:CSK-interferograms-%s" % int_version
int_collection_shortname = "CSK-interferograms-%s" % int_version
int_collection_label = "ISCE generated CSK interferograms %s" % int_version
int_collection_loc = "https://aria-dav.jpl.nasa.gov/repository/products/interferogram/%s" % int_version
doc.collection(int_collection, None, int_collection_shortname,
int_collection_label, [int_collection_loc],
instrument_ids.keys(), int_level, int_version,
label=int_collection_label, bundle=bndl)
output_ds = doc.granule("hysds:%s" % get_uuid(prod_url), None, [prod_url],
instrument_ids.keys(), int_collection, int_level,
int_version, label=id, bundle=bndl)
# runtime context
rt_ctx_id = "hysds:runtimeContext-ariamh-%s" % project
doc.runtimeContext(rt_ctx_id, [project], label=project, bundle=bndl)
# create process
doc.processStep("hysds:%s" % get_uuid(job_id), fake_time, fake_time,
[software], None, rt_ctx_id, input_ids.keys(),
[output_ds.identifier], label=job_id, bundle=bndl,
prov_type="hysds:create_interferogram")
# write
with open(prov_file, 'w') as f:
json.dump(json.loads(doc.serialize()), f, indent=2, sort_keys=True)
