def log_prov_es(job, prov_es_info, prov_es_file):
"""Log PROV-ES document. Create temp PROV-ES document to populate
attributes that only the worker has access to (e.g. PID)."""
# create PROV-ES doc to generate attributes that only verdi know
ps_id = "hysds:%s" % get_uuid(job['job_id'])
bundle_id = "hysds:%s" % get_uuid('bundle-%s' % job['job_id'])
doc = ProvEsDocument()
# get bundle
#bndl = doc.bundle(bundle_id)
bndl = None
# create sofware agent
sa_label = "hysds:pge_wrapper/%s/%d/%s" % (job['job_info']['execute_node'],
job['job_info']['pid'],
datetime.utcnow().isoformat())
sa_id = "hysds:%s" % get_uuid(sa_label)
doc.softwareAgent(sa_id,
str(job['job_info']['pid']),
job['job_info']['execute_node'],
role=job.get('username', None),
label=sa_label,
bundle=bndl)
# create processStep
doc.processStep(ps_id,
job['job_info']['cmd_start'],
job['job_info']['cmd_end'], [],
sa_id,
None, [], [],
bundle=bndl,
prov_type="hysds:%s" % job['type'])
# get json
pd = json.loads(doc.serialize())
# update software agent and process step
if 'bundle' in prov_es_info:
if len(prov_es_info['bundle']) == 1:
bundle_id_orig = list(prov_es_info['bundle'].keys())[0]
# update software agent
prov_es_info['bundle'][bundle_id_orig].setdefault(
'agent', {}).update(pd['bundle'][bundle_id]['agent'])
# update wasAssociatedWith
prov_es_info['bundle'][bundle_id_orig].setdefault(
'wasAssociatedWith',
{}).update(pd['bundle'][bundle_id]['wasAssociatedWith'])
# update activity
if 'activity' in prov_es_info['bundle'][bundle_id_orig]:
if len(prov_es_info['bundle'][bundle_id_orig]
['activity']) == 1:
ps_id_orig = list(prov_es_info['bundle'][bundle_id_orig]
['activity'].keys())[0]
prov_es_info['bundle'][bundle_id_orig]['activity'][
ps_id_orig]['prov:startTime'] = pd['bundle'][
bundle_id]['activity'][ps_id]['prov:startTime']
prov_es_info['bundle'][bundle_id_orig]['activity'][
ps_id_orig]['prov:endTime'] = pd['bundle'][bundle_id][
'activity'][ps_id]['prov:endTime']
prov_es_info['bundle'][bundle_id_orig]['activity'][
ps_id_orig]['hysds:job_id'] = job['job_id']
prov_es_info['bundle'][bundle_id_orig]['activity'][
ps_id_orig]['hysds:job_type'] = job['type']
prov_es_info['bundle'][bundle_id_orig]['activity'][
ps_id_orig]['hysds:job_url'] = job['job_info'][
'job_url']
prov_es_info['bundle'][bundle_id_orig]['activity'][
ps_id_orig]['hysds:mozart_url'] = app.conf.MOZART_URL
if 'prov:type' not in prov_es_info['bundle'][
bundle_id_orig]['activity'][ps_id_orig]:
prov_es_info['bundle'][bundle_id_orig]['activity'][
ps_id_orig]['prov:type'] = pd['bundle'][bundle_id][
'activity'][ps_id]['prov:type']
# update wasAssociatedWith activity ids
for waw_id in prov_es_info['bundle'][bundle_id_orig][
'wasAssociatedWith']:
if prov_es_info['bundle'][bundle_id_orig][
'wasAssociatedWith'][waw_id][
'prov:activity'] == ps_id:
prov_es_info['bundle'][bundle_id_orig][
'wasAssociatedWith'][waw_id][
'prov:activity'] = ps_id_orig
else:
prov_es_info['bundle'][bundle_id_orig]['activity'].update(
pd['bundle'][bundle_id]['activity'])
else:
prov_es_info['bundle'][bundle_id_orig]['activity'] = pd[
'bundle'][bundle_id]['activity']
else:
# update software agent
prov_es_info.setdefault('agent', {}).update(pd['agent'])
# update wasAssociatedWith
prov_es_info.setdefault('wasAssociatedWith',
{}).update(pd['wasAssociatedWith'])
# update process step
if 'activity' in prov_es_info:
if len(prov_es_info['activity']) == 1:
ps_id_orig = list(prov_es_info['activity'].keys())[0]
prov_es_info['activity'][ps_id_orig]['prov:startTime'] = pd[
'activity'][ps_id]['prov:startTime']
prov_es_info['activity'][ps_id_orig]['prov:endTime'] = pd[
'activity'][ps_id]['prov:endTime']
prov_es_info['activity'][ps_id_orig]['hysds:job_id'] = job[
'job_id']
prov_es_info['activity'][ps_id_orig]['hysds:job_type'] = job[
'type']
prov_es_info['activity'][ps_id_orig]['hysds:job_url'] = job[
'job_info']['job_url']
prov_es_info['activity'][ps_id_orig][
'hysds:mozart_url'] = app.conf.MOZART_URL
if 'prov:type' not in prov_es_info['activity'][ps_id_orig]:
prov_es_info['activity'][ps_id_orig]['prov:type'] = pd[
'activity'][ps_id]['prov:type']
# update wasAssociatedWith activity ids
for waw_id in prov_es_info['wasAssociatedWith']:
if prov_es_info['wasAssociatedWith'][waw_id][
'prov:activity'] == ps_id:
prov_es_info['wasAssociatedWith'][waw_id][
'prov:activity'] = ps_id_orig
else:
prov_es_info['activity'].update(pd['activity'])
else:
prov_es_info['activity'] = pd['activity']
# write prov
with open(prov_es_file, 'w') as f:
json.dump(prov_es_info, f, indent=2)
def log_publish_prov_es(prov_es_info, prov_es_file, prod_path, pub_urls,
prod_metrics, objectid):
"""Log publish step in PROV-ES document."""
# create PROV-ES doc
doc = ProvEsDocument(namespaces=prov_es_info['prefix'])
# get bundle
#bndl = doc.bundle(bundle_id)
bndl = None
# add input entity
execute_node = socket.getfqdn()
prod_url = "file://%s%s" % (execute_node, prod_path)
input_id = "hysds:%s" % get_uuid(prod_url)
input_ent = doc.granule(input_id,
None, [prod_url], [],
None,
None,
None,
label=os.path.basename(prod_url),
bundle=bndl)
# add output entity
output_id = "hysds:%s" % get_uuid(pub_urls[0])
output_ent = doc.product(output_id,
None, [pub_urls[0]], [],
None,
None,
None,
label=objectid,
bundle=bndl)
# software and algorithm
algorithm = "eos:product_publishing"
software_version = hysds.__version__
software_title = "%s v%s" % (hysds.__description__, software_version)
software = "eos:HySDS-%s" % software_version
software_location = hysds.__url__
doc.software(software, [algorithm],
software_version,
label=software_title,
location=software_location,
bundle=bndl)
# create sofware agent
pid = os.getpid()
sa_label = "hysds:publish_dataset/%s/%d/%s" % (execute_node, pid,
prod_metrics['time_start'])
sa_id = "hysds:%s" % get_uuid(sa_label)
doc.softwareAgent(sa_id,
str(pid),
execute_node,
role="invoked",
label=sa_label,
bundle=bndl)
# create processStep
job_id = "publish_dataset-%s" % os.path.basename(prod_path)
doc.processStep("hysds:%s" % get_uuid(job_id),
prod_metrics['time_start'],
prod_metrics['time_end'], [software],
sa_id,
None, [input_id], [output_id],
label=job_id,
bundle=bndl,
prov_type="hysds:publish_dataset")
# get json
pd = json.loads(doc.serialize())
# update input entity
orig_ent = prov_es_info.get('entity', {}).get(input_id, {})
pd['entity'][input_id].update(orig_ent)
# update output entity
for attr in orig_ent:
if attr in ('prov:location', 'prov:label', 'prov:type'):
continue
pd['entity'][output_id][attr] = orig_ent[attr]
# write prov
with open(prov_es_file, 'w') as f:
json.dump(pd, f, indent=2)
def create_prov_es_json(id, project, master_orbit_file, slave_orbit_file,
aria_dem_xml, aria_dem_file, work_dir, prov_file):
"""Create provenance JSON file."""
# get abs paths
work_dir = os.path.abspath(work_dir)
prod_dir = os.path.join(work_dir, id)
# get context
ctx_file = os.path.join(prod_dir, "%s.context.json" % id)
with open(ctx_file) as f:
context = json.load(f)
# put in fake start/end times so that prov:used and prov:generated
# are properly created by the prov lib
fake_time = datetime.utcnow().isoformat() + 'Z'
job_id = "create_interferogram-%s" % fake_time
bundle_id = "bundle-create_interferogram-%s" % fake_time
# create PROV-ES doc
doc = ProvEsDocument()
#bndl = doc.bundle("hysds:%s" % get_uuid(bundle_id))
bndl = None
# input and output identifiers
input_ids = {}
platform_ids = {}
instrument_ids = {}
# full url paths
work_url = "file://%s%s" % (socket.getfqdn(), work_dir)
prod_url = "%s/%s" % (work_url, id)
# add sentinel.ini file
ini_ent = doc.file("hysds:%s" % get_uuid("%s/sentinel.ini" % work_url),
["%s/sentinel.ini" % work_url],
label="sentinel.ini")
input_ids[ini_ent.identifier] = True
# add orbit files
master_orbit_ent = doc.file(
"hysds:%s" % get_uuid("%s/%s" % (work_url, master_orbit_file)),
["%s/%s" % (work_url, master_orbit_file)],
label=os.path.basename(master_orbit_file))
input_ids[master_orbit_ent.identifier] = True
slave_orbit_ent = doc.file(
"hysds:%s" % get_uuid("%s/%s" % (work_url, slave_orbit_file)),
["%s/%s" % (work_url, slave_orbit_file)],
label=os.path.basename(slave_orbit_file))
input_ids[slave_orbit_ent.identifier] = True
# get list of S1A urls
level = "L0"
version = "v1.0"
sensor = "eos:SAR"
sensor_title = "Synthetic-aperture radar (SAR)"
gov_org = "eos:ESA"
gov_org_title = "European Space Agency"
doc.governingOrganization(gov_org, label=gov_org_title, bundle=bndl)
instrument = ""
for i, url in enumerate(
[context.get('master_zip_url', ''),
context.get('slave_zip_url', '')]):
match = PLATFORM_RE.search(url)
if not match: continue
pf = match.group(1)
platform = "eos:%s" % pf
platform_title = "Sentinel1A Satellite"
instrument = "eos:%s-SAR" % pf
instrument_title = "%s-SAR" % pf
input_ds = doc.product("hysds:%s" % get_uuid(url),
None, [url], [instrument],
None,
level,
None,
label=os.path.basename(url),
bundle=bndl)
input_ids[input_ds.identifier] = True
if platform not in platform_ids:
doc.platform(platform, [instrument],
label=platform_title,
bundle=bndl)
platform_ids[platform] = True
if instrument not in instrument_ids:
doc.instrument(instrument,
platform, [sensor], [gov_org],
label=instrument_title,
bundle=bndl)
doc.sensor(sensor, instrument, label=sensor_title, bundle=bndl)
instrument_ids[instrument] = True
# add dem xml, file and related provenance
srtm_platform = "eos:SpaceShuttleEndeavour"
srtm_platform_title = "USS Endeavour"
srtm_instrument = "eos:SRTM"
srtm_instrument_title = "Shuttle Radar Topography Mission (SRTM)"
srtm_sensor = "eos:radar"
srtm_sensor_title = "radar"
srtm_gov_org = "eos:JPL"
srtm_gov_org_title = "Jet Propulsion Laboratory"
doc.governingOrganization(srtm_gov_org,
label=srtm_gov_org_title,
bundle=bndl)
dem_xml_ent = doc.file("hysds:%s" % get_uuid("%s/%s" %
(work_url, aria_dem_xml)),
["%s/%s" % (work_url, aria_dem_xml)],
label=os.path.basename(aria_dem_xml))
input_ids[dem_xml_ent.identifier] = True
dem_file_ent = doc.file("hysds:%s" % get_uuid("%s/%s" %
(work_url, aria_dem_file)),
["%s/%s" % (work_url, aria_dem_file)],
label=os.path.basename(aria_dem_file))
input_ids[dem_file_ent.identifier] = True
doc.platform(srtm_platform, [srtm_instrument],
label=srtm_platform_title,
bundle=bndl)
doc.instrument(srtm_instrument,
srtm_platform, [srtm_sensor], [srtm_gov_org],
label=srtm_instrument_title,
bundle=bndl)
doc.sensor(srtm_sensor,
srtm_instrument,
label=srtm_sensor_title,
bundle=bndl)
instrument_ids[srtm_instrument] = True
# software and algorithm
algorithm = "eos:interferogram_generation"
software_version = "2.0.0_201604"
software_title = "InSAR SCE (InSAR Scientific Computing Environment) v%s" % software_version
software = "eos:ISCE-%s" % software_version
software_location = "https://winsar.unavco.org/isce.html"
doc.software(software, [algorithm],
software_version,
label=software_title,
location=software_location,
bundle=bndl)
# output
int_level = "L2"
int_version = "v1.0"
int_collection = "eos:S1A-interferograms-%s" % int_version
int_collection_shortname = "S1A-interferograms-%s" % int_version
int_collection_label = "ISCE generated S1A interferograms %s" % int_version
int_collection_loc = "https://aria-dst-dav.jpl.nasa.gov/products/s1a_ifg/%s" % int_version
doc.collection(int_collection,
None,
int_collection_shortname,
int_collection_label, [int_collection_loc],
instrument_ids.keys(),
int_level,
int_version,
label=int_collection_label,
bundle=bndl)
output_ds = doc.granule("hysds:%s" % get_uuid(prod_url),
None, [prod_url],
instrument_ids.keys(),
int_collection,
int_level,
int_version,
label=id,
bundle=bndl)
# runtime context
rt_ctx_id = "hysds:runtimeContext-sentinel_ifg-%s" % project
doc.runtimeContext(rt_ctx_id, [project], label=project, bundle=bndl)
# create process
doc.processStep("hysds:%s" % get_uuid(job_id),
fake_time,
fake_time, [software],
None,
rt_ctx_id,
input_ids.keys(), [output_ds.identifier],
label=job_id,
bundle=bndl,
prov_type="hysds:create_interferogram")
# write
with open(prov_file, 'w') as f:
json.dump(json.loads(doc.serialize()), f, indent=2, sort_keys=True)
def create_prov_es_json(id, url, prod_dir, prov_file):
"""Create provenance JSON file."""
# get info
csk_files = glob(os.path.join(prod_dir, "CSKS*"))
pf = "CSKS?"
dtype = "NA"
repo_dir = "?"
if len(csk_files) > 0:
match = CSK_RE.search(os.path.basename(csk_files[0]))
if match: pf, dtype = match.groups()
if dtype == "RAW":
dtype = "RAW_B"
repo_dir = "csk_rawb"
elif dtype == "SCS":
dtype = "SCS_B"
repo_dir = "csk_scsb"
platform = "eos:%s" % pf
platform_title = "COSMO-SkyMed Satellite %s" % pf[-1]
instrument = "eos:%s-SAR" % pf
instrument_title = "%s-SAR" % pf
level = "L0"
version = "v1.0"
collection = "eos:CSK-%s-%s" % (dtype, version)
collection_shortname = "CSK-%s-%s" % (dtype, version)
collection_label = "CSK %s Scenes %s" % (dtype, version)
collection_loc = "https://aria-dav.jpl.nasa.gov/repository/products/%s/%s" % (
repo_dir, version)
sensor = "eos:SAR"
sensor_title = "Synthetic-aperture radar (SAR)"
gov_org = "eos:ASI"
gov_org_title = "Agenzia Spaziale Italiana"
software_version = "2.0.0_201604"
software_title = "InSAR SCE (InSAR Scientific Computing Environment) v%s" % software_version
software = "eos:ISCE-%s" % software_version
software_location = "https://winsar.unavco.org/isce.html"
algorithm = "eos:metadata_extraction"
prod_dir = "file://%s%s" % (socket.getfqdn(), prod_dir)
# put in fake start/end times so that prov:used and prov:generated
# are properly created by the prov lib
fake_time = datetime.utcnow().isoformat() + 'Z'
job_id = "ingest-%s-%s" % (id, fake_time)
bundle_id = "bundle-ingest-%s-%s" % (id, fake_time)
doc = ProvEsDocument()
#bndl = doc.bundle("hysds:%s" % get_uuid(bundle_id))
bndl = None
input_id = "hysds:%s" % get_uuid(url)
input_ds = doc.granule(input_id,
None, [url], [instrument],
None,
level,
None,
label=os.path.basename(url),
bundle=bndl)
doc.collection(collection,
None,
collection_shortname,
collection_label, [collection_loc], [instrument],
level,
version,
label=collection_label,
bundle=bndl)
output_id = "hysds:%s" % get_uuid(prod_dir)
output_ds = doc.granule(output_id,
None, [prod_dir], [instrument],
collection,
level,
version,
label=id,
bundle=bndl)
doc.governingOrganization(gov_org, label=gov_org_title, bundle=bndl)
doc.platform(platform, [instrument], label=platform_title, bundle=bndl)
doc.instrument(instrument,
platform, [sensor], [gov_org],
label=instrument_title,
bundle=bndl)
doc.sensor(sensor, instrument, label=sensor_title, bundle=bndl)
doc.software(software, [algorithm],
software_version,
label=software_title,
location=software_location,
bundle=bndl)
doc.processStep("hysds:%s" % get_uuid(job_id),
fake_time,
fake_time, [software],
None,
None, [input_ds.identifier], [output_ds.identifier],
label=job_id,
bundle=bndl,
prov_type="hysds:ingest")
with open(prov_file, 'w') as f:
json.dump(json.loads(doc.serialize()), f, indent=2, sort_keys=True)
def log_prov_es(job, prov_es_info, prov_es_file):
"""Log PROV-ES document. Create temp PROV-ES document to populate
attributes that only the worker has access to (e.g. PID)."""
# create PROV-ES doc to generate attributes that only verdi know
ps_id = "hysds:%s" % get_uuid(job["job_id"])
bundle_id = "hysds:%s" % get_uuid("bundle-%s" % job["job_id"])
doc = ProvEsDocument()
# get bundle
# bndl = doc.bundle(bundle_id)
bndl = None
# create sofware agent
sa_label = "hysds:pge_wrapper/%s/%d/%s" % (
job["job_info"]["execute_node"],
job["job_info"]["pid"],
datetime.utcnow().isoformat(),
)
sa_id = "hysds:%s" % get_uuid(sa_label)
doc.softwareAgent(
sa_id,
str(job["job_info"]["pid"]),
job["job_info"]["execute_node"],
role=job.get("username", None),
label=sa_label,
bundle=bndl,
)
# create processStep
doc.processStep(
ps_id,
job["job_info"]["cmd_start"],
job["job_info"]["cmd_end"],
[],
sa_id,
None,
[],
[],
bundle=bndl,
prov_type="hysds:%s" % job["type"],
)
# get json
pd = json.loads(doc.serialize())
# update software agent and process step
if "bundle" in prov_es_info:
if len(prov_es_info["bundle"]) == 1:
bundle_id_orig = list(prov_es_info["bundle"].keys())[0]
# update software agent
prov_es_info["bundle"][bundle_id_orig].setdefault(
"agent", {}).update(pd["bundle"][bundle_id]["agent"])
# update wasAssociatedWith
prov_es_info["bundle"][bundle_id_orig].setdefault(
"wasAssociatedWith",
{}).update(pd["bundle"][bundle_id]["wasAssociatedWith"])
# update activity
if "activity" in prov_es_info["bundle"][bundle_id_orig]:
if len(prov_es_info["bundle"][bundle_id_orig]
["activity"]) == 1:
ps_id_orig = list(prov_es_info["bundle"][bundle_id_orig]
["activity"].keys())[0]
prov_es_info["bundle"][bundle_id_orig]["activity"][
ps_id_orig]["prov:startTime"] = pd["bundle"][
bundle_id]["activity"][ps_id]["prov:startTime"]
prov_es_info["bundle"][bundle_id_orig]["activity"][
ps_id_orig]["prov:endTime"] = pd["bundle"][bundle_id][
"activity"][ps_id]["prov:endTime"]
prov_es_info["bundle"][bundle_id_orig]["activity"][
ps_id_orig]["hysds:job_id"] = job["job_id"]
prov_es_info["bundle"][bundle_id_orig]["activity"][
ps_id_orig]["hysds:job_type"] = job["type"]
prov_es_info["bundle"][bundle_id_orig]["activity"][
ps_id_orig]["hysds:job_url"] = job["job_info"][
"job_url"]
prov_es_info["bundle"][bundle_id_orig]["activity"][
ps_id_orig]["hysds:mozart_url"] = app.conf.MOZART_URL
if ("prov:type" not in prov_es_info["bundle"]
[bundle_id_orig]["activity"][ps_id_orig]):
prov_es_info["bundle"][bundle_id_orig]["activity"][
ps_id_orig]["prov:type"] = pd["bundle"][bundle_id][
"activity"][ps_id]["prov:type"]
# update wasAssociatedWith activity ids
for waw_id in prov_es_info["bundle"][bundle_id_orig][
"wasAssociatedWith"]:
if (prov_es_info["bundle"][bundle_id_orig]
["wasAssociatedWith"][waw_id]["prov:activity"] ==
ps_id):
prov_es_info["bundle"][bundle_id_orig][
"wasAssociatedWith"][waw_id][
"prov:activity"] = ps_id_orig
else:
prov_es_info["bundle"][bundle_id_orig]["activity"].update(
pd["bundle"][bundle_id]["activity"])
else:
prov_es_info["bundle"][bundle_id_orig]["activity"] = pd[
"bundle"][bundle_id]["activity"]
else:
# update software agent
prov_es_info.setdefault("agent", {}).update(pd["agent"])
# update wasAssociatedWith
prov_es_info.setdefault("wasAssociatedWith",
{}).update(pd["wasAssociatedWith"])
# update process step
if "activity" in prov_es_info:
if len(prov_es_info["activity"]) == 1:
ps_id_orig = list(prov_es_info["activity"].keys())[0]
prov_es_info["activity"][ps_id_orig]["prov:startTime"] = pd[
"activity"][ps_id]["prov:startTime"]
prov_es_info["activity"][ps_id_orig]["prov:endTime"] = pd[
"activity"][ps_id]["prov:endTime"]
prov_es_info["activity"][ps_id_orig]["hysds:job_id"] = job[
"job_id"]
prov_es_info["activity"][ps_id_orig]["hysds:job_type"] = job[
"type"]
prov_es_info["activity"][ps_id_orig]["hysds:job_url"] = job[
"job_info"]["job_url"]
prov_es_info["activity"][ps_id_orig][
"hysds:mozart_url"] = app.conf.MOZART_URL
if "prov:type" not in prov_es_info["activity"][ps_id_orig]:
prov_es_info["activity"][ps_id_orig]["prov:type"] = pd[
"activity"][ps_id]["prov:type"]
# update wasAssociatedWith activity ids
for waw_id in prov_es_info["wasAssociatedWith"]:
if (prov_es_info["wasAssociatedWith"][waw_id]
["prov:activity"] == ps_id):
prov_es_info["wasAssociatedWith"][waw_id][
"prov:activity"] = ps_id_orig
else:
prov_es_info["activity"].update(pd["activity"])
else:
prov_es_info["activity"] = pd["activity"]
# write prov
with open(prov_es_file, "w") as f:
json.dump(prov_es_info, f, indent=2)
def create_prov_es_json(ctx_file, id, prod_dir, prov_file):
"""Create provenance JSON file."""
# get abs path
prod_dir = os.path.abspath(prod_dir)
# get context
with open(ctx_file) as f:
context = json.load(f)
# get mission char
mis_char = MISSION_RE.search(context.get('file')).group(1)
mis_char_lc = mis_char.lower()
# get input url
input_url = context.get('localize_urls', [{'url': None}])[0]['url']
# get info
s1_files = glob(os.path.join(prod_dir, "s1%s-*.tiff" % mis_char_lc))
pf = "S1%s" % mis_char
dtype = "NA"
repo_dir = "?"
if len(s1_files) > 0:
match = S1_RE.search(os.path.basename(s1_files[0]))
if match: pf, swathnum, dtype = match.groups()
if dtype == "raw":
dtype = "RAW"
repo_dir = "s1%s_raw" % mis_char_lc
elif dtype == "slc":
dtype = "SLC"
repo_dir = "s1%s_slc" % mis_char_lc
platform = "eos:%s" % pf
platform_title = "Sentinel1%s Satellite" % mis_char
instrument = "eos:%s-SAR" % pf
instrument_title = "%s-SAR" % pf
level = "L0"
version = "v1.0"
collection = "eos:S1%s-%s-%s" % (mis_char, dtype, version)
collection_shortname = "S1%s-%s-%s" % (mis_char, dtype, version)
collection_label = "S1%s %s Scenes %s" % (mis_char, dtype, version)
collection_loc = "https://aria-dst-dav.jpl.nasa.gov/repository/products/%s/%s" % (
repo_dir, version)
sensor = "eos:SAR"
sensor_title = "Synthetic-aperture radar (SAR)"
gov_org = "eos:ESA"
gov_org_title = "European Space Agency"
software_version = "2.0.0_201604"
software_title = "InSAR SCE (InSAR Scientific Computing Environment) v%s" % software_version
software = "eos:ISCE-%s" % software_version
software_location = "https://winsar.unavco.org/isce.html"
algorithm = "eos:metadata_extraction"
prod_dir = "file://%s%s" % (socket.getfqdn(), prod_dir)
# put in fake start/end times so that prov:used and prov:generated
# are properly created by the prov lib
fake_time = datetime.utcnow().isoformat() + 'Z'
job_id = "ingest-%s-%s" % (id, fake_time)
bundle_id = "bundle-ingest-%s-%s" % (id, fake_time)
doc = ProvEsDocument()
#bndl = doc.bundle("hysds:%s" % get_uuid(bundle_id))
bndl = None
input_id = "hysds:%s" % get_uuid(input_url)
input_ds = doc.granule(input_id,
None, [input_url], [instrument],
None,
level,
None,
label=os.path.basename(input_url),
bundle=bndl)
doc.collection(collection,
None,
collection_shortname,
collection_label, [collection_loc], [instrument],
level,
version,
label=collection_label,
bundle=bndl)
output_id = "hysds:%s" % get_uuid(prod_dir)
output_ds = doc.granule(output_id,
None, [prod_dir], [instrument],
collection,
level,
version,
label=id,
bundle=bndl)
doc.governingOrganization(gov_org, label=gov_org_title, bundle=bndl)
doc.platform(platform, [instrument], label=platform_title, bundle=bndl)
doc.instrument(instrument,
platform, [sensor], [gov_org],
label=instrument_title,
bundle=bndl)
doc.sensor(sensor, instrument, label=sensor_title, bundle=bndl)
doc.software(software, [algorithm],
software_version,
label=software_title,
location=software_location,
bundle=bndl)
doc.processStep("hysds:%s" % get_uuid(job_id),
fake_time,
fake_time, [software],
None,
None, [input_ds.identifier], [output_ds.identifier],
label=job_id,
bundle=bndl,
prov_type="hysds:ingest")
with open(prov_file, 'w') as f:
json.dump(json.loads(doc.serialize()), f, indent=2, sort_keys=True)
def create_prov_es_json(id, netsel_file, jobdesc_file, project, aria_dem_xml,
aria_dem_file, prod_dir, work_dir, prov_file):
"""Create provenance JSON file."""
# put in fake start/end times so that prov:used and prov:generated
# are properly created by the prov lib
fake_time = datetime.utcnow().isoformat() + 'Z'
job_id = "create_interferogram-%s" % fake_time
bundle_id = "bundle-create_interferogram-%s" % fake_time
# create PROV-ES doc
doc = ProvEsDocument()
#bndl = doc.bundle("hysds:%s" % get_uuid(bundle_id))
bndl = None
# input and output identifiers
input_ids = {}
platform_ids = {}
instrument_ids = {}
# full url paths
work_url = "file://%s%s" % (socket.getfqdn(), work_dir)
prod_url = "%s/%s" % (work_url, prod_dir)
# add network selector file
#netsel_ent = bndl.entity("hysds:%s" % get_uuid("%s/%s" % (work_url, netsel_file)),
netsel_ent = doc.file("hysds:%s" % get_uuid("%s/%s" % (work_url, netsel_file)),
["%s/%s" % (work_url, netsel_file)],
label=os.path.basename(netsel_file))
input_ids[netsel_ent.identifier] = True
# add job description file
#jobdesc_ent = bndl.entity("hysds:%s" % get_uuid("%s/%s" % (work_url, jobdesc_file)),
jobdesc_ent = doc.file("hysds:%s" % get_uuid("%s/%s" % (work_url, jobdesc_file)),
["%s/%s" % (work_url, jobdesc_file)],
label=os.path.basename(jobdesc_file))
input_ids[jobdesc_ent.identifier] = True
# get list of CSK urls
level = "L0"
version = "v1.0"
sensor = "eos:SAR"
sensor_title = "Synthetic-aperture radar (SAR)"
gov_org = "eos:ASI"
gov_org_title = "Agenzia Spaziale Italiana"
doc.governingOrganization(gov_org, label=gov_org_title, bundle=bndl)
instrument = ""
for i, url in enumerate(get_netsel_urls(netsel_file)):
match = PLATFORM_RE.search(url)
if not match: continue
pf = match.group(1)
platform = "eos:%s" % pf
platform_title = "COSMO-SkyMed Satellite %s" % pf[-1]
instrument = "eos:%s-SAR" % pf
instrument_title = "%s-SAR" % pf
input_ds = doc.product("hysds:%s" % get_uuid(url), None,
[url], [instrument], None, level, version,
label=os.path.basename(url), bundle=bndl)
input_ids[input_ds.identifier] = True
if platform not in platform_ids:
doc.platform(platform, [instrument], label=platform_title,
bundle=bndl)
platform_ids[platform] = True
if instrument not in instrument_ids:
doc.instrument(instrument, platform, [sensor], [gov_org],
label=instrument_title, bundle=bndl)
doc.sensor(sensor, instrument, label=sensor_title, bundle=bndl)
instrument_ids[instrument] = True
# add dem xml, file and related provenance
srtm_platform = "eos:SpaceShuttleEndeavour"
srtm_platform_title = "USS Endeavour"
srtm_instrument = "eos:SRTM"
srtm_instrument_title = "Shuttle Radar Topography Mission (SRTM)"
srtm_sensor = "eos:radar"
srtm_sensor_title = "radar"
srtm_gov_org = "eos:JPL"
srtm_gov_org_title = "Jet Propulsion Laboratory"
doc.governingOrganization(srtm_gov_org, label=srtm_gov_org_title, bundle=bndl)
#dem_xml_ent = bndl.entity("hysds:%s" % get_uuid("%s/%s" % (work_url, aria_dem_xml)),
dem_xml_ent = doc.file("hysds:%s" % get_uuid("%s/%s" % (work_url, aria_dem_xml)),
["%s/%s" % (work_url, aria_dem_xml)],
label=os.path.basename(aria_dem_xml))
input_ids[dem_xml_ent.identifier] = True
#dem_file_ent = bndl.entity("hysds:%s" % get_uuid("%s/%s" % (work_url, aria_dem_file)),
dem_file_ent = doc.file("hysds:%s" % get_uuid("%s/%s" % (work_url, aria_dem_file)),
["%s/%s" % (work_url, aria_dem_file)],
label=os.path.basename(aria_dem_file))
input_ids[dem_file_ent.identifier] = True
doc.platform(srtm_platform, [srtm_instrument], label=srtm_platform_title,
bundle=bndl)
doc.instrument(srtm_instrument, srtm_platform, [srtm_sensor], [srtm_gov_org],
label=srtm_instrument_title, bundle=bndl)
doc.sensor(srtm_sensor, srtm_instrument, label=srtm_sensor_title, bundle=bndl)
instrument_ids[srtm_instrument] = True
# software and algorithm
algorithm = "eos:interferogram_generation"
software_version = "2.0.0_201604"
software_title = "InSAR SCE (InSAR Scientific Computing Environment) v%s" % software_version
software = "eos:ISCE-%s" % software_version
software_location = "https://winsar.unavco.org/isce.html"
doc.software(software, [algorithm], software_version, label=software_title,
location=software_location, bundle=bndl)
# output
int_level = "L2"
int_version = "v1.0"
int_collection = "eos:CSK-interferograms-%s" % int_version
int_collection_shortname = "CSK-interferograms-%s" % int_version
int_collection_label = "ISCE generated CSK interferograms %s" % int_version
int_collection_loc = "https://aria-dav.jpl.nasa.gov/repository/products/interferogram/%s" % int_version
doc.collection(int_collection, None, int_collection_shortname,
int_collection_label, [int_collection_loc],
instrument_ids.keys(), int_level, int_version,
label=int_collection_label, bundle=bndl)
output_ds = doc.granule("hysds:%s" % get_uuid(prod_url), None, [prod_url],
instrument_ids.keys(), int_collection, int_level,
int_version, label=id, bundle=bndl)
# runtime context
rt_ctx_id = "hysds:runtimeContext-ariamh-%s" % project
doc.runtimeContext(rt_ctx_id, [project], label=project, bundle=bndl)
# create process
doc.processStep("hysds:%s" % get_uuid(job_id), fake_time, fake_time,
[software], None, rt_ctx_id, input_ids.keys(),
[output_ds.identifier], label=job_id, bundle=bndl,
prov_type="hysds:create_interferogram")
# write
with open(prov_file, 'w') as f:
json.dump(json.loads(doc.serialize()), f, indent=2, sort_keys=True)
def test_ProvEsDocument():
"""Test dataset()."""
# create doc
doc = ProvEsDocument()
# input dataset
id = "hysds:INSAR2_RAW_HI_06_HH_RA_20140922062622_20140922062629"
doi = "10.5067/ARIAMH/INSAR/Scene"
downloadURL = "https://dav.domain.com/repository/products/insar/v0.2/2014/09/22/INSAR2_RAW_HI_06_HH_RA_20140922062622_20140922062629/INSAR20140922_913686_3720875"
instrument = "eos:INSAR2-SAR"
level = "L0"
doc.dataset(id, doi, [downloadURL], [instrument], None, level)
# input DEM
dem_id = "hysds:srtm/version2_1/SRTM1/Region_01/N31W114"
dem_doi = None
dem_downloadURL = "https://dav.domain.com/repository/products/srtm/version2_1/SRTM1/Region_01/N31W114.hgt.zip"
dem_level = "L0"
doc.dataset(dem_id, dem_doi, [dem_downloadURL], [], None, dem_level)
# platform
platform = "eos:INSAR2"
doc.platform(platform, [instrument])
# second instrument/platform from same org
instrument2 = "eos:INSAR4-SAR"
platform2 = "eos:INSAR4"
doc.platform(platform2, [instrument2])
# instrument
sensor = "eos:SAR"
gov_org = "eos:ASI"
doc.instrument(instrument, platform, [sensor], [gov_org])
doc.sensor(sensor, instrument)
doc.instrument(instrument2, platform2, [sensor], [gov_org])
doc.sensor(sensor, instrument2)
# software
software = "eos:ISCE"
algorithm = "eos:interferogram_creation"
doc.software(software, [algorithm])
# document
atbd_id = "eos:interferogram_creation_atbd"
atbd_doi = "10.5067/SOME/FAKE/ATBD_DOI"
atbd_url = "http://aria.domain.com/docs/ATBD.pdf"
doc.document(atbd_id, atbd_doi, [atbd_url])
# algorithm
doc.algorithm(algorithm, [software], [atbd_id])
# output dataset
out_id = "hysds:interferogram__T22_F314-330_INSAR1_20130828-INSAR1_20130609"
out_doi = "10.5067/ARIAMH/INSAR/Interferogram"
out_accessURL = 'https://aria-search.domain.com/?source={"query":{"bool":{"must":[{"term":{"dataset":"interferogram"}},{"query_string":{"query":""interferogram__T111_F330-343_INSAR1_20140922-INSAR1_20140906"","default_operator":"OR"}}]}},"sort":[{"_timestamp":{"order":"desc"}}],"fields":["_timestamp","_source"]}'
out_downloadURL = "https://dav.domain.com/repository/products/interferograms/v0.2/2014/09/06/interferogram__T111_F330-343_INSAR1_20140922-INSAR1_20140906/2014-09-22T224943.621648"
out_level = "L1"
doc.dataset(out_id, out_doi, [out_downloadURL], [instrument], None, out_level)
# software agent
sa_id = "hysds:ariamh-worker-32.domain.com/12353"
pid = "12353"
worker_node = "ariamh-worker-32.domain.com"
doc.softwareAgent(sa_id, pid, worker_node)
# runtime context
rt_ctx_id = "hysds:runtime_context"
doc.runtimeContext(rt_ctx_id, [downloadURL])
# process step
proc_id = "hysds:create_interferogram-INSAR20130625_673969_2940232"
start_time = datetime.utcnow()
end_time = start_time + timedelta(seconds=12233)
ps = doc.processStep(
proc_id,
start_time.isoformat() + "Z",
end_time.isoformat() + "Z",
[software],
sa_id,
rt_ctx_id,
[id, dem_id],
[out_id],
wasAssociatedWithRole="softwareAgent",
)
print doc.serialize(indent=2)
def test_ProvEsDocument():
"""Test dataset()."""
# create doc
doc = ProvEsDocument()
# input dataset
id = "hysds:INSAR2_RAW_HI_06_HH_RA_20140922062622_20140922062629"
doi = "10.5067/ARIAMH/INSAR/Scene"
downloadURL = 'https://dav.domain.com/repository/products/insar/v0.2/2014/09/22/INSAR2_RAW_HI_06_HH_RA_20140922062622_20140922062629/INSAR20140922_913686_3720875'
instrument = "eos:INSAR2-SAR"
level = "L0"
doc.dataset(id, doi, [downloadURL], [instrument], None, level)
# input DEM
dem_id = "hysds:srtm/version2_1/SRTM1/Region_01/N31W114"
dem_doi = None
dem_downloadURL = 'https://dav.domain.com/repository/products/srtm/version2_1/SRTM1/Region_01/N31W114.hgt.zip'
dem_level = "L0"
doc.dataset(dem_id, dem_doi, [dem_downloadURL], [], None, dem_level)
# platform
platform = "eos:INSAR2"
doc.platform(platform, [instrument])
# second instrument/platform from same org
instrument2 = "eos:INSAR4-SAR"
platform2 = "eos:INSAR4"
doc.platform(platform2, [instrument2])
# instrument
sensor = "eos:SAR"
gov_org = "eos:ASI"
doc.instrument(instrument, platform, [sensor], [gov_org])
doc.sensor(sensor, instrument)
doc.instrument(instrument2, platform2, [sensor], [gov_org])
doc.sensor(sensor, instrument2)
# software
software = "eos:ISCE"
algorithm = "eos:interferogram_creation"
doc.software(software, [algorithm])
# document
atbd_id = "eos:interferogram_creation_atbd"
atbd_doi = "10.5067/SOME/FAKE/ATBD_DOI"
atbd_url = "http://aria.domain.com/docs/ATBD.pdf"
doc.document(atbd_id, atbd_doi, [atbd_url])
# algorithm
doc.algorithm(algorithm, [software], [atbd_id])
# output dataset
out_id = "hysds:interferogram__T22_F314-330_INSAR1_20130828-INSAR1_20130609"
out_doi = "10.5067/ARIAMH/INSAR/Interferogram"
out_accessURL = 'https://aria-search.domain.com/?source={"query":{"bool":{"must":[{"term":{"dataset":"interferogram"}},{"query_string":{"query":"\"interferogram__T111_F330-343_INSAR1_20140922-INSAR1_20140906\"","default_operator":"OR"}}]}},"sort":[{"_timestamp":{"order":"desc"}}],"fields":["_timestamp","_source"]}'
out_downloadURL = 'https://dav.domain.com/repository/products/interferograms/v0.2/2014/09/06/interferogram__T111_F330-343_INSAR1_20140922-INSAR1_20140906/2014-09-22T224943.621648'
out_level = "L1"
doc.dataset(out_id, out_doi, [out_downloadURL], [instrument], None,
out_level)
# software agent
sa_id = "hysds:ariamh-worker-32.domain.com/12353"
pid = "12353"
worker_node = "ariamh-worker-32.domain.com"
doc.softwareAgent(sa_id, pid, worker_node)
# runtime context
rt_ctx_id = "hysds:runtime_context"
doc.runtimeContext(rt_ctx_id, [downloadURL])
# process step
proc_id = "hysds:create_interferogram-INSAR20130625_673969_2940232"
start_time = datetime.utcnow()
end_time = start_time + timedelta(seconds=12233)
ps = doc.processStep(proc_id,
start_time.isoformat() + 'Z',
end_time.isoformat() + 'Z', [software],
sa_id,
rt_ctx_id, [id, dem_id], [out_id],
wasAssociatedWithRole="softwareAgent")
print(doc.serialize(indent=2))
