def import_pkg(args):
"""Import HySDS package."""
conf = SettingsConf() # get user's SDS conf settings
# package tar file
tar_file = normpath(args.file)
if not os.path.isfile(tar_file):
logger.error("HySDS package file %s doesn't exist." % tar_file)
return 1
logger.debug("tar_file: %s" % tar_file)
# extract
outdir = os.path.dirname(tar_file)
with tarfile.open(tar_file) as tar:
export_name = tar.getnames()[0]
tar.extractall(outdir)
export_dir = os.path.join(outdir, export_name)
logger.debug("export_dir: %s" % export_dir)
# detect export dir
if not os.path.isdir(export_dir):
logger.error("Cannot find HySDS package dir %s." % export_dir)
return 1
# read in manifest
manifest_file = os.path.join(export_dir, 'manifest.json')
with open(manifest_file) as f:
manifest = json.load(f)
logger.debug("manifest: %s" %
json.dumps(manifest, indent=2, sort_keys=True))
# get code bucket
code_bucket = conf.get('CODE_BUCKET')
code_bucket_url = "s3://%s/%s" % (conf.get('S3_ENDPOINT'), code_bucket)
logger.debug("code_bucket: %s" % code_bucket)
logger.debug("code_bucket_url: %s" % code_bucket_url)
# upload container image to s3
cont_info = manifest['containers']
cont_image = os.path.join(export_dir, cont_info['url'])
cont_info['url'] = "{}/{}".format(code_bucket_url, cont_info['url'])
put(cont_image, cont_info['url'])
# index container in ES
indexed_container = mozart_es.index_document(index=CONTAINERS_INDEX,
body=cont_info,
id=cont_info['id'])
logger.debug(indexed_container)
# index job_specs in ES and upload any dependency containers
dep_images = {}
for job_spec in manifest['job_specs']:
# download dependency images
for d in job_spec.get('dependency_images', []):
if d['container_image_name'] in dep_images:
d['container_image_url'] = dep_images[
d['container_image_name']]
else:
# upload container
dep_img = os.path.join(export_dir, d['container_image_url'])
d['container_image_url'] = "%s/%s" % (code_bucket_url,
d['container_image_url'])
put(dep_img, d['container_image_url'])
dep_images[
d['container_image_name']] = d['container_image_url']
indexed_job_spec = mozart_es.index_document(index=JOB_SPECS_INDEX,
body=job_spec,
id=job_spec['id'])
logger.debug(indexed_job_spec)
# index hysds_ios to ES
for hysds_io in manifest['hysds_ios']:
component = hysds_io.get('component', 'tosca')
hysds_io_id = hysds_io['id']
if component in ('mozart', 'figaro'):
indexed_hysds_io = mozart_es.index_document(
index=HYSDS_IOS_MOZART_INDEX, body=hysds_io, id=hysds_io_id)
logger.debug(indexed_hysds_io)
else:
indexed_hysds_io = mozart_es.index_document(
index=HYSDS_IOS_GRQ_INDEX, body=hysds_io, id=hysds_io_id)
logger.debug(indexed_hysds_io)
shutil.rmtree(export_dir) # remove package dir
# create displacement tile layer
vrt_prod_file = "{}.vrt".format(unw_prod_file)
dis_layer = "displacement"
cmd = "{}/create_tiles.py merged/{} {}/{} -b 2 -m prism"
check_call(cmd.format(BASE_PATH, vrt_prod_file, 'tiles', dis_layer),
shell=True)
# create amplitude tile layer
amp_layer = "amplitude"
cmd = "{}/create_tiles.py merged/{} {}/{} -b 1 -m gray --clim_min 10 --clim_max_pct 80"
check_call(cmd.format(BASE_PATH, vrt_prod_file, 'tiles', amp_layer),
shell=True)
# upload tiles
put("tiles", "{}/tiles".format(prod_url))
# upsert new document
new_doc = {
"doc": {
"metadata": {
"tiles": True,
"tile_layers": [amp_layer, dis_layer]
}
},
"doc_as_upsert": True
}
r = requests.post('%s/%s/%s/%s/_update' %
(es_url, src, doc_type, hit['_id']),
data=json.dumps(new_doc))
result = r.json()
def import_pkg(args):
"""Import HySDS package."""
# get user's SDS conf settings
conf = SettingsConf()
# package tar file
tar_file = normpath(args.file)
if not os.path.isfile(tar_file):
logger.error("HySDS package file {} doesn't exist.".format(tar_file))
return 1
logger.debug("tar_file: {}".format(tar_file))
# extract
outdir = os.path.dirname(tar_file)
with tarfile.open(tar_file) as tar:
export_name = tar.getnames()[0]
tar.extractall(outdir)
export_dir = os.path.join(outdir, export_name)
logger.debug("export_dir: {}".format(export_dir))
# detect export dir
if not os.path.isdir(export_dir):
logger.error("Cannot find HySDS package dir {}.".format(export_dir))
return 1
# read in manifest
manifest_file = os.path.join(export_dir, 'manifest.json')
with open(manifest_file) as f:
manifest = json.load(f)
logger.debug("manifest: {}".format(
json.dumps(manifest, indent=2, sort_keys=True)))
# get code bucket
code_bucket = conf.get('CODE_BUCKET')
code_bucket_url = "s3://{}/{}".format(conf.get('S3_ENDPOINT'), code_bucket)
logger.debug("code_bucket: {}".format(code_bucket))
logger.debug("code_bucket_url: {}".format(code_bucket_url))
# get ES endpoints
mozart_es_url = "http://{}:9200".format(conf.get('MOZART_ES_PVT_IP'))
grq_es_url = "http://{}:9200".format(conf.get('GRQ_ES_PVT_IP'))
# upload container image and index container in ES
cont_info = manifest['containers']
cont_image = os.path.join(export_dir, cont_info['url'])
cont_info['url'] = "{}/{}".format(code_bucket_url, cont_info['url'])
put(cont_image, cont_info['url'])
r = requests.put("{}/containers/container/{}".format(
mozart_es_url, cont_info['id']),
data=json.dumps(cont_info))
r.raise_for_status()
logger.debug(r.json())
# index job_specs in ES and upload any dependency containers
dep_images = {}
for job_spec in manifest['job_specs']:
# download dependency images
for d in job_spec.get('dependency_images', []):
if d['container_image_name'] in dep_images:
d['container_image_url'] = dep_images[
d['container_image_name']]
else:
# upload container
dep_img = os.path.join(export_dir, d['container_image_url'])
d['container_image_url'] = "{}/{}".format(
code_bucket_url, d['container_image_url'])
put(dep_img, d['container_image_url'])
dep_images[
d['container_image_name']] = d['container_image_url']
r = requests.put("{}/job_specs/job_spec/{}".format(
mozart_es_url, job_spec['id']),
data=json.dumps(job_spec))
r.raise_for_status()
logger.debug(r.json())
# index hysds_ios in ES
for hysds_io in manifest['hysds_ios']:
component = hysds_io.get('component', 'tosca')
es_url = mozart_es_url if component == 'mozart' else grq_es_url
r = requests.put("{}/hysds_ios/hysds_io/{}".format(
es_url, hysds_io['id']),
data=json.dumps(hysds_io))
r.raise_for_status()
logger.debug(r.json())
# remove package dir
shutil.rmtree(export_dir)