# default to latest Hail build if none specified through metadata if not hash: hash = Popen([ 'gsutil', 'cat', 'gs://hail-common/latest-hash-spark{}.txt'.format(spark) ], stdout=PIPE, stderr=PIPE).communicate()[0].strip() # Hail jar jar = Popen('/usr/share/google/get_metadata_value attributes/JAR', shell=True, stdout=PIPE).communicate()[0].strip() if jar: hail_jar = jar.rsplit('/')[-1] jar_path = jar else: hail_jar = 'hail-hail-is-master-all-spark{0}-{1}.jar'.format( spark, hash) jar_path = 'gs://hail-common/' + hail_jar # CUSTOM HAIL JAR hail_jar = "hail-all-spark.jar" jar_path = "gs://seqr-hail/hail-jar/hail-all-spark.jar" # Hail zip zip = Popen('/usr/share/google/get_metadata_value attributes/ZIP', shell=True, stdout=PIPE).communicate()[0].strip() if zip:
# get Hail hash and Spark version to use for Jupyter notebook, if set through cluster startup metadata spark = Popen('/usr/share/google/get_metadata_value attributes/SPARK', shell=True, stdout=PIPE).communicate()[0].strip() hash = Popen('/usr/share/google/get_metadata_value attributes/HASH', shell=True, stdout=PIPE).communicate()[0].strip() # default to Spark 2.0.2 if not otherwise specified through metadata if not spark: spark = '2.0.2' # default to latest Hail build if none specified through metadata if not hash: hash = Popen(['gsutil', 'cat', 'gs://hail-common/latest-hash-spark{}.txt'.format(spark)], stdout=PIPE, stderr=PIPE).communicate()[0].strip() # Hail jar jar = Popen('/usr/share/google/get_metadata_value attributes/JAR', shell=True, stdout=PIPE).communicate()[0].strip() if jar: hail_jar = jar.rsplit('/')[-1] jar_path = jar else: hail_jar = 'hail-hail-is-master-all-spark{0}-{1}.jar'.format(spark, hash) jar_path = 'gs://hail-common/' + hail_jar # Hail zip zip = Popen('/usr/share/google/get_metadata_value attributes/ZIP', shell=True, stdout=PIPE).communicate()[0].strip() if zip: hail_zip = zip.rsplit('/')[-1] zip_path = zip else: hail_zip = 'pyhail-hail-is-master-{}.zip'.format(hash) zip_path = 'gs://hail-common/' + hail_zip # make directory for Hail and Jupyter notebook related files