def setUpTempProjects(thing):
thing.old_workspace_id = dxpy.WORKSPACE_ID
thing.proj_id = dxpy.api.project_new({
'name': 'azure-test-project',
'region': testutil.TEST_AZURE
})['id']
dxpy.set_workspace_id(thing.proj_id)
def setUp(self):
os.environ['IFS'] = IFS
os.environ['_ARGCOMPLETE'] = '1'
os.environ['_DX_ARC_DEBUG'] = '1'
os.environ['COMP_WORDBREAKS'] = '"\'@><=;|&(:'
os.environ['DX_PROJECT_CONTEXT_ID'] = self.project_id
dxpy.set_workspace_id(self.project_id)
def setUp(self):
os.environ["IFS"] = IFS
os.environ["_ARGCOMPLETE"] = "1"
os.environ["_DX_ARC_DEBUG"] = "1"
os.environ["COMP_WORDBREAKS"] = "\"'@><=;|&(:"
os.environ["DX_PROJECT_CONTEXT_ID"] = self.project_id
dxpy.set_workspace_id(self.project_id)
def setUp(self):
os.environ['IFS'] = IFS
os.environ['_ARGCOMPLETE'] = '1'
os.environ['_DX_ARC_DEBUG'] = '1'
os.environ['COMP_WORDBREAKS'] = '"\'@><=;|&(:'
os.environ['DX_PROJECT_CONTEXT_ID'] = self.project_id
dxpy.set_workspace_id(self.project_id)
def run_bwa_mem(sample, fastq_dict, mapper_app_dxid, ref_genome_index,
project_id):
'''
Description: Maps sample fastq files to a reference genome
Args:
sample (dict) - sample[<barcode>] = [<fastq files>]
mapper (dxid)
ref_genome (dxid)
'''
## Stock DNAnexus BWA-MEM app
#mapper_app_name = 'bwa_mem_fastq_read_mapper'
#mapper_app_version = '1.5.0'
#mapper_app = MapperApp(name=mapper_app_name, version=mapper_app_version) # DXApp object
dxpy.set_workspace_id(project_id)
# Create dict to store mapper app inputs
mapper_app = dxpy.DXApp(mapper_app_dxid)
mapper_input = {
'genomeindex_targz': dxpy.dxlink(ref_genome_index)
} # hg19 : file-B6qq53v2J35Qyg04XxG0000V
# Add fastq files to mapper app input dict
if len(fastq_dict) == 0:
print 'Error: No fastq files listed for sample %s' % sample
sys.exit()
elif len(fastq_dict) == 1:
mapper_input['reads_fastqgz'] = dxpy.dxlink(fastq_dict['1'])
elif len(fastq_dict) == 2:
mapper_input['reads_fastqgz'] = dxpy.dxlink(fastq_dict['1'])
mapper_input['reads2_fastqgz'] = dxpy.dxlink(fastq_dict['2'])
else:
print 'Error: More than 2 fastq files passed for mapping sample %s' % sample
sys.exit()
print mapper_input
mapper_job = mapper_app.run(mapper_input)
mapper_output = {
"BAM": {
"job": mapper_job.get_id(),
"field": "sorted_bam"
},
"BAI": {
"job": mapper_job.get_id(),
"field": "sorted_bai"
}
}
return mapper_output
def _sync_dxpy_state(self):
dxpy.set_api_server_info(host=environ.get("DX_APISERVER_HOST", None),
port=environ.get("DX_APISERVER_PORT", None),
protocol=environ.get("DX_APISERVER_PROTOCOL", None))
if "DX_SECURITY_CONTEXT" in environ:
dxpy.set_security_context(json.loads(environ["DX_SECURITY_CONTEXT"]))
if "DX_JOB_ID" in environ:
dxpy.set_job_id(environ["DX_JOB_ID"])
dxpy.set_workspace_id(environ.get("DX_WORKSPACE_ID"))
else:
dxpy.set_job_id(None)
dxpy.set_workspace_id(environ.get("DX_PROJECT_CONTEXT_ID"))
dxpy.set_project_context(environ.get("DX_PROJECT_CONTEXT_ID"))
def test_mapping():
dxpy.set_workspace_id('project-BpBjyqQ0Jk0Xv2B11Q8P6X59')
applet = dxpy.find_one_data_object(
name='bwa_mem_fastq_read_mapper',
classname='applet',
return_handler=True,
zero_ok=False,
project='project-B406G0x2fz2B3GVk65200003')
applet.run({
'genomeindex_targz':
dxpy.dxlink('file-B6qq53v2J35Qyg04XxG0000V'),
'reads_fastqgz':
dxpy.dxlink('file-BpBjzFQ0Jk0Xk73YqQgJKg9Z'),
'reads2_fastqgz':
dxpy.dxlink('file-BpBk0400Jk0Xk73YqQgJKg9f')
})
def _sync_dxpy_state(self):
dxpy.set_api_server_info(host=environ.get("DX_APISERVER_HOST", None),
port=environ.get("DX_APISERVER_PORT", None),
protocol=environ.get("DX_APISERVER_PROTOCOL", None))
if "DX_SECURITY_CONTEXT" in environ:
dxpy.set_security_context(json.loads(environ["DX_SECURITY_CONTEXT"]))
if "DX_JOB_ID" in environ:
dxpy.set_job_id(environ["DX_JOB_ID"])
dxpy.set_workspace_id(environ.get("DX_WORKSPACE_ID"))
else:
dxpy.set_job_id(None)
dxpy.set_workspace_id(environ.get("DX_PROJECT_CONTEXT_ID"))
dxpy.set_project_context(environ.get("DX_PROJECT_CONTEXT_ID"))
def main(token):
# Configure dxpy authentication
dxpy.set_security_context({'auth_token_type': 'Bearer', 'auth_token': token})
# Resolve FACTORY_PROJECT by ID
proj = dxpy.DXProject(FACTORY_PROJECT)
print 'Resolved project:', proj.describe()['name'], proj.get_id()
# Set FACTORY_PROJECT as the workspace for subsequent operations
# (sort of like the current working directory)
dxpy.set_workspace_id(FACTORY_PROJECT)
# Resolve the workflow by name. (Could also store ID like the project)
wf = list(dxpy.search.find_data_objects(classname="workflow", name="RNA-seq pipeline",
return_handler=True))[0]
print 'Resolved workflow:', wf.describe()['name'], wf.get_id()
# TODO: Stage the inputs. Here we find them in the IN folder
left_reads = list(dxpy.search.find_data_objects(classname="file", name="ENCFF001JPX.1k.fastq.gz",
folder="/IN", return_handler=True))[0]
print 'Resolved left reads:', left_reads.describe()['name'], left_reads.get_id()
right_reads = list(dxpy.search.find_data_objects(classname="file", name="ENCFF001JQB.1k.fastq.gz",
folder="/IN", return_handler=True))[0]
print 'Resolved right reads:', right_reads.describe()['name'], right_reads.get_id()
# Launch the workflow
analysis = wf.run({'0.fastqs': [dxpy.dxlink(left_reads.get_id())],
'0.fastq_pairs': [dxpy.dxlink(right_reads.get_id())]})
print 'Launched analysis:', analysis.get_id()
print 'Analysis state:', analysis.describe()['state']
# TODO: Poll for (or come back when) analysis state 'done' or 'failed'.
# Handle any failures.
# Cooking-show-style substitution with completed analysis
analysis = dxpy.DXAnalysis(COMPLETED_ANALYSIS)
print 'Analysis state:', analysis.describe()['state']
# Enumerate outputs
print 'Analysis outputs:'
for one_output_name, one_output_link in analysis.describe()['output'].iteritems():
one_output = dxpy.get_handler(one_output_link) # one_output : dxpy.DXFile
one_file_name = one_output.describe()['name']
one_file_url, _ = one_output.get_download_url(preauthenticated=True, filename=one_file_name)
print one_file_name, one_file_url
def main():
import argparse
parser = argparse.ArgumentParser(
description="Module to grab QC metrics from ENCODE pipelines",
epilog=EPILOG,
formatter_class=argparse.RawDescriptionHelpFormatter,
)
group = parser.add_mutually_exclusive_group(required=True)
group.add_argument('--encode-url', help="ENCODE assay search URL")
group.add_argument('--dx-file', help="Text file with DNA NEXUS IDs")
args = parser.parse_args()
try:
dxpy.set_workspace_id(data['dx_project'])
except DXError:
print "Please enter a valid project ID in auth.json"
else:
if args.encode_url:
if 'assay_term_name' not in args.encode_url:
print "Please select exactly one assay type."
sys.exit(1)
for exp in get_assay_JSON(args.encode_url):
exp_details = {
'assay_term_name': exp['assay_term_name'],
'assay_term_id': exp['assay_term_id'],
'accession': exp['accession']
}
print "Started processing the experimet - %s" % \
exp['accession']
for f in exp['original_files']:
f_json = get_encode_object(f)
if 'notes' in f_json:
print " Processing file - %s" % f_json.get('accession')
status = load_metadata(f_json, exp_details)
if not status:
print " Update failed - %s" % f_json['accession']
else:
print " ...Done"
print "Finished processing the experiment - %s" % \
exp['accession']
elif args.dx_file:
# Have to include the code to make the script work with
# DNA Nexus analyis IDs
pass
def run_analysis(self):
#pdb.set_trace()
self.record = dxpy.DXRecord(dxid=self.record_id, project=self.dashboard_project_id)
properties = self.record.get_properties()
if not 'analysis_started' in properties.keys():
print 'Warning: Could not determine whether or not analysis had been started'
dxpy.set_workspace_id(dxid=self.project_id)
self.workflow_object = dxpy.DXWorkflow(
dxid=self.workflow_id,
project=self.workflow_project_id)
print 'Launching workflow %s with input: %s' % (
self.workflow_object.describe()['id'],
self.analysis_input)
self.workflow_object.run(
workflow_input=self.analysis_input,
project=self.project_id,
folder='/')
self.record.set_properties({'analysis_started': 'true'})
elif properties['analysis_started'] == 'true':
print 'Info: Analysis has already been started; skipping.'
pass
elif properties['analysis_started'] == 'false':
dxpy.set_workspace_id(dxid=self.project_id)
self.workflow_object = dxpy.DXWorkflow(
dxid=self.workflow_id,
project=self.workflow_project_id)
print 'Launching workflow %s with input: %s' % (
self.workflow_object.describe()['id'],
self.analysis_input)
self.workflow_object.run(
workflow_input=self.analysis_input,
project=self.project_id,
folder='/')
self.record.set_properties({'analysis_started': 'true'})
# Create new pipeline run in LIMS
if not self.develop:
if self.lane_index == 1:
param_dict = {'started': True}
json = self.connection.createpipelinerun(self.run_name, param_dict)
self.record.set_properties({'pipeline_id': str(json['id'])})
print 'Info: Created new LIMS pipeline run %s' % str(json['id'])
def test_get_handler(self):
dxpy.set_workspace_id(self.second_proj_id)
dxrecord = dxpy.new_dxrecord(project=self.proj_id)
# Simple DXLink
dxlink = {'$dnanexus_link': dxrecord.get_id()}
handler = dxpy.get_handler(dxlink)
self.assertEqual(handler.get_id(), dxrecord.get_id())
# Default project is not going to be the correct one
self.assertNotEqual(handler.get_proj_id(), self.proj_id)
# Extended DXLink
dxlink = {'$dnanexus_link': {'id': dxrecord.get_id(),
'project': self.proj_id}}
handler = dxpy.get_handler(dxlink)
self.assertEqual(handler.get_id(), dxrecord.get_id())
self.assertEqual(handler.get_proj_id(), self.proj_id)
# Handle project IDs
dxproject = dxpy.get_handler(self.proj_id)
def test_completion_with_bad_current_project(self):
os.environ['DX_PROJECT_CONTEXT_ID'] = ''
dxpy.set_workspace_id('')
self.assert_completion("dx select ", "tab-completion project\\:")
self.assert_completion("dx cd ta", "tab-completion project\\:")
def tearDownTempProjects(thing):
dxpy.api.project_destroy(thing.proj_id, {'terminateJobs': True})
dxpy.api.project_destroy(thing.second_proj_id, {'terminateJobs': True})
dxpy.set_workspace_id(thing.old_workspace_id)
def test_completion_with_bad_current_project(self):
os.environ['DX_PROJECT_CONTEXT_ID'] = ''
dxpy.set_workspace_id('')
self.assert_completion("dx select ", "tab-completion project\\:")
self.assert_completion("dx cd ta", "tab-completion project\\:")
def _build_app_remote(
mode,
src_dir,
publish=False,
destination_override=None,
version_override=None,
bill_to_override=None,
dx_toolkit_autodep="stable",
do_version_autonumbering=True,
do_try_update=True,
do_parallel_build=True,
do_check_syntax=True,
region=None,
):
if mode == "app":
builder_app = "app-tarball_app_builder"
else:
builder_app = "app-tarball_applet_builder"
temp_dir = tempfile.mkdtemp()
# TODO: this is vestigial, the "auto" setting should be removed.
if dx_toolkit_autodep == "auto":
dx_toolkit_autodep = "stable"
build_options = {"dx_toolkit_autodep": dx_toolkit_autodep}
if version_override:
build_options["version_override"] = version_override
elif do_version_autonumbering:
# If autonumbering is DISABLED, the interior run of dx-build-app
# will detect the correct version to use without our help. If it
# is ENABLED, the version suffix might depend on the state of
# the git repository. Since we'll remove the .git directory
# before uploading, we need to determine the correct version to
# use here and pass it in to the interior run of dx-build-app.
if do_version_autonumbering:
app_spec = _parse_app_spec(src_dir)
original_version = app_spec["version"]
app_describe = None
try:
app_describe = dxpy.api.app_describe(
"app-" + app_spec["name"], alias=original_version, always_retry=False
)
except dxpy.exceptions.DXAPIError as e:
if e.name == "ResourceNotFound" or (mode == "applet" and e.name == "PermissionDenied"):
pass
else:
raise e
if app_describe is not None:
if "published" in app_describe or not do_try_update:
# The version we wanted was taken; fall back to the
# autogenerated version number.
build_options["version_override"] = original_version + _get_version_suffix(
src_dir, original_version
)
# The following flags are basically passed through verbatim.
if bill_to_override:
build_options["bill_to_override"] = bill_to_override
if not do_version_autonumbering:
build_options["do_version_autonumbering"] = False
if not do_try_update:
build_options["do_try_update"] = False
if not do_parallel_build:
build_options["do_parallel_build"] = False
if not do_check_syntax:
build_options["do_check_syntax"] = False
using_temp_project_for_remote_build = False
# If building an applet, run the builder app in the destination
# project. If building an app, run the builder app in a temporary
# project.
dest_folder = None
dest_applet_name = None
if mode == "applet":
# Translate the --destination flag as follows. If --destination
# is PROJ:FOLDER/NAME,
#
# 1. Run the builder app in PROJ
# 2. Make the output folder FOLDER
# 3. Supply --destination=NAME to the interior call of dx-build-applet.
build_project_id = dxpy.WORKSPACE_ID
if destination_override:
build_project_id, dest_folder, dest_applet_name = parse_destination(destination_override)
if build_project_id is None:
parser.error(
"Can't create an applet without specifying a destination project; please use the -d/--destination flag to explicitly specify a project"
)
if dest_applet_name:
build_options["destination_override"] = "/" + dest_applet_name
elif mode == "app":
using_temp_project_for_remote_build = True
try:
if region:
build_project_id = dxpy.api.project_new(
{"name": "dx-build-app --remote temporary project", "region": region}
)["id"]
else:
build_project_id = dxpy.api.project_new({"name": "dx-build-app --remote temporary project"})["id"]
except:
err_exit()
try:
# Resolve relative paths and symlinks here so we have something
# reasonable to write in the job name below.
src_dir = os.path.realpath(src_dir)
# Show the user some progress as the tarball is being generated.
# Hopefully this will help them to understand when their tarball
# is huge (e.g. the target directory already has a whole bunch
# of binaries in it) and interrupt before uploading begins.
app_tarball_file = os.path.join(temp_dir, "app_tarball.tar.gz")
tar_subprocess = subprocess.Popen(
["tar", "-czf", "-", "--exclude", "./.git", "."], cwd=src_dir, stdout=subprocess.PIPE
)
with open(app_tarball_file, "wb") as tar_output_file:
total_num_bytes = 0
last_console_update = 0
start_time = time.time()
printed_static_message = False
# Pipe the output of tar into the output file
while True:
tar_exitcode = tar_subprocess.poll()
data = tar_subprocess.stdout.read(4 * 1024 * 1024)
if tar_exitcode is not None and len(data) == 0:
break
tar_output_file.write(data)
total_num_bytes += len(data)
current_time = time.time()
# Don't show status messages at all for very short tar
# operations (< 1.0 sec)
if current_time - last_console_update > 0.25 and current_time - start_time > 1.0:
if sys.stderr.isatty():
if last_console_update > 0:
sys.stderr.write("\r")
sys.stderr.write(
"Compressing target directory {dir}... ({kb_so_far:,} kb)".format(
dir=src_dir, kb_so_far=total_num_bytes // 1024
)
)
sys.stderr.flush()
last_console_update = current_time
elif not printed_static_message:
# Print a message (once only) when stderr is not
# going to a live console
sys.stderr.write("Compressing target directory %s..." % (src_dir,))
printed_static_message = True
if last_console_update > 0:
sys.stderr.write("\n")
if tar_exitcode != 0:
raise Exception("tar exited with non-zero exit code " + str(tar_exitcode))
dxpy.set_workspace_id(build_project_id)
remote_file = dxpy.upload_local_file(
app_tarball_file, media_type="application/gzip", wait_on_close=True, show_progress=True
)
try:
input_hash = {"input_file": dxpy.dxlink(remote_file), "build_options": build_options}
if mode == "app":
input_hash["publish"] = publish
api_options = {
"name": "Remote build of %s" % (os.path.basename(src_dir),),
"input": input_hash,
"project": build_project_id,
}
if dest_folder:
api_options["folder"] = dest_folder
app_run_result = dxpy.api.app_run(builder_app, input_params=api_options)
job_id = app_run_result["id"]
print("Started builder job %s" % (job_id,))
try:
subprocess.check_call(["dx", "watch", job_id])
except subprocess.CalledProcessError as e:
if e.returncode == 3:
# Some kind of failure to build the app. The reason
# for the failure is probably self-evident from the
# job log (and if it's not, the CalledProcessError
# is not informative anyway), so just propagate the
# return code without additional remarks.
sys.exit(3)
else:
raise e
dxpy.DXJob(job_id).wait_on_done(interval=1)
if mode == "applet":
applet_id, _ = dxpy.get_dxlink_ids(dxpy.api.job_describe(job_id)["output"]["output_applet"])
return applet_id
else:
# TODO: determine and return the app ID, to allow
# running the app if args.run is specified
return None
finally:
if not using_temp_project_for_remote_build:
dxpy.DXProject(build_project_id).remove_objects([remote_file.get_id()])
finally:
if using_temp_project_for_remote_build:
dxpy.api.project_destroy(build_project_id, {"terminateJobs": True})
shutil.rmtree(temp_dir)
def _build_app_remote(mode, src_dir, publish=False, destination_override=None,
version_override=None, bill_to_override=None, dx_toolkit_autodep="stable",
do_version_autonumbering=True, do_try_update=True, do_parallel_build=True,
do_check_syntax=True, region=None, watch=True):
if mode == 'app':
builder_app = 'app-tarball_app_builder'
else:
builder_app = 'app-tarball_applet_builder'
app_spec = _parse_app_spec(src_dir)
if app_spec['runSpec'].get('release') == '14.04':
builder_app += "_trusty"
temp_dir = tempfile.mkdtemp()
build_options = {'dx_toolkit_autodep': dx_toolkit_autodep}
if version_override:
build_options['version_override'] = version_override
elif do_version_autonumbering:
# If autonumbering is DISABLED, the interior run of dx-build-app
# will detect the correct version to use without our help. If it
# is ENABLED, the version suffix might depend on the state of
# the git repository. Since we'll remove the .git directory
# before uploading, we need to determine the correct version to
# use here and pass it in to the interior run of dx-build-app.
if do_version_autonumbering:
original_version = app_spec['version']
app_describe = None
try:
app_describe = dxpy.api.app_describe("app-" + app_spec["name"], alias=original_version, always_retry=False)
except dxpy.exceptions.DXAPIError as e:
if e.name == 'ResourceNotFound' or (mode == 'applet' and e.name == 'PermissionDenied'):
pass
else:
raise e
if app_describe is not None:
if 'published' in app_describe or not do_try_update:
# The version we wanted was taken; fall back to the
# autogenerated version number.
build_options['version_override'] = original_version + _get_version_suffix(src_dir, original_version)
# The following flags are basically passed through verbatim.
if bill_to_override:
build_options['bill_to_override'] = bill_to_override
if not do_version_autonumbering:
build_options['do_version_autonumbering'] = False
if not do_try_update:
build_options['do_try_update'] = False
if not do_parallel_build:
build_options['do_parallel_build'] = False
if not do_check_syntax:
build_options['do_check_syntax'] = False
using_temp_project_for_remote_build = False
# If building an applet, run the builder app in the destination
# project. If building an app, run the builder app in a temporary
# project.
dest_folder = None
dest_applet_name = None
if mode == "applet":
# Translate the --destination flag as follows. If --destination
# is PROJ:FOLDER/NAME,
#
# 1. Run the builder app in PROJ
# 2. Make the output folder FOLDER
# 3. Supply --destination=NAME to the interior call of dx-build-applet.
build_project_id = dxpy.WORKSPACE_ID
if destination_override:
build_project_id, dest_folder, dest_applet_name = parse_destination(destination_override)
if build_project_id is None:
parser.error("Can't create an applet without specifying a destination project; please use the -d/--destination flag to explicitly specify a project")
if dest_applet_name:
build_options['destination_override'] = '/' + dest_applet_name
elif mode == "app":
using_temp_project_for_remote_build = True
try:
project_input = {}
project_input["name"] = "dx-build-app --remote temporary project"
if bill_to_override:
project_input["billTo"] = bill_to_override
if region:
project_input["region"] = region
build_project_id = dxpy.api.project_new(project_input)["id"]
except:
err_exit()
try:
# Resolve relative paths and symlinks here so we have something
# reasonable to write in the job name below.
src_dir = os.path.realpath(src_dir)
# Show the user some progress as the tarball is being generated.
# Hopefully this will help them to understand when their tarball
# is huge (e.g. the target directory already has a whole bunch
# of binaries in it) and interrupt before uploading begins.
app_tarball_file = os.path.join(temp_dir, "app_tarball.tar.gz")
tar_subprocess = subprocess.Popen(["tar", "-czf", "-", "--exclude", "./.git", "."], cwd=src_dir, stdout=subprocess.PIPE)
with open(app_tarball_file, 'wb') as tar_output_file:
total_num_bytes = 0
last_console_update = 0
start_time = time.time()
printed_static_message = False
# Pipe the output of tar into the output file
while True:
tar_exitcode = tar_subprocess.poll()
data = tar_subprocess.stdout.read(4 * 1024 * 1024)
if tar_exitcode is not None and len(data) == 0:
break
tar_output_file.write(data)
total_num_bytes += len(data)
current_time = time.time()
# Don't show status messages at all for very short tar
# operations (< 1.0 sec)
if current_time - last_console_update > 0.25 and current_time - start_time > 1.0:
if sys.stderr.isatty():
if last_console_update > 0:
sys.stderr.write("\r")
sys.stderr.write("Compressing target directory {dir}... ({kb_so_far:,} kb)".format(dir=src_dir, kb_so_far=total_num_bytes // 1024))
sys.stderr.flush()
last_console_update = current_time
elif not printed_static_message:
# Print a message (once only) when stderr is not
# going to a live console
sys.stderr.write("Compressing target directory %s..." % (src_dir,))
printed_static_message = True
if last_console_update > 0:
sys.stderr.write("\n")
if tar_exitcode != 0:
raise Exception("tar exited with non-zero exit code " + str(tar_exitcode))
dxpy.set_workspace_id(build_project_id)
remote_file = dxpy.upload_local_file(app_tarball_file, media_type="application/gzip",
wait_on_close=True, show_progress=True)
try:
input_hash = {
"input_file": dxpy.dxlink(remote_file),
"build_options": build_options
}
if mode == 'app':
input_hash["publish"] = publish
api_options = {
"name": "Remote build of %s" % (os.path.basename(src_dir),),
"input": input_hash,
"project": build_project_id,
}
if dest_folder:
api_options["folder"] = dest_folder
app_run_result = dxpy.api.app_run(builder_app, input_params=api_options)
job_id = app_run_result["id"]
print("Started builder job %s" % (job_id,))
if watch:
try:
subprocess.check_call(["dx", "watch", job_id])
except subprocess.CalledProcessError as e:
if e.returncode == 3:
# Some kind of failure to build the app. The reason
# for the failure is probably self-evident from the
# job log (and if it's not, the CalledProcessError
# is not informative anyway), so just propagate the
# return code without additional remarks.
sys.exit(3)
else:
raise e
dxpy.DXJob(job_id).wait_on_done(interval=1)
if mode == 'applet':
applet_id, _ = dxpy.get_dxlink_ids(dxpy.api.job_describe(job_id)['output']['output_applet'])
return applet_id
else:
# TODO: determine and return the app ID, to allow
# running the app if args.run is specified
return None
finally:
if not using_temp_project_for_remote_build:
dxpy.DXProject(build_project_id).remove_objects([remote_file.get_id()])
finally:
if using_temp_project_for_remote_build:
dxpy.api.project_destroy(build_project_id, {"terminateJobs": True})
shutil.rmtree(temp_dir)
def setUpTempProject(thing):
thing.old_workspace_id = dxpy.WORKSPACE_ID
thing.proj_id = dxpy.api.project_new({'name':
'symlink test project'})['id']
dxpy.set_workspace_id(thing.proj_id)
def tearDownTempProject(thing):
dxpy.api.project_destroy(thing.proj_id, {'terminateJobs': True})
dxpy.set_workspace_id(thing.old_workspace_id)
def setUpClass(cls):
cls.project_id = dxpy.api.project_new({"name": "tab-completion project"})['id']
os.environ['DX_PROJECT_CONTEXT_ID'] = cls.project_id
dxpy.set_workspace_id(cls.project_id)
def setUpTempProjects(thing):
thing.old_workspace_id = dxpy.WORKSPACE_ID
thing.proj_id = dxpy.api.project_new({'name': 'test project 1'})['id']
thing.second_proj_id = dxpy.api.project_new({'name': 'test project 2'})['id']
dxpy.set_workspace_id(thing.proj_id)
def main(token):
# Configure dxpy authentication
dxpy.set_security_context({
'auth_token_type': 'Bearer',
'auth_token': token
})
# Resolve FACTORY_PROJECT by ID
proj = dxpy.DXProject(FACTORY_PROJECT)
print 'Resolved project:', proj.describe()['name'], proj.get_id()
# Set FACTORY_PROJECT as the workspace for subsequent operations
# (sort of like the current working directory)
dxpy.set_workspace_id(FACTORY_PROJECT)
# Resolve the workflow by name. (Could also store ID like the project)
wf = list(
dxpy.search.find_data_objects(classname="workflow",
name="RNA-seq pipeline",
return_handler=True))[0]
print 'Resolved workflow:', wf.describe()['name'], wf.get_id()
# TODO: Stage the inputs. Here we find them in the IN folder
left_reads = list(
dxpy.search.find_data_objects(classname="file",
name="ENCFF001JPX.1k.fastq.gz",
folder="/IN",
return_handler=True))[0]
print 'Resolved left reads:', left_reads.describe(
)['name'], left_reads.get_id()
right_reads = list(
dxpy.search.find_data_objects(classname="file",
name="ENCFF001JQB.1k.fastq.gz",
folder="/IN",
return_handler=True))[0]
print 'Resolved right reads:', right_reads.describe(
)['name'], right_reads.get_id()
# Launch the workflow
analysis = wf.run({
'0.fastqs': [dxpy.dxlink(left_reads.get_id())],
'0.fastq_pairs': [dxpy.dxlink(right_reads.get_id())]
})
print 'Launched analysis:', analysis.get_id()
print 'Analysis state:', analysis.describe()['state']
# TODO: Poll for (or come back when) analysis state 'done' or 'failed'.
# Handle any failures.
# Cooking-show-style substitution with completed analysis
analysis = dxpy.DXAnalysis(COMPLETED_ANALYSIS)
print 'Analysis state:', analysis.describe()['state']
# Enumerate outputs
print 'Analysis outputs:'
for one_output_name, one_output_link in analysis.describe(
)['output'].iteritems():
one_output = dxpy.get_handler(
one_output_link) # one_output : dxpy.DXFile
one_file_name = one_output.describe()['name']
one_file_url, _ = one_output.get_download_url(preauthenticated=True,
filename=one_file_name)
print one_file_name, one_file_url
def setUpTempProjects(thing):
thing.old_workspace_id = dxpy.WORKSPACE_ID
thing.proj_id = dxpy.api.project_new({'name': 'azure-test-project', 'region': testutil.TEST_AZURE})['id']
dxpy.set_workspace_id(thing.proj_id)
def _build_app_remote(mode, src_dir, destination=None, publish=False,
dx_toolkit_autodep="auto", version_override=None,
bill_to=None, version_autonumbering=True, update=True,
parallel_build=True):
if mode == 'app':
builder_app = 'app-tarball_app_builder'
else:
builder_app = 'app-tarball_applet_builder'
temp_dir = tempfile.mkdtemp()
# We have to resolve the correct dx-toolkit dependency type here and
# explicitly pass it into the interior call of dx-build-app, because
# within the execution environment of tarball_app(let)_builder,
# APISERVER_HOST is set to the address of the proxy (a 10.x.x.x
# address) and doesn't give us any information about whether we are
# talking to preprod.
if dx_toolkit_autodep == "auto":
# "auto" (the default) means dx-toolkit (stable) on preprod and prod, and
# dx-toolkit-beta on all other systems.
if dxpy.APISERVER_HOST == "preprodapi.dnanexus.com" or dxpy.APISERVER_HOST == "api.dnanexus.com":
dx_toolkit_autodep_flag = "--dx-toolkit-stable-autodep"
else:
dx_toolkit_autodep_flag = "--dx-toolkit-beta-autodep"
elif dx_toolkit_autodep == "git":
dx_toolkit_autodep_flag = "--dx-toolkit-legacy-git-autodep"
elif dx_toolkit_autodep == "stable":
dx_toolkit_autodep_flag = "--dx-toolkit-stable-autodep"
elif dx_toolkit_autodep == "beta":
dx_toolkit_autodep_flag = "--dx-toolkit-beta-autodep"
elif dx_toolkit_autodep == "unstable":
dx_toolkit_autodep_flag = "--dx-toolkit-unstable-autodep"
elif dx_toolkit_autodep == False:
dx_toolkit_autodep_flag = "--no-dx-toolkit-autodep"
extra_flags = [dx_toolkit_autodep_flag]
# These flags are basically passed through verbatim.
if version_override:
extra_flags.extend(['--version', version_override])
if bill_to:
extra_flags.extend(['--bill-to', bill_to])
if not version_autonumbering:
extra_flags.append('--no-version-autonumbering')
if not update:
extra_flags.append('--no-update')
if not parallel_build:
extra_flags.append('--no-parallel-build')
using_temp_project_for_remote_build = False
# If building an applet, run the builder app in the destination
# project. If building an app, run the builder app in a temporary
# project.
dest_folder = None
dest_applet_name = None
if mode == "applet":
# Translate the --destination flag as follows. If --destination
# is PROJ:FOLDER/NAME,
#
# 1. Run the builder app in PROJ
# 2. Make the output folder FOLDER
# 3. Supply --destination=NAME to the interior call of dx-build-applet.
build_project_id = dxpy.WORKSPACE_ID
if destination:
build_project_id, dest_folder, dest_applet_name = parse_destination(destination)
if build_project_id is None:
parser.error("Can't create an applet without specifying a destination project; please use the -d/--destination flag to explicitly specify a project")
if dest_applet_name:
# TODO: escape this correctly, or find a way of passing the
# extra_flags that doesn't get screwed up by names that have
# spaces in them.
extra_flags.extend(['--destination', '/' + dest_applet_name])
elif mode == "app":
using_temp_project_for_remote_build = True
build_project_id = dxpy.api.projectNew({"name": "dx-build-app --remote temporary project"})["id"]
try:
# Resolve relative paths and symlinks here so we have something
# reasonable to write in the job name below.
src_dir = os.path.realpath(src_dir)
# Show the user some progress as the tarball is being generated.
# Hopefully this will help them to understand when their tarball
# is huge (e.g. the target directory already has a whole bunch
# of binaries in it) and interrupt before uploading begins.
app_tarball_file = os.path.join(temp_dir, "app_tarball.tar.gz")
# TODO: figure out if we can use --exclude-vcs here (conditional
# on presence of GNU tar). This might require propagating the
# --version directly to the interior dx-build-app since in
# general that can depend on the git metadata.
tar_subprocess = subprocess.Popen(["tar", "-czf", "-", "."], cwd=src_dir, stdout=subprocess.PIPE)
with open(app_tarball_file, 'w') as tar_output_file:
total_num_bytes = 0
last_console_update = 0
start_time = time.time()
printed_static_message = False
# Pipe the output of tar into the output file, and
while True:
tar_exitcode = tar_subprocess.poll()
data = tar_subprocess.stdout.read(4 * 1024 * 1024)
if tar_exitcode is not None and len(data) == 0:
break
tar_output_file.write(data)
total_num_bytes += len(data)
current_time = time.time()
# Don't show status messages at all for very short tar
# operations (< 1.0 sec)
if current_time - last_console_update > 0.25 and current_time - start_time > 1.0:
if sys.stderr.isatty():
if last_console_update > 0:
sys.stderr.write("\r")
sys.stderr.write("Compressing target directory %s... (%s kb)" % (src_dir, locale.format("%d", (total_num_bytes / 1024,), grouping=True),))
sys.stderr.flush()
last_console_update = current_time
elif not printed_static_message:
# Print a message (once only) when stderr is not
# going to a live console
sys.stderr.write("Compressing target directory %s..." % (src_dir,))
printed_static_message = True
if last_console_update > 0:
sys.stderr.write("\n")
if tar_exitcode != 0:
raise Exception("tar exited with non-zero exit code " + str(tar_exitcode))
dxpy.set_workspace_id(build_project_id)
remote_file = dxpy.upload_local_file(app_tarball_file, media_type="application/gzip",
wait_on_close=True, show_progress=True)
try:
extra_flags_str = " ".join(pipes.quote(s) for s in extra_flags)
input_hash = {
"input_file": dxpy.dxlink(remote_file),
"extra_flags": extra_flags_str
}
if mode == 'app':
input_hash["publish"] = publish
api_options = {
"name": "Remote build of %s" % (os.path.basename(src_dir),),
"input": input_hash,
"project": build_project_id,
}
if dest_folder:
api_options["folder"] = dest_folder
app_run_result = dxpy.api.appRun(builder_app, input_params=api_options)
job_id = app_run_result["id"]
print "Started builder job %s" % (job_id,)
subprocess.check_call(["dx", "watch", job_id])
finally:
if not using_temp_project_for_remote_build:
dxpy.DXProject(build_project_id).remove_objects([remote_file.get_id()])
finally:
if using_temp_project_for_remote_build:
dxpy.api.projectDestroy(build_project_id)
shutil.rmtree(temp_dir)
return
