def unpack_tar(self, tar_file_dxlink):
'''
DEV: Eventually integrate dx-toolkit into trajectoread repo so I can
transition to using 'dx-download-all-inputs' to handle unpacking
all input files.
Pipeline used to store lane file dxids as project properties
and then pass to "dx download"
Description: Download and untar metadata and lane data files
(/Data/Intensities/BaseCalls)
'''
if dxpy.is_dxlink(tar_file_dxlink):
file_handler = dxpy.get_handler(tar_file_dxlink)
filename = file_handler.name
else:
print 'Error: Cannot unpack %s; not a valid DXLink object'
sys.exit()
# ('file-dxid', 'project-dxid') = dxpy.get_dxlink_ids(dxlink)
file_dxid = dxpy.get_dxlink_ids(tar_file_dxlink)[0]
project_id = dxpy.get_dxlink_ids(tar_file_dxlink)[1]
# Download file from DNAnexus objectstore to virtual machine
dxpy.download_dxfile(dxid=file_dxid,
filename=filename,
project=project_id)
# Untar file
## DEV: Check if this is even in use anymore; also should have some method for
## checking what type of compression was used.
## But I don't think this is in use
command = 'tar -xf %s --owner root --group root --no-same-owner' % filename
self.createSubprocess(cmd=command, pipeStdout=False)
def format_data_object_reference(item):
if dxpy.is_dxlink(item):
# Bare dxlink
obj_id, proj_id = dxpy.get_dxlink_ids(item)
return (proj_id + ":" if proj_id else '') + obj_id
if dxpy.is_dxlink(item.get('value')):
# value is set
obj_id, proj_id = dxpy.get_dxlink_ids(item['value'])
return (proj_id + ":" if proj_id else '') + obj_id + (' (%s)' % item['name'] if item.get('name') else '')
if item.get('project') and item.get('path'):
# project and folder path
return item['project'] + ':' + item['path'] + "/" + obj_class + "-*" + (' (%s)' % item['name'] if item.get('name') else '')
return str(item)
def format_data_object_reference(item):
if dxpy.is_dxlink(item):
# Bare dxlink
obj_id, proj_id = dxpy.get_dxlink_ids(item)
return (proj_id + ":" if proj_id else '') + obj_id
if dxpy.is_dxlink(item.get('value')):
# value is set
obj_id, proj_id = dxpy.get_dxlink_ids(item['value'])
return (proj_id + ":" if proj_id else '') + obj_id + (' (%s)' % item['name'] if item.get('name') else '')
if item.get('project') and item.get('path'):
# project and folder path
return item['project'] + ':' + item['path'] + "/" + obj_class + "-*" + (' (%s)' % item['name'] if item.get('name') else '')
return str(item)
def build_asset(args):
if args.src_dir is None:
args.src_dir = os.getcwd()
dest_project_name = None
dest_folder_name = None
dest_asset_name = None
make_file = None
asset_file = None
conf_file = None
try:
asset_conf = parse_asset_spec(args.src_dir)
validate_conf(asset_conf)
asset_conf_file = os.path.join(args.src_dir, "dxasset.json")
dxpy.api.system_whoami()
dest_project_name, dest_folder_name, dest_asset_name = parse_destination(args.destination)
if dest_project_name is None:
raise AssetBuilderException("Can't build an asset without specifying a destination project; \
please use the -d/--destination flag to explicitly specify a project")
if dest_asset_name is None:
dest_asset_name = asset_conf['name']
# If dx build_asset is launched form a job, set json flag to True to avoid watching the job log
if dxpy.JOB_ID:
args.json = True
if not args.json:
print("Uploading input files for the AssetBuilder", file=sys.stderr)
conf_file = dx_upload(asset_conf_file, dest_project_name, dest_folder_name, args.json)
make_file = get_asset_make(args.src_dir, dest_project_name, dest_folder_name, args.json)
asset_file = get_asset_tarball(asset_conf['name'], args.src_dir, dest_project_name,
dest_folder_name, args.json)
input_hash = {"conf_json": dxpy.dxlink(conf_file)}
if asset_file:
input_hash["custom_asset"] = dxpy.dxlink(asset_file)
if make_file:
input_hash["asset_makefile"] = dxpy.dxlink(make_file)
builder_run_options = {
"name": dest_asset_name,
"input": input_hash
}
# Add the default destination project to app run options, if it is not run from a job
if not dxpy.JOB_ID:
builder_run_options["project"] = dest_project_name
if 'instanceType' in asset_conf:
builder_run_options["systemRequirements"] = {"*": {"instanceType": asset_conf["instanceType"]}}
if dest_folder_name:
builder_run_options["folder"] = dest_folder_name
if asset_conf['release'] == "12.04":
app_run_result = dxpy.api.app_run(ASSET_BUILDER_PRECISE, input_params=builder_run_options)
elif asset_conf['release'] == "14.04":
app_run_result = dxpy.api.app_run(ASSET_BUILDER_TRUSTY, input_params=builder_run_options)
job_id = app_run_result["id"]
if not args.json:
print("\nStarted job '" + str(job_id) + "' to build the asset bundle.\n", file=sys.stderr)
if args.watch:
try:
subprocess.check_call(["dx", "watch", job_id])
except subprocess.CalledProcessError as e:
if e.returncode == 3:
# Some kind of failure to build the asset. The reason
# for the failure is probably self-evident from the
# job log (and if it's not, the CalledProcessError
# is not informative anyway), so just propagate the
# return code without additional remarks.
sys.exit(3)
else:
raise e
dxpy.DXJob(job_id).wait_on_done(interval=1)
asset_id, _ = dxpy.get_dxlink_ids(dxpy.api.job_describe(job_id)['output']['asset_bundle'])
if args.json:
print(json.dumps({"id": asset_id}))
else:
print("\nAsset bundle '" + asset_id +
"' is built and can now be used in your app/applet's dxapp.json\n", file=sys.stderr)
except Exception as de:
print(de.__class__.__name__ + ": " + str(de), file=sys.stderr)
sys.exit(1)
finally:
if conf_file:
try:
conf_file.remove()
except:
pass
if make_file:
try:
make_file.remove()
except:
pass
if asset_file:
try:
asset_file.remove()
except:
pass
def build_and_upload_locally(
src_dir,
mode,
overwrite=False,
archive=False,
publish=False,
destination_override=None,
version_override=None,
bill_to_override=None,
use_temp_build_project=True,
do_parallel_build=True,
do_version_autonumbering=True,
do_try_update=True,
dx_toolkit_autodep="stable",
do_check_syntax=True,
dry_run=False,
return_object_dump=False,
confirm=True,
ensure_upload=False,
region=None,
**kwargs
):
dxpy.app_builder.build(src_dir, parallel_build=do_parallel_build)
app_json = _parse_app_spec(src_dir)
_verify_app_source_dir(src_dir, mode, enforce=do_check_syntax)
if mode == "app" and not dry_run:
_verify_app_writable(app_json["name"])
working_project = None
using_temp_project = False
override_folder = None
override_applet_name = None
if mode == "applet" and destination_override:
working_project, override_folder, override_applet_name = parse_destination(destination_override)
elif mode == "app" and use_temp_build_project and not dry_run:
# Create a temp project
try:
if region:
working_project = dxpy.api.project_new(
{"name": "Temporary build project for dx-build-app", "region": region}
)["id"]
else:
working_project = dxpy.api.project_new({"name": "Temporary build project for dx-build-app"})["id"]
except:
err_exit()
logger.debug("Created temporary project %s to build in" % (working_project,))
using_temp_project = True
try:
if mode == "applet" and working_project is None and dxpy.WORKSPACE_ID is None:
parser.error(
"Can't create an applet without specifying a destination project; please use the -d/--destination flag to explicitly specify a project"
)
if "buildOptions" in app_json:
if app_json["buildOptions"].get("dx_toolkit_autodep") == False:
dx_toolkit_autodep = False
# Perform check for existence of applet with same name in
# destination for case in which neither "-f" nor "-a" is
# given BEFORE uploading resources.
if mode == "applet" and not overwrite and not archive:
try:
dest_name = override_applet_name or app_json.get("name") or os.path.basename(os.path.abspath(src_dir))
except:
raise dxpy.app_builder.AppBuilderException(
"Could not determine applet name from specification + "
"(dxapp.json) or from working directory (%r)" % (src_dir,)
)
dest_folder = override_folder or app_json.get("folder") or "/"
if not dest_folder.endswith("/"):
dest_folder = dest_folder + "/"
dest_project = working_project if working_project else dxpy.WORKSPACE_ID
for result in dxpy.find_data_objects(
classname="applet", name=dest_name, folder=dest_folder, project=dest_project, recurse=False
):
dest_path = dest_folder + dest_name
msg = "An applet already exists at {} (id {}) and neither".format(dest_path, result["id"])
msg += " -f/--overwrite nor -a/--archive were given."
raise dxpy.app_builder.AppBuilderException(msg)
bundled_resources = (
dxpy.app_builder.upload_resources(
src_dir, project=working_project, folder=override_folder, ensure_upload=ensure_upload
)
if not dry_run
else []
)
try:
# TODO: the "auto" setting is vestigial and should be removed.
if dx_toolkit_autodep == "auto":
dx_toolkit_autodep = "stable"
applet_id, applet_spec = dxpy.app_builder.upload_applet(
src_dir,
bundled_resources,
check_name_collisions=(mode == "applet"),
overwrite=overwrite and mode == "applet",
archive=archive and mode == "applet",
project=working_project,
override_folder=override_folder,
override_name=override_applet_name,
dx_toolkit_autodep=dx_toolkit_autodep,
dry_run=dry_run,
**kwargs
)
except:
# Avoid leaking any bundled_resources files we may have
# created, if applet creation fails. Note that if
# using_temp_project, the entire project gets destroyed at
# the end, so we don't bother.
if not using_temp_project:
objects_to_delete = [
dxpy.get_dxlink_ids(bundled_resource_obj["id"])[0] for bundled_resource_obj in bundled_resources
]
if objects_to_delete:
dxpy.api.project_remove_objects(
dxpy.app_builder.get_destination_project(src_dir, project=working_project),
input_params={"objects": objects_to_delete},
)
raise
if dry_run:
return
applet_name = applet_spec["name"]
logger.debug("Created applet " + applet_id + " successfully")
if mode == "app":
if "version" not in app_json:
parser.error('dxapp.json contains no "version" field, but it is required to build an app')
version = app_json["version"]
try_versions = [version_override or version]
if not version_override and do_version_autonumbering:
try_versions.append(version + _get_version_suffix(src_dir, version))
app_id = dxpy.app_builder.create_app(
applet_id,
applet_name,
src_dir,
publish=publish,
set_default=publish,
billTo=bill_to_override,
try_versions=try_versions,
try_update=do_try_update,
confirm=confirm,
)
app_describe = dxpy.api.app_describe(app_id)
if publish:
print(
"Uploaded and published app %s/%s (%s) successfully"
% (app_describe["name"], app_describe["version"], app_id),
file=sys.stderr,
)
else:
print(
"Uploaded app %s/%s (%s) successfully" % (app_describe["name"], app_describe["version"], app_id),
file=sys.stderr,
)
print("You can publish this app with:", file=sys.stderr)
print(
' dx api app-%s/%s publish "{\\"makeDefault\\": true}"'
% (app_describe["name"], app_describe["version"]),
file=sys.stderr,
)
return app_describe if return_object_dump else {"id": app_id}
elif mode == "applet":
return dxpy.api.applet_describe(applet_id) if return_object_dump else {"id": applet_id}
else:
raise dxpy.app_builder.AppBuilderException("Unrecognized mode %r" % (mode,))
finally:
# Clean up after ourselves.
if using_temp_project:
dxpy.api.project_destroy(working_project)
def _build_app_remote(
mode,
src_dir,
publish=False,
destination_override=None,
version_override=None,
bill_to_override=None,
dx_toolkit_autodep="stable",
do_version_autonumbering=True,
do_try_update=True,
do_parallel_build=True,
do_check_syntax=True,
region=None,
):
if mode == "app":
builder_app = "app-tarball_app_builder"
else:
builder_app = "app-tarball_applet_builder"
temp_dir = tempfile.mkdtemp()
# TODO: this is vestigial, the "auto" setting should be removed.
if dx_toolkit_autodep == "auto":
dx_toolkit_autodep = "stable"
build_options = {"dx_toolkit_autodep": dx_toolkit_autodep}
if version_override:
build_options["version_override"] = version_override
elif do_version_autonumbering:
# If autonumbering is DISABLED, the interior run of dx-build-app
# will detect the correct version to use without our help. If it
# is ENABLED, the version suffix might depend on the state of
# the git repository. Since we'll remove the .git directory
# before uploading, we need to determine the correct version to
# use here and pass it in to the interior run of dx-build-app.
if do_version_autonumbering:
app_spec = _parse_app_spec(src_dir)
original_version = app_spec["version"]
app_describe = None
try:
app_describe = dxpy.api.app_describe(
"app-" + app_spec["name"], alias=original_version, always_retry=False
)
except dxpy.exceptions.DXAPIError as e:
if e.name == "ResourceNotFound" or (mode == "applet" and e.name == "PermissionDenied"):
pass
else:
raise e
if app_describe is not None:
if "published" in app_describe or not do_try_update:
# The version we wanted was taken; fall back to the
# autogenerated version number.
build_options["version_override"] = original_version + _get_version_suffix(
src_dir, original_version
)
# The following flags are basically passed through verbatim.
if bill_to_override:
build_options["bill_to_override"] = bill_to_override
if not do_version_autonumbering:
build_options["do_version_autonumbering"] = False
if not do_try_update:
build_options["do_try_update"] = False
if not do_parallel_build:
build_options["do_parallel_build"] = False
if not do_check_syntax:
build_options["do_check_syntax"] = False
using_temp_project_for_remote_build = False
# If building an applet, run the builder app in the destination
# project. If building an app, run the builder app in a temporary
# project.
dest_folder = None
dest_applet_name = None
if mode == "applet":
# Translate the --destination flag as follows. If --destination
# is PROJ:FOLDER/NAME,
#
# 1. Run the builder app in PROJ
# 2. Make the output folder FOLDER
# 3. Supply --destination=NAME to the interior call of dx-build-applet.
build_project_id = dxpy.WORKSPACE_ID
if destination_override:
build_project_id, dest_folder, dest_applet_name = parse_destination(destination_override)
if build_project_id is None:
parser.error(
"Can't create an applet without specifying a destination project; please use the -d/--destination flag to explicitly specify a project"
)
if dest_applet_name:
build_options["destination_override"] = "/" + dest_applet_name
elif mode == "app":
using_temp_project_for_remote_build = True
try:
if region:
build_project_id = dxpy.api.project_new(
{"name": "dx-build-app --remote temporary project", "region": region}
)["id"]
else:
build_project_id = dxpy.api.project_new({"name": "dx-build-app --remote temporary project"})["id"]
except:
err_exit()
try:
# Resolve relative paths and symlinks here so we have something
# reasonable to write in the job name below.
src_dir = os.path.realpath(src_dir)
# Show the user some progress as the tarball is being generated.
# Hopefully this will help them to understand when their tarball
# is huge (e.g. the target directory already has a whole bunch
# of binaries in it) and interrupt before uploading begins.
app_tarball_file = os.path.join(temp_dir, "app_tarball.tar.gz")
tar_subprocess = subprocess.Popen(
["tar", "-czf", "-", "--exclude", "./.git", "."], cwd=src_dir, stdout=subprocess.PIPE
)
with open(app_tarball_file, "wb") as tar_output_file:
total_num_bytes = 0
last_console_update = 0
start_time = time.time()
printed_static_message = False
# Pipe the output of tar into the output file
while True:
tar_exitcode = tar_subprocess.poll()
data = tar_subprocess.stdout.read(4 * 1024 * 1024)
if tar_exitcode is not None and len(data) == 0:
break
tar_output_file.write(data)
total_num_bytes += len(data)
current_time = time.time()
# Don't show status messages at all for very short tar
# operations (< 1.0 sec)
if current_time - last_console_update > 0.25 and current_time - start_time > 1.0:
if sys.stderr.isatty():
if last_console_update > 0:
sys.stderr.write("\r")
sys.stderr.write(
"Compressing target directory {dir}... ({kb_so_far:,} kb)".format(
dir=src_dir, kb_so_far=total_num_bytes // 1024
)
)
sys.stderr.flush()
last_console_update = current_time
elif not printed_static_message:
# Print a message (once only) when stderr is not
# going to a live console
sys.stderr.write("Compressing target directory %s..." % (src_dir,))
printed_static_message = True
if last_console_update > 0:
sys.stderr.write("\n")
if tar_exitcode != 0:
raise Exception("tar exited with non-zero exit code " + str(tar_exitcode))
dxpy.set_workspace_id(build_project_id)
remote_file = dxpy.upload_local_file(
app_tarball_file, media_type="application/gzip", wait_on_close=True, show_progress=True
)
try:
input_hash = {"input_file": dxpy.dxlink(remote_file), "build_options": build_options}
if mode == "app":
input_hash["publish"] = publish
api_options = {
"name": "Remote build of %s" % (os.path.basename(src_dir),),
"input": input_hash,
"project": build_project_id,
}
if dest_folder:
api_options["folder"] = dest_folder
app_run_result = dxpy.api.app_run(builder_app, input_params=api_options)
job_id = app_run_result["id"]
print("Started builder job %s" % (job_id,))
try:
subprocess.check_call(["dx", "watch", job_id])
except subprocess.CalledProcessError as e:
if e.returncode == 3:
# Some kind of failure to build the app. The reason
# for the failure is probably self-evident from the
# job log (and if it's not, the CalledProcessError
# is not informative anyway), so just propagate the
# return code without additional remarks.
sys.exit(3)
else:
raise e
dxpy.DXJob(job_id).wait_on_done(interval=1)
if mode == "applet":
applet_id, _ = dxpy.get_dxlink_ids(dxpy.api.job_describe(job_id)["output"]["output_applet"])
return applet_id
else:
# TODO: determine and return the app ID, to allow
# running the app if args.run is specified
return None
finally:
if not using_temp_project_for_remote_build:
dxpy.DXProject(build_project_id).remove_objects([remote_file.get_id()])
finally:
if using_temp_project_for_remote_build:
dxpy.api.project_destroy(build_project_id, {"terminateJobs": True})
shutil.rmtree(temp_dir)
def build_and_upload_locally(src_dir, mode, overwrite=False, publish=False, destination_override=None, version_override=None, bill_to_override=None, use_temp_build_project=True, do_parallel_build=True, do_version_autonumbering=True, do_try_update=True, dx_toolkit_autodep="stable", do_build_step=True, do_upload_step=True, do_check_syntax=True, dry_run=False, return_object_dump=False):
app_json = _parse_app_spec(src_dir)
_verify_app_source_dir(src_dir, enforce=do_check_syntax)
working_project = None
using_temp_project = False
override_folder = None
override_applet_name = None
if mode == "applet" and destination_override:
working_project, override_folder, override_applet_name = parse_destination(destination_override)
elif mode == "app" and use_temp_build_project and not dry_run:
# Create a temp project
working_project = dxpy.api.project_new({"name": "Temporary build project for dx-build-app"})["id"]
print >> sys.stderr, "Created temporary project %s to build in" % (working_project,)
using_temp_project = True
try:
if mode == "applet" and working_project is None and dxpy.WORKSPACE_ID is None:
parser.error("Can't create an applet without specifying a destination project; please use the -d/--destination flag to explicitly specify a project")
if "buildOptions" in app_json:
if app_json["buildOptions"].get("dx_toolkit_autodep") == False:
dx_toolkit_autodep = False
del app_json["buildOptions"]
if do_build_step:
dxpy.app_builder.build(src_dir, parallel_build=do_parallel_build)
if not do_upload_step:
return
bundled_resources = dxpy.app_builder.upload_resources(src_dir, project=working_project) if not dry_run else []
try:
# TODO: the "auto" setting is vestigial and should be removed.
if dx_toolkit_autodep == "auto":
dx_toolkit_autodep = "stable"
applet_id, applet_spec = dxpy.app_builder.upload_applet(
src_dir,
bundled_resources,
check_name_collisions=(mode == "applet"),
overwrite=overwrite and mode == "applet",
project=working_project,
override_folder=override_folder,
override_name=override_applet_name,
dx_toolkit_autodep=dx_toolkit_autodep,
dry_run=dry_run)
except:
# Avoid leaking any bundled_resources files we may have
# created, if applet creation fails. Note that if
# using_temp_project, the entire project gets destroyed at
# the end, so we don't bother.
if not using_temp_project:
objects_to_delete = [dxpy.get_dxlink_ids(bundled_resource_obj['id'])[0] for bundled_resource_obj in bundled_resources]
if objects_to_delete:
dxpy.api.project_remove_objects(dxpy.app_builder.get_destination_project(src_dir, project=working_project),
input_params={"objects": objects_to_delete})
raise
if dry_run:
return
applet_name = applet_spec['name']
print >> sys.stderr, "Created applet " + applet_id + " successfully"
if mode == "app":
if 'version' not in app_json:
parser.error("dxapp.json contains no \"version\" field, but it is required to build an app")
version = app_json['version']
try_versions = [version_override or version]
if not version_override and do_version_autonumbering:
try_versions.append(version + _get_version_suffix(src_dir, version))
app_id = dxpy.app_builder.create_app(applet_id,
applet_name,
src_dir,
publish=publish,
set_default=publish,
billTo=bill_to_override,
try_versions=try_versions,
try_update=do_try_update)
app_describe = dxpy.api.app_describe(app_id)
if publish:
print >> sys.stderr, "Uploaded and published app %s/%s (%s) successfully" % (app_describe["name"], app_describe["version"], app_id)
else:
print >> sys.stderr, "Uploaded app %s/%s (%s) successfully" % (app_describe["name"], app_describe["version"], app_id)
print >> sys.stderr, "You can publish this app with:"
print >> sys.stderr, " dx api app-%s/%s publish \"{\\\"makeDefault\\\": true}\"" % (app_describe["name"], app_describe["version"])
return app_describe if return_object_dump else None
elif mode == "applet":
return dxpy.api.applet_describe(applet_id) if return_object_dump else None
else:
raise dxpy.app_builder.AppBuilderException("Unrecognized mode %r" % (mode,))
finally:
# Clean up after ourselves.
if using_temp_project:
dxpy.api.project_destroy(working_project)
def build_asset(args):
if args.src_dir is None:
args.src_dir = os.getcwd()
dest_project_name = None
dest_folder_name = None
dest_asset_name = None
make_file = None
asset_file = None
conf_file = None
try:
asset_conf = parse_asset_spec(args.src_dir)
validate_conf(asset_conf)
asset_conf_file = os.path.join(args.src_dir, "dxasset.json")
dxpy.api.system_whoami()
dest_project_name, dest_folder_name, dest_asset_name = parse_destination(args.destination)
if dest_project_name is None:
raise AssetBuilderException("Can't build an asset without specifying a destination project; \
please use the -d/--destination flag to explicitly specify a project")
if dest_asset_name is None:
dest_asset_name = asset_conf['name']
# If dx build_asset is launched form a job, set json flag to True to avoid watching the job log
if dxpy.JOB_ID:
args.json = True
if not args.json:
print("Uploading input files for the AssetBuilder", file=sys.stderr)
conf_file = dx_upload(asset_conf_file, dest_project_name, dest_folder_name, args.json)
make_file = get_asset_make(args.src_dir, dest_project_name, dest_folder_name, args.json)
asset_file = get_asset_tarball(asset_conf['name'], args.src_dir, dest_project_name,
dest_folder_name, args.json)
input_hash = {"conf_json": dxpy.dxlink(conf_file)}
if asset_file:
input_hash["custom_asset"] = dxpy.dxlink(asset_file)
if make_file:
input_hash["asset_makefile"] = dxpy.dxlink(make_file)
builder_run_options = {
"name": dest_asset_name,
"input": input_hash
}
if args.priority is not None:
builder_run_options["priority"] = args.priority
# Add the default destination project to app run options, if it is not run from a job
if not dxpy.JOB_ID:
builder_run_options["project"] = dest_project_name
if 'instanceType' in asset_conf:
builder_run_options["systemRequirements"] = {"*": {"instanceType": asset_conf["instanceType"]}}
if dest_folder_name:
builder_run_options["folder"] = dest_folder_name
if asset_conf['release'] == "12.04":
app_run_result = dxpy.api.app_run(ASSET_BUILDER_PRECISE, input_params=builder_run_options)
elif asset_conf['release'] == "14.04":
app_run_result = dxpy.api.app_run(ASSET_BUILDER_TRUSTY, input_params=builder_run_options)
elif asset_conf['release'] == "16.04" and asset_conf['runSpecVersion'] == '1':
app_run_result = dxpy.api.app_run(ASSET_BUILDER_XENIAL_V1, input_params=builder_run_options)
elif asset_conf['release'] == "16.04":
app_run_result = dxpy.api.app_run(ASSET_BUILDER_XENIAL, input_params=builder_run_options)
elif asset_conf['release'] == "20.04":
app_run_result = dxpy.api.app_run(ASSET_BUILDER_FOCAL, input_params=builder_run_options)
job_id = app_run_result["id"]
if not args.json:
print("\nStarted job '" + str(job_id) + "' to build the asset bundle.\n", file=sys.stderr)
if args.watch:
try:
subprocess.check_call(["dx", "watch", job_id])
except subprocess.CalledProcessError as e:
if e.returncode == 3:
# Some kind of failure to build the asset. The reason
# for the failure is probably self-evident from the
# job log (and if it's not, the CalledProcessError
# is not informative anyway), so just propagate the
# return code without additional remarks.
sys.exit(3)
else:
raise e
dxpy.DXJob(job_id).wait_on_done(interval=1)
asset_id, _ = dxpy.get_dxlink_ids(dxpy.api.job_describe(job_id)['output']['asset_bundle'])
if args.json:
print(json.dumps({"id": asset_id}))
else:
print("\nAsset bundle '" + asset_id +
"' is built and can now be used in your app/applet's dxapp.json\n", file=sys.stderr)
except Exception as de:
print(de.__class__.__name__ + ": " + str(de), file=sys.stderr)
sys.exit(1)
finally:
if conf_file:
try:
conf_file.remove()
except:
pass
if make_file:
try:
make_file.remove()
except:
pass
if asset_file:
try:
asset_file.remove()
except:
pass
def build_and_upload_locally(src_dir, mode, overwrite=False, archive=False, publish=False, destination_override=None,
version_override=None, bill_to_override=None, use_temp_build_project=True,
do_parallel_build=True, do_version_autonumbering=True, do_try_update=True,
dx_toolkit_autodep="stable", do_check_syntax=True, dry_run=False,
return_object_dump=False, confirm=True, ensure_upload=False, force_symlinks=False,
region=None, **kwargs):
dxpy.app_builder.build(src_dir, parallel_build=do_parallel_build)
app_json = _parse_app_spec(src_dir)
_check_suggestions(app_json, publish=publish)
_verify_app_source_dir(src_dir, mode, enforce=do_check_syntax)
if mode == "app" and not dry_run:
_verify_app_writable(app_json['name'])
working_project = None
using_temp_project = False
override_folder = None
override_applet_name = None
enabled_regions = dxpy.app_builder.get_enabled_regions(app_json, region)
# Cannot build multi-region app if `use_temp_build_project` is falsy.
if enabled_regions is not None and len(enabled_regions) > 1 and not use_temp_build_project:
raise dxpy.app_builder.AppBuilderException("Cannot specify --no-temp-build-project when building multi-region apps")
projects_by_region = None
if mode == "applet" and destination_override:
working_project, override_folder, override_applet_name = parse_destination(destination_override)
region = dxpy.api.project_describe(working_project,
input_params={"fields": {"region": True}})["region"]
projects_by_region = {region: working_project}
elif mode == "app" and use_temp_build_project and not dry_run:
projects_by_region = {}
if enabled_regions is not None:
# Create temporary projects in each enabled region.
try:
for region in enabled_regions:
project_input = {
"name": "Temporary build project for dx-build-app in {r}".format(r=region),
"region": region
}
if bill_to_override:
project_input["billTo"] = bill_to_override
working_project = dxpy.api.project_new(project_input)["id"]
projects_by_region[region] = working_project
logger.debug("Created temporary project %s to build in" % (working_project,))
except:
# A /project/new request may fail if the requesting user is
# not authorized to create projects in a certain region.
delete_temporary_projects(projects_by_region.values())
err_exit()
else:
# Create a temp project
try:
project_input = {"name": "Temporary build project for dx-build-app"}
if bill_to_override:
project_input["billTo"] = bill_to_override
working_project = dxpy.api.project_new(project_input)["id"]
except:
err_exit()
region = dxpy.api.project_describe(working_project,
input_params={"fields": {"region": True}})["region"]
projects_by_region[region] = working_project
logger.debug("Created temporary project %s to build in" % (working_project,))
using_temp_project = True
elif mode == "app" and not dry_run:
# If we are not using temporary project(s) to build the executable,
# then we should have a project context somewhere.
try:
project = app_json.get("project", dxpy.WORKSPACE_ID)
region = dxpy.api.project_describe(project,
input_params={"fields": {"region": True}})["region"]
except Exception:
err_exit()
projects_by_region = {region: project}
try:
if mode == "applet" and working_project is None and dxpy.WORKSPACE_ID is None:
parser.error("Can't create an applet without specifying a destination project; please use the -d/--destination flag to explicitly specify a project")
if mode == "applet":
dest_project = working_project or dxpy.WORKSPACE_ID or app_json.get("project", False)
try:
region = dxpy.api.project_describe(dest_project,
input_params={"fields": {"region": True}})["region"]
except Exception:
err_exit()
projects_by_region = {region: dest_project}
if not overwrite and not archive:
# If we cannot overwite or archive an existing applet and an
# applet in the destination exists with the same name as this
# one, then we should err out *before* uploading resources.
try:
dest_name = override_applet_name or app_json.get('name') or os.path.basename(os.path.abspath(src_dir))
except:
raise dxpy.app_builder.AppBuilderException("Could not determine applet name from specification + "
"(dxapp.json) or from working directory (%r)" % (src_dir,))
dest_folder = override_folder or app_json.get('folder') or '/'
if not dest_folder.endswith('/'):
dest_folder = dest_folder + '/'
for result in dxpy.find_data_objects(classname="applet", name=dest_name, folder=dest_folder,
project=dest_project, recurse=False):
dest_path = dest_folder + dest_name
msg = "An applet already exists at {} (id {}) and neither".format(dest_path, result["id"])
msg += " -f/--overwrite nor -a/--archive were given."
raise dxpy.app_builder.AppBuilderException(msg)
if "buildOptions" in app_json:
if app_json["buildOptions"].get("dx_toolkit_autodep") is False:
dx_toolkit_autodep = False
if dry_run:
# Set a dummy "projects_by_region" so that we can exercise the dry
# run flows for uploading resources bundles and applets below.
projects_by_region = {"dummy-cloud:dummy-region": "project-dummy"}
if projects_by_region is None:
raise AssertionError("'projects_by_region' should not be None at this point")
# "resources" can be used only with an app enabled in a single region and when
# "regionalOptions" field is not specified.
if "resources" in app_json and ("regionalOptions" in app_json or len(projects_by_region) > 1):
error_message = "dxapp.json cannot contain a top-level \"resources\" field "
error_message += "when the \"regionalOptions\" field is used or when "
error_message += "the app is enabled in multiple regions"
raise dxpy.app_builder.AppBuilderException(error_message)
resources_bundles_by_region = {}
for region, project in projects_by_region.items():
resources_bundles_by_region[region] = dxpy.app_builder.upload_resources(
src_dir,
project=project,
folder=override_folder,
ensure_upload=ensure_upload,
force_symlinks=force_symlinks) if not dry_run else []
# TODO: Clean up these applets if the app build fails.
applet_ids_by_region = {}
try:
for region, project in projects_by_region.items():
applet_id, applet_spec = dxpy.app_builder.upload_applet(
src_dir,
resources_bundles_by_region[region],
check_name_collisions=(mode == "applet"),
overwrite=overwrite and mode == "applet",
archive=archive and mode == "applet",
project=project,
override_folder=override_folder,
override_name=override_applet_name,
dx_toolkit_autodep=dx_toolkit_autodep,
dry_run=dry_run,
**kwargs)
if not dry_run:
logger.debug("Created applet " + applet_id + " successfully")
applet_ids_by_region[region] = applet_id
except:
# Avoid leaking any bundled_resources files we may have
# created, if applet creation fails. Note that if
# using_temp_project, the entire project gets destroyed at
# the end, so we don't bother.
if not using_temp_project:
for region, project in projects_by_region.items():
objects_to_delete = [dxpy.get_dxlink_ids(bundled_resource_obj['id'])[0] for bundled_resource_obj in resources_bundles_by_region[region]]
if objects_to_delete:
dxpy.api.project_remove_objects(
dxpy.app_builder.get_destination_project(src_dir, project=project),
input_params={"objects": objects_to_delete})
raise
if dry_run:
return
applet_name = applet_spec['name']
if mode == "app":
if 'version' not in app_json:
parser.error("dxapp.json contains no \"version\" field, but it is required to build an app")
version = app_json['version']
try_versions = [version_override or version]
if not version_override and do_version_autonumbering:
try_versions.append(version + _get_version_suffix(src_dir, version))
additional_resources_by_region = {}
if "regionalOptions" in app_json:
for region, region_config in app_json["regionalOptions"].items():
if "resources" in region_config:
additional_resources_by_region[region] = region_config["resources"]
elif "resources" in app_json:
additional_resources_by_region[projects_by_region.keys()[0]] = app_json["resources"]
regional_options = {}
for region in projects_by_region:
regional_options[region] = {"applet": applet_ids_by_region[region]}
if region in additional_resources_by_region:
regional_options[region]["resources"] = additional_resources_by_region[region]
app_id = dxpy.app_builder.create_app_multi_region(regional_options,
applet_name,
src_dir,
publish=publish,
set_default=publish,
billTo=bill_to_override,
try_versions=try_versions,
try_update=do_try_update,
confirm=confirm)
app_describe = dxpy.api.app_describe(app_id)
if publish:
print("Uploaded and published app %s/%s (%s) successfully" % (app_describe["name"], app_describe["version"], app_id), file=sys.stderr)
else:
print("Uploaded app %s/%s (%s) successfully" % (app_describe["name"], app_describe["version"], app_id), file=sys.stderr)
print("You can publish this app with:", file=sys.stderr)
print(" dx api app-%s/%s publish \"{\\\"makeDefault\\\": true}\"" % (app_describe["name"], app_describe["version"]), file=sys.stderr)
return app_describe if return_object_dump else {"id": app_id}
elif mode == "applet":
return dxpy.api.applet_describe(applet_id) if return_object_dump else {"id": applet_id}
else:
raise dxpy.app_builder.AppBuilderException("Unrecognized mode %r" % (mode,))
finally:
# Clean up after ourselves.
if using_temp_project:
delete_temporary_projects(projects_by_region.values())
def _build_app_remote(mode, src_dir, publish=False, destination_override=None,
version_override=None, bill_to_override=None, dx_toolkit_autodep="stable",
do_version_autonumbering=True, do_try_update=True, do_parallel_build=True,
do_check_syntax=True, region=None, watch=True):
if mode == 'app':
builder_app = 'app-tarball_app_builder'
else:
builder_app = 'app-tarball_applet_builder'
app_spec = _parse_app_spec(src_dir)
if app_spec['runSpec'].get('release') == '14.04':
builder_app += "_trusty"
temp_dir = tempfile.mkdtemp()
build_options = {'dx_toolkit_autodep': dx_toolkit_autodep}
if version_override:
build_options['version_override'] = version_override
elif do_version_autonumbering:
# If autonumbering is DISABLED, the interior run of dx-build-app
# will detect the correct version to use without our help. If it
# is ENABLED, the version suffix might depend on the state of
# the git repository. Since we'll remove the .git directory
# before uploading, we need to determine the correct version to
# use here and pass it in to the interior run of dx-build-app.
if do_version_autonumbering:
original_version = app_spec['version']
app_describe = None
try:
app_describe = dxpy.api.app_describe("app-" + app_spec["name"], alias=original_version, always_retry=False)
except dxpy.exceptions.DXAPIError as e:
if e.name == 'ResourceNotFound' or (mode == 'applet' and e.name == 'PermissionDenied'):
pass
else:
raise e
if app_describe is not None:
if 'published' in app_describe or not do_try_update:
# The version we wanted was taken; fall back to the
# autogenerated version number.
build_options['version_override'] = original_version + _get_version_suffix(src_dir, original_version)
# The following flags are basically passed through verbatim.
if bill_to_override:
build_options['bill_to_override'] = bill_to_override
if not do_version_autonumbering:
build_options['do_version_autonumbering'] = False
if not do_try_update:
build_options['do_try_update'] = False
if not do_parallel_build:
build_options['do_parallel_build'] = False
if not do_check_syntax:
build_options['do_check_syntax'] = False
using_temp_project_for_remote_build = False
# If building an applet, run the builder app in the destination
# project. If building an app, run the builder app in a temporary
# project.
dest_folder = None
dest_applet_name = None
if mode == "applet":
# Translate the --destination flag as follows. If --destination
# is PROJ:FOLDER/NAME,
#
# 1. Run the builder app in PROJ
# 2. Make the output folder FOLDER
# 3. Supply --destination=NAME to the interior call of dx-build-applet.
build_project_id = dxpy.WORKSPACE_ID
if destination_override:
build_project_id, dest_folder, dest_applet_name = parse_destination(destination_override)
if build_project_id is None:
parser.error("Can't create an applet without specifying a destination project; please use the -d/--destination flag to explicitly specify a project")
if dest_applet_name:
build_options['destination_override'] = '/' + dest_applet_name
elif mode == "app":
using_temp_project_for_remote_build = True
try:
project_input = {}
project_input["name"] = "dx-build-app --remote temporary project"
if bill_to_override:
project_input["billTo"] = bill_to_override
if region:
project_input["region"] = region
build_project_id = dxpy.api.project_new(project_input)["id"]
except:
err_exit()
try:
# Resolve relative paths and symlinks here so we have something
# reasonable to write in the job name below.
src_dir = os.path.realpath(src_dir)
# Show the user some progress as the tarball is being generated.
# Hopefully this will help them to understand when their tarball
# is huge (e.g. the target directory already has a whole bunch
# of binaries in it) and interrupt before uploading begins.
app_tarball_file = os.path.join(temp_dir, "app_tarball.tar.gz")
tar_subprocess = subprocess.Popen(["tar", "-czf", "-", "--exclude", "./.git", "."], cwd=src_dir, stdout=subprocess.PIPE)
with open(app_tarball_file, 'wb') as tar_output_file:
total_num_bytes = 0
last_console_update = 0
start_time = time.time()
printed_static_message = False
# Pipe the output of tar into the output file
while True:
tar_exitcode = tar_subprocess.poll()
data = tar_subprocess.stdout.read(4 * 1024 * 1024)
if tar_exitcode is not None and len(data) == 0:
break
tar_output_file.write(data)
total_num_bytes += len(data)
current_time = time.time()
# Don't show status messages at all for very short tar
# operations (< 1.0 sec)
if current_time - last_console_update > 0.25 and current_time - start_time > 1.0:
if sys.stderr.isatty():
if last_console_update > 0:
sys.stderr.write("\r")
sys.stderr.write("Compressing target directory {dir}... ({kb_so_far:,} kb)".format(dir=src_dir, kb_so_far=total_num_bytes // 1024))
sys.stderr.flush()
last_console_update = current_time
elif not printed_static_message:
# Print a message (once only) when stderr is not
# going to a live console
sys.stderr.write("Compressing target directory %s..." % (src_dir,))
printed_static_message = True
if last_console_update > 0:
sys.stderr.write("\n")
if tar_exitcode != 0:
raise Exception("tar exited with non-zero exit code " + str(tar_exitcode))
dxpy.set_workspace_id(build_project_id)
remote_file = dxpy.upload_local_file(app_tarball_file, media_type="application/gzip",
wait_on_close=True, show_progress=True)
try:
input_hash = {
"input_file": dxpy.dxlink(remote_file),
"build_options": build_options
}
if mode == 'app':
input_hash["publish"] = publish
api_options = {
"name": "Remote build of %s" % (os.path.basename(src_dir),),
"input": input_hash,
"project": build_project_id,
}
if dest_folder:
api_options["folder"] = dest_folder
app_run_result = dxpy.api.app_run(builder_app, input_params=api_options)
job_id = app_run_result["id"]
print("Started builder job %s" % (job_id,))
if watch:
try:
subprocess.check_call(["dx", "watch", job_id])
except subprocess.CalledProcessError as e:
if e.returncode == 3:
# Some kind of failure to build the app. The reason
# for the failure is probably self-evident from the
# job log (and if it's not, the CalledProcessError
# is not informative anyway), so just propagate the
# return code without additional remarks.
sys.exit(3)
else:
raise e
dxpy.DXJob(job_id).wait_on_done(interval=1)
if mode == 'applet':
applet_id, _ = dxpy.get_dxlink_ids(dxpy.api.job_describe(job_id)['output']['output_applet'])
return applet_id
else:
# TODO: determine and return the app ID, to allow
# running the app if args.run is specified
return None
finally:
if not using_temp_project_for_remote_build:
dxpy.DXProject(build_project_id).remove_objects([remote_file.get_id()])
finally:
if using_temp_project_for_remote_build:
dxpy.api.project_destroy(build_project_id, {"terminateJobs": True})
shutil.rmtree(temp_dir)
def build_and_upload_locally(src_dir,
mode,
overwrite=False,
archive=False,
publish=False,
destination_override=None,
version_override=None,
bill_to_override=None,
use_temp_build_project=True,
do_parallel_build=True,
do_version_autonumbering=True,
do_try_update=True,
dx_toolkit_autodep="stable",
do_build_step=True,
do_upload_step=True,
do_check_syntax=True,
dry_run=False,
return_object_dump=False,
**kwargs):
app_json = _parse_app_spec(src_dir)
_verify_app_source_dir(src_dir, enforce=do_check_syntax)
if mode == "app" and not dry_run:
_verify_app_writable(app_json['name'])
working_project = None
using_temp_project = False
override_folder = None
override_applet_name = None
if mode == "applet" and destination_override:
working_project, override_folder, override_applet_name = parse_destination(
destination_override)
elif mode == "app" and use_temp_build_project and not dry_run:
# Create a temp project
working_project = dxpy.api.project_new(
{"name": "Temporary build project for dx-build-app"})["id"]
print >> sys.stderr, "Created temporary project %s to build in" % (
working_project, )
using_temp_project = True
try:
if mode == "applet" and working_project is None and dxpy.WORKSPACE_ID is None:
parser.error(
"Can't create an applet without specifying a destination project; please use the -d/--destination flag to explicitly specify a project"
)
if "buildOptions" in app_json:
if app_json["buildOptions"].get("dx_toolkit_autodep") == False:
dx_toolkit_autodep = False
del app_json["buildOptions"]
if do_build_step:
dxpy.app_builder.build(src_dir, parallel_build=do_parallel_build)
if not do_upload_step:
return
bundled_resources = dxpy.app_builder.upload_resources(
src_dir, project=working_project,
folder=override_folder) if not dry_run else []
try:
# TODO: the "auto" setting is vestigial and should be removed.
if dx_toolkit_autodep == "auto":
dx_toolkit_autodep = "stable"
applet_id, applet_spec = dxpy.app_builder.upload_applet(
src_dir,
bundled_resources,
check_name_collisions=(mode == "applet"),
overwrite=overwrite and mode == "applet",
archive=archive and mode == "applet",
project=working_project,
override_folder=override_folder,
override_name=override_applet_name,
dx_toolkit_autodep=dx_toolkit_autodep,
dry_run=dry_run,
**kwargs)
except:
# Avoid leaking any bundled_resources files we may have
# created, if applet creation fails. Note that if
# using_temp_project, the entire project gets destroyed at
# the end, so we don't bother.
if not using_temp_project:
objects_to_delete = [
dxpy.get_dxlink_ids(bundled_resource_obj['id'])[0]
for bundled_resource_obj in bundled_resources
]
if objects_to_delete:
dxpy.api.project_remove_objects(
dxpy.app_builder.get_destination_project(
src_dir, project=working_project),
input_params={"objects": objects_to_delete})
raise
if dry_run:
return
applet_name = applet_spec['name']
print >> sys.stderr, "Created applet " + applet_id + " successfully"
if mode == "app":
if 'version' not in app_json:
parser.error(
"dxapp.json contains no \"version\" field, but it is required to build an app"
)
version = app_json['version']
try_versions = [version_override or version]
if not version_override and do_version_autonumbering:
try_versions.append(version +
_get_version_suffix(src_dir, version))
app_id = dxpy.app_builder.create_app(applet_id,
applet_name,
src_dir,
publish=publish,
set_default=publish,
billTo=bill_to_override,
try_versions=try_versions,
try_update=do_try_update)
app_describe = dxpy.api.app_describe(app_id)
if publish:
print >> sys.stderr, "Uploaded and published app %s/%s (%s) successfully" % (
app_describe["name"], app_describe["version"], app_id)
else:
print >> sys.stderr, "Uploaded app %s/%s (%s) successfully" % (
app_describe["name"], app_describe["version"], app_id)
print >> sys.stderr, "You can publish this app with:"
print >> sys.stderr, " dx api app-%s/%s publish \"{\\\"makeDefault\\\": true}\"" % (
app_describe["name"], app_describe["version"])
return app_describe if return_object_dump else {"id": app_id}
elif mode == "applet":
return dxpy.api.applet_describe(
applet_id) if return_object_dump else {
"id": applet_id
}
else:
raise dxpy.app_builder.AppBuilderException("Unrecognized mode %r" %
(mode, ))
finally:
# Clean up after ourselves.
if using_temp_project:
dxpy.api.project_destroy(working_project)
def build_and_upload_locally(src_dir, mode, overwrite=False, archive=False, publish=False, destination_override=None,
version_override=None, bill_to_override=None, use_temp_build_project=True,
do_parallel_build=True, do_version_autonumbering=True, do_try_update=True,
dx_toolkit_autodep="stable", do_check_syntax=True, dry_run=False,
return_object_dump=False, confirm=True, ensure_upload=False, **kwargs):
dxpy.app_builder.build(src_dir, parallel_build=do_parallel_build)
app_json = _parse_app_spec(src_dir)
_verify_app_source_dir(src_dir, mode, enforce=do_check_syntax)
if mode == "app" and not dry_run:
_verify_app_writable(app_json['name'])
working_project = None
using_temp_project = False
override_folder = None
override_applet_name = None
if mode == "applet" and destination_override:
working_project, override_folder, override_applet_name = parse_destination(destination_override)
elif mode == "app" and use_temp_build_project and not dry_run:
# Create a temp project
working_project = dxpy.api.project_new({"name": "Temporary build project for dx-build-app"})["id"]
logger.debug("Created temporary project %s to build in" % (working_project,))
using_temp_project = True
try:
if mode == "applet" and working_project is None and dxpy.WORKSPACE_ID is None:
parser.error("Can't create an applet without specifying a destination project; please use the -d/--destination flag to explicitly specify a project")
if "buildOptions" in app_json:
if app_json["buildOptions"].get("dx_toolkit_autodep") == False:
dx_toolkit_autodep = False
# Perform check for existence of applet with same name in
# destination for case in which neither "-f" nor "-a" is
# given BEFORE uploading resources.
if mode == "applet" and not overwrite and not archive:
try:
dest_name = override_applet_name or app_json.get('name') or os.path.basename(os.path.abspath(src_dir))
except:
raise dxpy.app_builder.AppBuilderException("Could not determine applet name from specification + "
"(dxapp.json) or from working directory (%r)" % (src_dir,))
dest_folder = override_folder or app_json.get('folder') or '/'
if not dest_folder.endswith('/'):
dest_folder = dest_folder + '/'
dest_project = working_project if working_project else dxpy.WORKSPACE_ID
for result in dxpy.find_data_objects(classname="applet", name=dest_name, folder=dest_folder,
project=dest_project, recurse=False):
dest_path = dest_folder + dest_name
msg = "An applet already exists at {} (id {}) and neither".format(dest_path, result["id"])
msg += " -f/--overwrite nor -a/--archive were given."
raise dxpy.app_builder.AppBuilderException(msg)
bundled_resources = dxpy.app_builder.upload_resources(src_dir,
project=working_project,
folder=override_folder,
ensure_upload=ensure_upload) if not dry_run else []
try:
# TODO: the "auto" setting is vestigial and should be removed.
if dx_toolkit_autodep == "auto":
dx_toolkit_autodep = "stable"
applet_id, applet_spec = dxpy.app_builder.upload_applet(
src_dir,
bundled_resources,
check_name_collisions=(mode == "applet"),
overwrite=overwrite and mode == "applet",
archive=archive and mode == "applet",
project=working_project,
override_folder=override_folder,
override_name=override_applet_name,
dx_toolkit_autodep=dx_toolkit_autodep,
dry_run=dry_run,
**kwargs)
except:
# Avoid leaking any bundled_resources files we may have
# created, if applet creation fails. Note that if
# using_temp_project, the entire project gets destroyed at
# the end, so we don't bother.
if not using_temp_project:
objects_to_delete = [dxpy.get_dxlink_ids(bundled_resource_obj['id'])[0] for bundled_resource_obj in bundled_resources]
if objects_to_delete:
dxpy.api.project_remove_objects(dxpy.app_builder.get_destination_project(src_dir, project=working_project),
input_params={"objects": objects_to_delete})
raise
if dry_run:
return
applet_name = applet_spec['name']
logger.debug("Created applet " + applet_id + " successfully")
if mode == "app":
if 'version' not in app_json:
parser.error("dxapp.json contains no \"version\" field, but it is required to build an app")
version = app_json['version']
try_versions = [version_override or version]
if not version_override and do_version_autonumbering:
try_versions.append(version + _get_version_suffix(src_dir, version))
app_id = dxpy.app_builder.create_app(applet_id,
applet_name,
src_dir,
publish=publish,
set_default=publish,
billTo=bill_to_override,
try_versions=try_versions,
try_update=do_try_update,
confirm=confirm)
app_describe = dxpy.api.app_describe(app_id)
if publish:
print("Uploaded and published app %s/%s (%s) successfully" % (app_describe["name"], app_describe["version"], app_id), file=sys.stderr)
else:
print("Uploaded app %s/%s (%s) successfully" % (app_describe["name"], app_describe["version"], app_id), file=sys.stderr)
print("You can publish this app with:", file=sys.stderr)
print(" dx api app-%s/%s publish \"{\\\"makeDefault\\\": true}\"" % (app_describe["name"], app_describe["version"]), file=sys.stderr)
return app_describe if return_object_dump else {"id": app_id}
elif mode == "applet":
return dxpy.api.applet_describe(applet_id) if return_object_dump else {"id": applet_id}
else:
raise dxpy.app_builder.AppBuilderException("Unrecognized mode %r" % (mode,))
finally:
# Clean up after ourselves.
if using_temp_project:
dxpy.api.project_destroy(working_project)
def main(**kwargs):
if len(kwargs) == 0:
args = parser.parse_args()
else:
args = parser.parse_args(kwargs)
if dxpy.AUTH_HELPER is None:
parser.error('Authentication required to build an executable on the platform; please run "dx login" first')
if args.src_dir is None:
args.src_dir = os.getcwd()
if not os.path.isdir(args.src_dir):
parser.error("%s is not a directory" % args.src_dir)
if not os.path.exists(os.path.join(args.src_dir, "dxapp.json")):
parser.error("Directory %s does not contain dxapp.json: not a valid DNAnexus app source directory" % args.src_dir)
if args.mode == "app" and args.destination:
parser.error("--destination cannot be used when creating an app (only an applet)")
if args.remote:
# The following flags might be useful in conjunction with
# --remote. To enable these, we need to learn how to pass these
# options through to the interior call of dx_build_app(let).
if args.dry_run:
parser.error('--remote cannot be combined with --dry-run')
# The following flags are probably not useful in conjunction
# with --remote.
if not args.build_step:
parser.error('--remote cannot be combined with --no-build-step')
if not args.upload_step:
parser.error('--remote cannot be combined with --no-upload-step')
if args.json:
parser.error('--remote cannot be combined with --json')
if not args.use_temp_build_project:
parser.error('--remote cannot be combined with --no-temp-build-project')
more_kwargs = {}
if args.version_override:
more_kwargs['version_override'] = args.version_override
if args.bill_to:
more_kwargs['bill_to'] = args.bill_to
if not args.version_autonumbering:
more_kwargs['version_autonumbering'] = False
if not args.update:
more_kwargs['update'] = False
if not args.parallel_build:
more_kwargs['parallel_build'] = False
return _build_app_remote(args.mode, args.src_dir, destination=args.destination, publish=args.publish, dx_toolkit_autodep=args.dx_toolkit_autodep, **more_kwargs)
working_project = None
using_temp_project = False
override_folder = None
override_applet_name = None
if args.mode == "applet" and args.destination:
working_project, override_folder, override_applet_name = parse_destination(args.destination)
elif args.mode == "app" and args.use_temp_build_project and not args.dry_run:
# Create a temp project
working_project = dxpy.api.projectNew({"name": "Temporary build project for dx-build-app"})["id"]
print >> sys.stderr, "Created temporary project %s to build in" % (working_project,)
using_temp_project = True
if args.mode == "applet" and working_project is None and dxpy.WORKSPACE_ID is None:
parser.error("Can't create an applet without specifying a destination project; please use the -d/--destination flag to explicitly specify a project")
try:
with open(os.path.join(args.src_dir, "dxapp.json")) as app_desc:
try:
app_json = json.load(app_desc)
except:
parser.error("Could not parse dxapp.json file as valid JSON")
if "buildOptions" in app_json:
if app_json["buildOptions"].get("dx_toolkit_autodep") == False:
args.dx_toolkit_autodep = False
del app_json["buildOptions"]
if args.build_step:
dxpy.app_builder.build(args.src_dir, parallel_build=args.parallel_build)
if not args.upload_step:
return
bundled_resources = dxpy.app_builder.upload_resources(args.src_dir, project=working_project) if not args.dry_run else []
try:
if args.dx_toolkit_autodep == "auto":
# "auto" (the default) means dx-toolkit (stable) on preprod and prod,
# and dx-toolkit-beta on all other systems.
if dxpy.APISERVER_HOST == "preprodapi.dnanexus.com" or dxpy.APISERVER_HOST == "api.dnanexus.com":
args.dx_toolkit_autodep = "stable"
else:
args.dx_toolkit_autodep = "beta"
applet_id, applet_spec = dxpy.app_builder.upload_applet(
args.src_dir,
bundled_resources,
check_name_collisions=(args.mode == "applet"),
overwrite=args.overwrite and args.mode == "applet",
project=working_project,
override_folder=override_folder,
override_name=override_applet_name,
dx_toolkit_autodep=args.dx_toolkit_autodep,
dry_run=args.dry_run)
except:
# Avoid leaking any bundled_resources files we may have
# created, if applet creation fails. Note that if
# using_temp_project, the entire project gets destroyed at
# the end, so we don't bother.
if not using_temp_project:
objects_to_delete = [dxpy.get_dxlink_ids(bundled_resource_obj['id'])[0] for bundled_resource_obj in bundled_resources]
if objects_to_delete:
dxpy.api.projectRemoveObjects(dxpy.app_builder.get_destination_project(args.src_dir, project=working_project),
input_params={"objects": objects_to_delete})
raise
if args.dry_run:
return
applet_name = applet_spec['name']
print >> sys.stderr, "Created applet " + applet_id + " successfully"
if args.mode == "app":
if 'version' not in app_json:
parser.error("dxapp.json contains no \"version\" field, but it is required to build an app")
version = app_json['version']
try_versions = [args.version_override or version]
if not args.version_override and args.version_autonumbering:
try_versions.append(version + get_version_suffix(args.src_dir))
app_id = dxpy.app_builder.create_app(applet_id,
applet_name,
args.src_dir,
publish=args.publish,
set_default=args.publish,
billTo=args.bill_to,
try_versions=try_versions,
try_update=args.update)
app_describe = dxpy.api.appDescribe(app_id)
if args.json:
print json.dumps(app_describe)
if args.publish:
print >> sys.stderr, "Uploaded and published app %s/%s (%s) successfully" % (app_describe["name"], app_describe["version"], app_id)
else:
print >> sys.stderr, "Uploaded app %s/%s (%s) successfully" % (app_describe["name"], app_describe["version"], app_id)
print >> sys.stderr, "You can publish this app with:"
print >> sys.stderr, " dx api app-%s/%s publish \"{\\\"makeDefault\\\": true}\"" % (app_describe["name"], app_describe["version"])
elif args.mode == "applet":
if args.json:
print json.dumps(dxpy.api.appletDescribe(applet_id))
else:
raise dxpy.app_builder.AppBuilderException("Unrecognized mode %r" % (args.mode,))
except dxpy.app_builder.AppBuilderException as e:
# AppBuilderException represents errors during app or applet building
# that could reasonably have been anticipated by the user.
print >> sys.stderr, "Error: %s" % (e.message,)
sys.exit(3)
finally:
# Clean up after ourselves.
if using_temp_project:
dxpy.api.projectDestroy(working_project)
