def unpack_tar(self, tar_file_dxlink): ''' DEV: Eventually integrate dx-toolkit into trajectoread repo so I can transition to using 'dx-download-all-inputs' to handle unpacking all input files. Pipeline used to store lane file dxids as project properties and then pass to "dx download" Description: Download and untar metadata and lane data files (/Data/Intensities/BaseCalls) ''' if dxpy.is_dxlink(tar_file_dxlink): file_handler = dxpy.get_handler(tar_file_dxlink) filename = file_handler.name else: print 'Error: Cannot unpack %s; not a valid DXLink object' sys.exit() # ('file-dxid', 'project-dxid') = dxpy.get_dxlink_ids(dxlink) file_dxid = dxpy.get_dxlink_ids(tar_file_dxlink)[0] project_id = dxpy.get_dxlink_ids(tar_file_dxlink)[1] # Download file from DNAnexus objectstore to virtual machine dxpy.download_dxfile(dxid=file_dxid, filename=filename, project=project_id) # Untar file ## DEV: Check if this is even in use anymore; also should have some method for ## checking what type of compression was used. ## But I don't think this is in use command = 'tar -xf %s --owner root --group root --no-same-owner' % filename self.createSubprocess(cmd=command, pipeStdout=False)
def format_data_object_reference(item): if dxpy.is_dxlink(item): # Bare dxlink obj_id, proj_id = dxpy.get_dxlink_ids(item) return (proj_id + ":" if proj_id else '') + obj_id if dxpy.is_dxlink(item.get('value')): # value is set obj_id, proj_id = dxpy.get_dxlink_ids(item['value']) return (proj_id + ":" if proj_id else '') + obj_id + (' (%s)' % item['name'] if item.get('name') else '') if item.get('project') and item.get('path'): # project and folder path return item['project'] + ':' + item['path'] + "/" + obj_class + "-*" + (' (%s)' % item['name'] if item.get('name') else '') return str(item)
def build_asset(args): if args.src_dir is None: args.src_dir = os.getcwd() dest_project_name = None dest_folder_name = None dest_asset_name = None make_file = None asset_file = None conf_file = None try: asset_conf = parse_asset_spec(args.src_dir) validate_conf(asset_conf) asset_conf_file = os.path.join(args.src_dir, "dxasset.json") dxpy.api.system_whoami() dest_project_name, dest_folder_name, dest_asset_name = parse_destination(args.destination) if dest_project_name is None: raise AssetBuilderException("Can't build an asset without specifying a destination project; \ please use the -d/--destination flag to explicitly specify a project") if dest_asset_name is None: dest_asset_name = asset_conf['name'] # If dx build_asset is launched form a job, set json flag to True to avoid watching the job log if dxpy.JOB_ID: args.json = True if not args.json: print("Uploading input files for the AssetBuilder", file=sys.stderr) conf_file = dx_upload(asset_conf_file, dest_project_name, dest_folder_name, args.json) make_file = get_asset_make(args.src_dir, dest_project_name, dest_folder_name, args.json) asset_file = get_asset_tarball(asset_conf['name'], args.src_dir, dest_project_name, dest_folder_name, args.json) input_hash = {"conf_json": dxpy.dxlink(conf_file)} if asset_file: input_hash["custom_asset"] = dxpy.dxlink(asset_file) if make_file: input_hash["asset_makefile"] = dxpy.dxlink(make_file) builder_run_options = { "name": dest_asset_name, "input": input_hash } # Add the default destination project to app run options, if it is not run from a job if not dxpy.JOB_ID: builder_run_options["project"] = dest_project_name if 'instanceType' in asset_conf: builder_run_options["systemRequirements"] = {"*": {"instanceType": asset_conf["instanceType"]}} if dest_folder_name: builder_run_options["folder"] = dest_folder_name if asset_conf['release'] == "12.04": app_run_result = dxpy.api.app_run(ASSET_BUILDER_PRECISE, input_params=builder_run_options) elif asset_conf['release'] == "14.04": app_run_result = dxpy.api.app_run(ASSET_BUILDER_TRUSTY, input_params=builder_run_options) job_id = app_run_result["id"] if not args.json: print("\nStarted job '" + str(job_id) + "' to build the asset bundle.\n", file=sys.stderr) if args.watch: try: subprocess.check_call(["dx", "watch", job_id]) except subprocess.CalledProcessError as e: if e.returncode == 3: # Some kind of failure to build the asset. The reason # for the failure is probably self-evident from the # job log (and if it's not, the CalledProcessError # is not informative anyway), so just propagate the # return code without additional remarks. sys.exit(3) else: raise e dxpy.DXJob(job_id).wait_on_done(interval=1) asset_id, _ = dxpy.get_dxlink_ids(dxpy.api.job_describe(job_id)['output']['asset_bundle']) if args.json: print(json.dumps({"id": asset_id})) else: print("\nAsset bundle '" + asset_id + "' is built and can now be used in your app/applet's dxapp.json\n", file=sys.stderr) except Exception as de: print(de.__class__.__name__ + ": " + str(de), file=sys.stderr) sys.exit(1) finally: if conf_file: try: conf_file.remove() except: pass if make_file: try: make_file.remove() except: pass if asset_file: try: asset_file.remove() except: pass
def build_and_upload_locally( src_dir, mode, overwrite=False, archive=False, publish=False, destination_override=None, version_override=None, bill_to_override=None, use_temp_build_project=True, do_parallel_build=True, do_version_autonumbering=True, do_try_update=True, dx_toolkit_autodep="stable", do_check_syntax=True, dry_run=False, return_object_dump=False, confirm=True, ensure_upload=False, region=None, **kwargs ): dxpy.app_builder.build(src_dir, parallel_build=do_parallel_build) app_json = _parse_app_spec(src_dir) _verify_app_source_dir(src_dir, mode, enforce=do_check_syntax) if mode == "app" and not dry_run: _verify_app_writable(app_json["name"]) working_project = None using_temp_project = False override_folder = None override_applet_name = None if mode == "applet" and destination_override: working_project, override_folder, override_applet_name = parse_destination(destination_override) elif mode == "app" and use_temp_build_project and not dry_run: # Create a temp project try: if region: working_project = dxpy.api.project_new( {"name": "Temporary build project for dx-build-app", "region": region} )["id"] else: working_project = dxpy.api.project_new({"name": "Temporary build project for dx-build-app"})["id"] except: err_exit() logger.debug("Created temporary project %s to build in" % (working_project,)) using_temp_project = True try: if mode == "applet" and working_project is None and dxpy.WORKSPACE_ID is None: parser.error( "Can't create an applet without specifying a destination project; please use the -d/--destination flag to explicitly specify a project" ) if "buildOptions" in app_json: if app_json["buildOptions"].get("dx_toolkit_autodep") == False: dx_toolkit_autodep = False # Perform check for existence of applet with same name in # destination for case in which neither "-f" nor "-a" is # given BEFORE uploading resources. if mode == "applet" and not overwrite and not archive: try: dest_name = override_applet_name or app_json.get("name") or os.path.basename(os.path.abspath(src_dir)) except: raise dxpy.app_builder.AppBuilderException( "Could not determine applet name from specification + " "(dxapp.json) or from working directory (%r)" % (src_dir,) ) dest_folder = override_folder or app_json.get("folder") or "/" if not dest_folder.endswith("/"): dest_folder = dest_folder + "/" dest_project = working_project if working_project else dxpy.WORKSPACE_ID for result in dxpy.find_data_objects( classname="applet", name=dest_name, folder=dest_folder, project=dest_project, recurse=False ): dest_path = dest_folder + dest_name msg = "An applet already exists at {} (id {}) and neither".format(dest_path, result["id"]) msg += " -f/--overwrite nor -a/--archive were given." raise dxpy.app_builder.AppBuilderException(msg) bundled_resources = ( dxpy.app_builder.upload_resources( src_dir, project=working_project, folder=override_folder, ensure_upload=ensure_upload ) if not dry_run else [] ) try: # TODO: the "auto" setting is vestigial and should be removed. if dx_toolkit_autodep == "auto": dx_toolkit_autodep = "stable" applet_id, applet_spec = dxpy.app_builder.upload_applet( src_dir, bundled_resources, check_name_collisions=(mode == "applet"), overwrite=overwrite and mode == "applet", archive=archive and mode == "applet", project=working_project, override_folder=override_folder, override_name=override_applet_name, dx_toolkit_autodep=dx_toolkit_autodep, dry_run=dry_run, **kwargs ) except: # Avoid leaking any bundled_resources files we may have # created, if applet creation fails. Note that if # using_temp_project, the entire project gets destroyed at # the end, so we don't bother. if not using_temp_project: objects_to_delete = [ dxpy.get_dxlink_ids(bundled_resource_obj["id"])[0] for bundled_resource_obj in bundled_resources ] if objects_to_delete: dxpy.api.project_remove_objects( dxpy.app_builder.get_destination_project(src_dir, project=working_project), input_params={"objects": objects_to_delete}, ) raise if dry_run: return applet_name = applet_spec["name"] logger.debug("Created applet " + applet_id + " successfully") if mode == "app": if "version" not in app_json: parser.error('dxapp.json contains no "version" field, but it is required to build an app') version = app_json["version"] try_versions = [version_override or version] if not version_override and do_version_autonumbering: try_versions.append(version + _get_version_suffix(src_dir, version)) app_id = dxpy.app_builder.create_app( applet_id, applet_name, src_dir, publish=publish, set_default=publish, billTo=bill_to_override, try_versions=try_versions, try_update=do_try_update, confirm=confirm, ) app_describe = dxpy.api.app_describe(app_id) if publish: print( "Uploaded and published app %s/%s (%s) successfully" % (app_describe["name"], app_describe["version"], app_id), file=sys.stderr, ) else: print( "Uploaded app %s/%s (%s) successfully" % (app_describe["name"], app_describe["version"], app_id), file=sys.stderr, ) print("You can publish this app with:", file=sys.stderr) print( ' dx api app-%s/%s publish "{\\"makeDefault\\": true}"' % (app_describe["name"], app_describe["version"]), file=sys.stderr, ) return app_describe if return_object_dump else {"id": app_id} elif mode == "applet": return dxpy.api.applet_describe(applet_id) if return_object_dump else {"id": applet_id} else: raise dxpy.app_builder.AppBuilderException("Unrecognized mode %r" % (mode,)) finally: # Clean up after ourselves. if using_temp_project: dxpy.api.project_destroy(working_project)
def _build_app_remote( mode, src_dir, publish=False, destination_override=None, version_override=None, bill_to_override=None, dx_toolkit_autodep="stable", do_version_autonumbering=True, do_try_update=True, do_parallel_build=True, do_check_syntax=True, region=None, ): if mode == "app": builder_app = "app-tarball_app_builder" else: builder_app = "app-tarball_applet_builder" temp_dir = tempfile.mkdtemp() # TODO: this is vestigial, the "auto" setting should be removed. if dx_toolkit_autodep == "auto": dx_toolkit_autodep = "stable" build_options = {"dx_toolkit_autodep": dx_toolkit_autodep} if version_override: build_options["version_override"] = version_override elif do_version_autonumbering: # If autonumbering is DISABLED, the interior run of dx-build-app # will detect the correct version to use without our help. If it # is ENABLED, the version suffix might depend on the state of # the git repository. Since we'll remove the .git directory # before uploading, we need to determine the correct version to # use here and pass it in to the interior run of dx-build-app. if do_version_autonumbering: app_spec = _parse_app_spec(src_dir) original_version = app_spec["version"] app_describe = None try: app_describe = dxpy.api.app_describe( "app-" + app_spec["name"], alias=original_version, always_retry=False ) except dxpy.exceptions.DXAPIError as e: if e.name == "ResourceNotFound" or (mode == "applet" and e.name == "PermissionDenied"): pass else: raise e if app_describe is not None: if "published" in app_describe or not do_try_update: # The version we wanted was taken; fall back to the # autogenerated version number. build_options["version_override"] = original_version + _get_version_suffix( src_dir, original_version ) # The following flags are basically passed through verbatim. if bill_to_override: build_options["bill_to_override"] = bill_to_override if not do_version_autonumbering: build_options["do_version_autonumbering"] = False if not do_try_update: build_options["do_try_update"] = False if not do_parallel_build: build_options["do_parallel_build"] = False if not do_check_syntax: build_options["do_check_syntax"] = False using_temp_project_for_remote_build = False # If building an applet, run the builder app in the destination # project. If building an app, run the builder app in a temporary # project. dest_folder = None dest_applet_name = None if mode == "applet": # Translate the --destination flag as follows. If --destination # is PROJ:FOLDER/NAME, # # 1. Run the builder app in PROJ # 2. Make the output folder FOLDER # 3. Supply --destination=NAME to the interior call of dx-build-applet. build_project_id = dxpy.WORKSPACE_ID if destination_override: build_project_id, dest_folder, dest_applet_name = parse_destination(destination_override) if build_project_id is None: parser.error( "Can't create an applet without specifying a destination project; please use the -d/--destination flag to explicitly specify a project" ) if dest_applet_name: build_options["destination_override"] = "/" + dest_applet_name elif mode == "app": using_temp_project_for_remote_build = True try: if region: build_project_id = dxpy.api.project_new( {"name": "dx-build-app --remote temporary project", "region": region} )["id"] else: build_project_id = dxpy.api.project_new({"name": "dx-build-app --remote temporary project"})["id"] except: err_exit() try: # Resolve relative paths and symlinks here so we have something # reasonable to write in the job name below. src_dir = os.path.realpath(src_dir) # Show the user some progress as the tarball is being generated. # Hopefully this will help them to understand when their tarball # is huge (e.g. the target directory already has a whole bunch # of binaries in it) and interrupt before uploading begins. app_tarball_file = os.path.join(temp_dir, "app_tarball.tar.gz") tar_subprocess = subprocess.Popen( ["tar", "-czf", "-", "--exclude", "./.git", "."], cwd=src_dir, stdout=subprocess.PIPE ) with open(app_tarball_file, "wb") as tar_output_file: total_num_bytes = 0 last_console_update = 0 start_time = time.time() printed_static_message = False # Pipe the output of tar into the output file while True: tar_exitcode = tar_subprocess.poll() data = tar_subprocess.stdout.read(4 * 1024 * 1024) if tar_exitcode is not None and len(data) == 0: break tar_output_file.write(data) total_num_bytes += len(data) current_time = time.time() # Don't show status messages at all for very short tar # operations (< 1.0 sec) if current_time - last_console_update > 0.25 and current_time - start_time > 1.0: if sys.stderr.isatty(): if last_console_update > 0: sys.stderr.write("\r") sys.stderr.write( "Compressing target directory {dir}... ({kb_so_far:,} kb)".format( dir=src_dir, kb_so_far=total_num_bytes // 1024 ) ) sys.stderr.flush() last_console_update = current_time elif not printed_static_message: # Print a message (once only) when stderr is not # going to a live console sys.stderr.write("Compressing target directory %s..." % (src_dir,)) printed_static_message = True if last_console_update > 0: sys.stderr.write("\n") if tar_exitcode != 0: raise Exception("tar exited with non-zero exit code " + str(tar_exitcode)) dxpy.set_workspace_id(build_project_id) remote_file = dxpy.upload_local_file( app_tarball_file, media_type="application/gzip", wait_on_close=True, show_progress=True ) try: input_hash = {"input_file": dxpy.dxlink(remote_file), "build_options": build_options} if mode == "app": input_hash["publish"] = publish api_options = { "name": "Remote build of %s" % (os.path.basename(src_dir),), "input": input_hash, "project": build_project_id, } if dest_folder: api_options["folder"] = dest_folder app_run_result = dxpy.api.app_run(builder_app, input_params=api_options) job_id = app_run_result["id"] print("Started builder job %s" % (job_id,)) try: subprocess.check_call(["dx", "watch", job_id]) except subprocess.CalledProcessError as e: if e.returncode == 3: # Some kind of failure to build the app. The reason # for the failure is probably self-evident from the # job log (and if it's not, the CalledProcessError # is not informative anyway), so just propagate the # return code without additional remarks. sys.exit(3) else: raise e dxpy.DXJob(job_id).wait_on_done(interval=1) if mode == "applet": applet_id, _ = dxpy.get_dxlink_ids(dxpy.api.job_describe(job_id)["output"]["output_applet"]) return applet_id else: # TODO: determine and return the app ID, to allow # running the app if args.run is specified return None finally: if not using_temp_project_for_remote_build: dxpy.DXProject(build_project_id).remove_objects([remote_file.get_id()]) finally: if using_temp_project_for_remote_build: dxpy.api.project_destroy(build_project_id, {"terminateJobs": True}) shutil.rmtree(temp_dir)
def build_and_upload_locally(src_dir, mode, overwrite=False, publish=False, destination_override=None, version_override=None, bill_to_override=None, use_temp_build_project=True, do_parallel_build=True, do_version_autonumbering=True, do_try_update=True, dx_toolkit_autodep="stable", do_build_step=True, do_upload_step=True, do_check_syntax=True, dry_run=False, return_object_dump=False): app_json = _parse_app_spec(src_dir) _verify_app_source_dir(src_dir, enforce=do_check_syntax) working_project = None using_temp_project = False override_folder = None override_applet_name = None if mode == "applet" and destination_override: working_project, override_folder, override_applet_name = parse_destination(destination_override) elif mode == "app" and use_temp_build_project and not dry_run: # Create a temp project working_project = dxpy.api.project_new({"name": "Temporary build project for dx-build-app"})["id"] print >> sys.stderr, "Created temporary project %s to build in" % (working_project,) using_temp_project = True try: if mode == "applet" and working_project is None and dxpy.WORKSPACE_ID is None: parser.error("Can't create an applet without specifying a destination project; please use the -d/--destination flag to explicitly specify a project") if "buildOptions" in app_json: if app_json["buildOptions"].get("dx_toolkit_autodep") == False: dx_toolkit_autodep = False del app_json["buildOptions"] if do_build_step: dxpy.app_builder.build(src_dir, parallel_build=do_parallel_build) if not do_upload_step: return bundled_resources = dxpy.app_builder.upload_resources(src_dir, project=working_project) if not dry_run else [] try: # TODO: the "auto" setting is vestigial and should be removed. if dx_toolkit_autodep == "auto": dx_toolkit_autodep = "stable" applet_id, applet_spec = dxpy.app_builder.upload_applet( src_dir, bundled_resources, check_name_collisions=(mode == "applet"), overwrite=overwrite and mode == "applet", project=working_project, override_folder=override_folder, override_name=override_applet_name, dx_toolkit_autodep=dx_toolkit_autodep, dry_run=dry_run) except: # Avoid leaking any bundled_resources files we may have # created, if applet creation fails. Note that if # using_temp_project, the entire project gets destroyed at # the end, so we don't bother. if not using_temp_project: objects_to_delete = [dxpy.get_dxlink_ids(bundled_resource_obj['id'])[0] for bundled_resource_obj in bundled_resources] if objects_to_delete: dxpy.api.project_remove_objects(dxpy.app_builder.get_destination_project(src_dir, project=working_project), input_params={"objects": objects_to_delete}) raise if dry_run: return applet_name = applet_spec['name'] print >> sys.stderr, "Created applet " + applet_id + " successfully" if mode == "app": if 'version' not in app_json: parser.error("dxapp.json contains no \"version\" field, but it is required to build an app") version = app_json['version'] try_versions = [version_override or version] if not version_override and do_version_autonumbering: try_versions.append(version + _get_version_suffix(src_dir, version)) app_id = dxpy.app_builder.create_app(applet_id, applet_name, src_dir, publish=publish, set_default=publish, billTo=bill_to_override, try_versions=try_versions, try_update=do_try_update) app_describe = dxpy.api.app_describe(app_id) if publish: print >> sys.stderr, "Uploaded and published app %s/%s (%s) successfully" % (app_describe["name"], app_describe["version"], app_id) else: print >> sys.stderr, "Uploaded app %s/%s (%s) successfully" % (app_describe["name"], app_describe["version"], app_id) print >> sys.stderr, "You can publish this app with:" print >> sys.stderr, " dx api app-%s/%s publish \"{\\\"makeDefault\\\": true}\"" % (app_describe["name"], app_describe["version"]) return app_describe if return_object_dump else None elif mode == "applet": return dxpy.api.applet_describe(applet_id) if return_object_dump else None else: raise dxpy.app_builder.AppBuilderException("Unrecognized mode %r" % (mode,)) finally: # Clean up after ourselves. if using_temp_project: dxpy.api.project_destroy(working_project)
def build_asset(args): if args.src_dir is None: args.src_dir = os.getcwd() dest_project_name = None dest_folder_name = None dest_asset_name = None make_file = None asset_file = None conf_file = None try: asset_conf = parse_asset_spec(args.src_dir) validate_conf(asset_conf) asset_conf_file = os.path.join(args.src_dir, "dxasset.json") dxpy.api.system_whoami() dest_project_name, dest_folder_name, dest_asset_name = parse_destination(args.destination) if dest_project_name is None: raise AssetBuilderException("Can't build an asset without specifying a destination project; \ please use the -d/--destination flag to explicitly specify a project") if dest_asset_name is None: dest_asset_name = asset_conf['name'] # If dx build_asset is launched form a job, set json flag to True to avoid watching the job log if dxpy.JOB_ID: args.json = True if not args.json: print("Uploading input files for the AssetBuilder", file=sys.stderr) conf_file = dx_upload(asset_conf_file, dest_project_name, dest_folder_name, args.json) make_file = get_asset_make(args.src_dir, dest_project_name, dest_folder_name, args.json) asset_file = get_asset_tarball(asset_conf['name'], args.src_dir, dest_project_name, dest_folder_name, args.json) input_hash = {"conf_json": dxpy.dxlink(conf_file)} if asset_file: input_hash["custom_asset"] = dxpy.dxlink(asset_file) if make_file: input_hash["asset_makefile"] = dxpy.dxlink(make_file) builder_run_options = { "name": dest_asset_name, "input": input_hash } if args.priority is not None: builder_run_options["priority"] = args.priority # Add the default destination project to app run options, if it is not run from a job if not dxpy.JOB_ID: builder_run_options["project"] = dest_project_name if 'instanceType' in asset_conf: builder_run_options["systemRequirements"] = {"*": {"instanceType": asset_conf["instanceType"]}} if dest_folder_name: builder_run_options["folder"] = dest_folder_name if asset_conf['release'] == "12.04": app_run_result = dxpy.api.app_run(ASSET_BUILDER_PRECISE, input_params=builder_run_options) elif asset_conf['release'] == "14.04": app_run_result = dxpy.api.app_run(ASSET_BUILDER_TRUSTY, input_params=builder_run_options) elif asset_conf['release'] == "16.04" and asset_conf['runSpecVersion'] == '1': app_run_result = dxpy.api.app_run(ASSET_BUILDER_XENIAL_V1, input_params=builder_run_options) elif asset_conf['release'] == "16.04": app_run_result = dxpy.api.app_run(ASSET_BUILDER_XENIAL, input_params=builder_run_options) elif asset_conf['release'] == "20.04": app_run_result = dxpy.api.app_run(ASSET_BUILDER_FOCAL, input_params=builder_run_options) job_id = app_run_result["id"] if not args.json: print("\nStarted job '" + str(job_id) + "' to build the asset bundle.\n", file=sys.stderr) if args.watch: try: subprocess.check_call(["dx", "watch", job_id]) except subprocess.CalledProcessError as e: if e.returncode == 3: # Some kind of failure to build the asset. The reason # for the failure is probably self-evident from the # job log (and if it's not, the CalledProcessError # is not informative anyway), so just propagate the # return code without additional remarks. sys.exit(3) else: raise e dxpy.DXJob(job_id).wait_on_done(interval=1) asset_id, _ = dxpy.get_dxlink_ids(dxpy.api.job_describe(job_id)['output']['asset_bundle']) if args.json: print(json.dumps({"id": asset_id})) else: print("\nAsset bundle '" + asset_id + "' is built and can now be used in your app/applet's dxapp.json\n", file=sys.stderr) except Exception as de: print(de.__class__.__name__ + ": " + str(de), file=sys.stderr) sys.exit(1) finally: if conf_file: try: conf_file.remove() except: pass if make_file: try: make_file.remove() except: pass if asset_file: try: asset_file.remove() except: pass
def build_and_upload_locally(src_dir, mode, overwrite=False, archive=False, publish=False, destination_override=None, version_override=None, bill_to_override=None, use_temp_build_project=True, do_parallel_build=True, do_version_autonumbering=True, do_try_update=True, dx_toolkit_autodep="stable", do_check_syntax=True, dry_run=False, return_object_dump=False, confirm=True, ensure_upload=False, force_symlinks=False, region=None, **kwargs): dxpy.app_builder.build(src_dir, parallel_build=do_parallel_build) app_json = _parse_app_spec(src_dir) _check_suggestions(app_json, publish=publish) _verify_app_source_dir(src_dir, mode, enforce=do_check_syntax) if mode == "app" and not dry_run: _verify_app_writable(app_json['name']) working_project = None using_temp_project = False override_folder = None override_applet_name = None enabled_regions = dxpy.app_builder.get_enabled_regions(app_json, region) # Cannot build multi-region app if `use_temp_build_project` is falsy. if enabled_regions is not None and len(enabled_regions) > 1 and not use_temp_build_project: raise dxpy.app_builder.AppBuilderException("Cannot specify --no-temp-build-project when building multi-region apps") projects_by_region = None if mode == "applet" and destination_override: working_project, override_folder, override_applet_name = parse_destination(destination_override) region = dxpy.api.project_describe(working_project, input_params={"fields": {"region": True}})["region"] projects_by_region = {region: working_project} elif mode == "app" and use_temp_build_project and not dry_run: projects_by_region = {} if enabled_regions is not None: # Create temporary projects in each enabled region. try: for region in enabled_regions: project_input = { "name": "Temporary build project for dx-build-app in {r}".format(r=region), "region": region } if bill_to_override: project_input["billTo"] = bill_to_override working_project = dxpy.api.project_new(project_input)["id"] projects_by_region[region] = working_project logger.debug("Created temporary project %s to build in" % (working_project,)) except: # A /project/new request may fail if the requesting user is # not authorized to create projects in a certain region. delete_temporary_projects(projects_by_region.values()) err_exit() else: # Create a temp project try: project_input = {"name": "Temporary build project for dx-build-app"} if bill_to_override: project_input["billTo"] = bill_to_override working_project = dxpy.api.project_new(project_input)["id"] except: err_exit() region = dxpy.api.project_describe(working_project, input_params={"fields": {"region": True}})["region"] projects_by_region[region] = working_project logger.debug("Created temporary project %s to build in" % (working_project,)) using_temp_project = True elif mode == "app" and not dry_run: # If we are not using temporary project(s) to build the executable, # then we should have a project context somewhere. try: project = app_json.get("project", dxpy.WORKSPACE_ID) region = dxpy.api.project_describe(project, input_params={"fields": {"region": True}})["region"] except Exception: err_exit() projects_by_region = {region: project} try: if mode == "applet" and working_project is None and dxpy.WORKSPACE_ID is None: parser.error("Can't create an applet without specifying a destination project; please use the -d/--destination flag to explicitly specify a project") if mode == "applet": dest_project = working_project or dxpy.WORKSPACE_ID or app_json.get("project", False) try: region = dxpy.api.project_describe(dest_project, input_params={"fields": {"region": True}})["region"] except Exception: err_exit() projects_by_region = {region: dest_project} if not overwrite and not archive: # If we cannot overwite or archive an existing applet and an # applet in the destination exists with the same name as this # one, then we should err out *before* uploading resources. try: dest_name = override_applet_name or app_json.get('name') or os.path.basename(os.path.abspath(src_dir)) except: raise dxpy.app_builder.AppBuilderException("Could not determine applet name from specification + " "(dxapp.json) or from working directory (%r)" % (src_dir,)) dest_folder = override_folder or app_json.get('folder') or '/' if not dest_folder.endswith('/'): dest_folder = dest_folder + '/' for result in dxpy.find_data_objects(classname="applet", name=dest_name, folder=dest_folder, project=dest_project, recurse=False): dest_path = dest_folder + dest_name msg = "An applet already exists at {} (id {}) and neither".format(dest_path, result["id"]) msg += " -f/--overwrite nor -a/--archive were given." raise dxpy.app_builder.AppBuilderException(msg) if "buildOptions" in app_json: if app_json["buildOptions"].get("dx_toolkit_autodep") is False: dx_toolkit_autodep = False if dry_run: # Set a dummy "projects_by_region" so that we can exercise the dry # run flows for uploading resources bundles and applets below. projects_by_region = {"dummy-cloud:dummy-region": "project-dummy"} if projects_by_region is None: raise AssertionError("'projects_by_region' should not be None at this point") # "resources" can be used only with an app enabled in a single region and when # "regionalOptions" field is not specified. if "resources" in app_json and ("regionalOptions" in app_json or len(projects_by_region) > 1): error_message = "dxapp.json cannot contain a top-level \"resources\" field " error_message += "when the \"regionalOptions\" field is used or when " error_message += "the app is enabled in multiple regions" raise dxpy.app_builder.AppBuilderException(error_message) resources_bundles_by_region = {} for region, project in projects_by_region.items(): resources_bundles_by_region[region] = dxpy.app_builder.upload_resources( src_dir, project=project, folder=override_folder, ensure_upload=ensure_upload, force_symlinks=force_symlinks) if not dry_run else [] # TODO: Clean up these applets if the app build fails. applet_ids_by_region = {} try: for region, project in projects_by_region.items(): applet_id, applet_spec = dxpy.app_builder.upload_applet( src_dir, resources_bundles_by_region[region], check_name_collisions=(mode == "applet"), overwrite=overwrite and mode == "applet", archive=archive and mode == "applet", project=project, override_folder=override_folder, override_name=override_applet_name, dx_toolkit_autodep=dx_toolkit_autodep, dry_run=dry_run, **kwargs) if not dry_run: logger.debug("Created applet " + applet_id + " successfully") applet_ids_by_region[region] = applet_id except: # Avoid leaking any bundled_resources files we may have # created, if applet creation fails. Note that if # using_temp_project, the entire project gets destroyed at # the end, so we don't bother. if not using_temp_project: for region, project in projects_by_region.items(): objects_to_delete = [dxpy.get_dxlink_ids(bundled_resource_obj['id'])[0] for bundled_resource_obj in resources_bundles_by_region[region]] if objects_to_delete: dxpy.api.project_remove_objects( dxpy.app_builder.get_destination_project(src_dir, project=project), input_params={"objects": objects_to_delete}) raise if dry_run: return applet_name = applet_spec['name'] if mode == "app": if 'version' not in app_json: parser.error("dxapp.json contains no \"version\" field, but it is required to build an app") version = app_json['version'] try_versions = [version_override or version] if not version_override and do_version_autonumbering: try_versions.append(version + _get_version_suffix(src_dir, version)) additional_resources_by_region = {} if "regionalOptions" in app_json: for region, region_config in app_json["regionalOptions"].items(): if "resources" in region_config: additional_resources_by_region[region] = region_config["resources"] elif "resources" in app_json: additional_resources_by_region[projects_by_region.keys()[0]] = app_json["resources"] regional_options = {} for region in projects_by_region: regional_options[region] = {"applet": applet_ids_by_region[region]} if region in additional_resources_by_region: regional_options[region]["resources"] = additional_resources_by_region[region] app_id = dxpy.app_builder.create_app_multi_region(regional_options, applet_name, src_dir, publish=publish, set_default=publish, billTo=bill_to_override, try_versions=try_versions, try_update=do_try_update, confirm=confirm) app_describe = dxpy.api.app_describe(app_id) if publish: print("Uploaded and published app %s/%s (%s) successfully" % (app_describe["name"], app_describe["version"], app_id), file=sys.stderr) else: print("Uploaded app %s/%s (%s) successfully" % (app_describe["name"], app_describe["version"], app_id), file=sys.stderr) print("You can publish this app with:", file=sys.stderr) print(" dx api app-%s/%s publish \"{\\\"makeDefault\\\": true}\"" % (app_describe["name"], app_describe["version"]), file=sys.stderr) return app_describe if return_object_dump else {"id": app_id} elif mode == "applet": return dxpy.api.applet_describe(applet_id) if return_object_dump else {"id": applet_id} else: raise dxpy.app_builder.AppBuilderException("Unrecognized mode %r" % (mode,)) finally: # Clean up after ourselves. if using_temp_project: delete_temporary_projects(projects_by_region.values())
def _build_app_remote(mode, src_dir, publish=False, destination_override=None, version_override=None, bill_to_override=None, dx_toolkit_autodep="stable", do_version_autonumbering=True, do_try_update=True, do_parallel_build=True, do_check_syntax=True, region=None, watch=True): if mode == 'app': builder_app = 'app-tarball_app_builder' else: builder_app = 'app-tarball_applet_builder' app_spec = _parse_app_spec(src_dir) if app_spec['runSpec'].get('release') == '14.04': builder_app += "_trusty" temp_dir = tempfile.mkdtemp() build_options = {'dx_toolkit_autodep': dx_toolkit_autodep} if version_override: build_options['version_override'] = version_override elif do_version_autonumbering: # If autonumbering is DISABLED, the interior run of dx-build-app # will detect the correct version to use without our help. If it # is ENABLED, the version suffix might depend on the state of # the git repository. Since we'll remove the .git directory # before uploading, we need to determine the correct version to # use here and pass it in to the interior run of dx-build-app. if do_version_autonumbering: original_version = app_spec['version'] app_describe = None try: app_describe = dxpy.api.app_describe("app-" + app_spec["name"], alias=original_version, always_retry=False) except dxpy.exceptions.DXAPIError as e: if e.name == 'ResourceNotFound' or (mode == 'applet' and e.name == 'PermissionDenied'): pass else: raise e if app_describe is not None: if 'published' in app_describe or not do_try_update: # The version we wanted was taken; fall back to the # autogenerated version number. build_options['version_override'] = original_version + _get_version_suffix(src_dir, original_version) # The following flags are basically passed through verbatim. if bill_to_override: build_options['bill_to_override'] = bill_to_override if not do_version_autonumbering: build_options['do_version_autonumbering'] = False if not do_try_update: build_options['do_try_update'] = False if not do_parallel_build: build_options['do_parallel_build'] = False if not do_check_syntax: build_options['do_check_syntax'] = False using_temp_project_for_remote_build = False # If building an applet, run the builder app in the destination # project. If building an app, run the builder app in a temporary # project. dest_folder = None dest_applet_name = None if mode == "applet": # Translate the --destination flag as follows. If --destination # is PROJ:FOLDER/NAME, # # 1. Run the builder app in PROJ # 2. Make the output folder FOLDER # 3. Supply --destination=NAME to the interior call of dx-build-applet. build_project_id = dxpy.WORKSPACE_ID if destination_override: build_project_id, dest_folder, dest_applet_name = parse_destination(destination_override) if build_project_id is None: parser.error("Can't create an applet without specifying a destination project; please use the -d/--destination flag to explicitly specify a project") if dest_applet_name: build_options['destination_override'] = '/' + dest_applet_name elif mode == "app": using_temp_project_for_remote_build = True try: project_input = {} project_input["name"] = "dx-build-app --remote temporary project" if bill_to_override: project_input["billTo"] = bill_to_override if region: project_input["region"] = region build_project_id = dxpy.api.project_new(project_input)["id"] except: err_exit() try: # Resolve relative paths and symlinks here so we have something # reasonable to write in the job name below. src_dir = os.path.realpath(src_dir) # Show the user some progress as the tarball is being generated. # Hopefully this will help them to understand when their tarball # is huge (e.g. the target directory already has a whole bunch # of binaries in it) and interrupt before uploading begins. app_tarball_file = os.path.join(temp_dir, "app_tarball.tar.gz") tar_subprocess = subprocess.Popen(["tar", "-czf", "-", "--exclude", "./.git", "."], cwd=src_dir, stdout=subprocess.PIPE) with open(app_tarball_file, 'wb') as tar_output_file: total_num_bytes = 0 last_console_update = 0 start_time = time.time() printed_static_message = False # Pipe the output of tar into the output file while True: tar_exitcode = tar_subprocess.poll() data = tar_subprocess.stdout.read(4 * 1024 * 1024) if tar_exitcode is not None and len(data) == 0: break tar_output_file.write(data) total_num_bytes += len(data) current_time = time.time() # Don't show status messages at all for very short tar # operations (< 1.0 sec) if current_time - last_console_update > 0.25 and current_time - start_time > 1.0: if sys.stderr.isatty(): if last_console_update > 0: sys.stderr.write("\r") sys.stderr.write("Compressing target directory {dir}... ({kb_so_far:,} kb)".format(dir=src_dir, kb_so_far=total_num_bytes // 1024)) sys.stderr.flush() last_console_update = current_time elif not printed_static_message: # Print a message (once only) when stderr is not # going to a live console sys.stderr.write("Compressing target directory %s..." % (src_dir,)) printed_static_message = True if last_console_update > 0: sys.stderr.write("\n") if tar_exitcode != 0: raise Exception("tar exited with non-zero exit code " + str(tar_exitcode)) dxpy.set_workspace_id(build_project_id) remote_file = dxpy.upload_local_file(app_tarball_file, media_type="application/gzip", wait_on_close=True, show_progress=True) try: input_hash = { "input_file": dxpy.dxlink(remote_file), "build_options": build_options } if mode == 'app': input_hash["publish"] = publish api_options = { "name": "Remote build of %s" % (os.path.basename(src_dir),), "input": input_hash, "project": build_project_id, } if dest_folder: api_options["folder"] = dest_folder app_run_result = dxpy.api.app_run(builder_app, input_params=api_options) job_id = app_run_result["id"] print("Started builder job %s" % (job_id,)) if watch: try: subprocess.check_call(["dx", "watch", job_id]) except subprocess.CalledProcessError as e: if e.returncode == 3: # Some kind of failure to build the app. The reason # for the failure is probably self-evident from the # job log (and if it's not, the CalledProcessError # is not informative anyway), so just propagate the # return code without additional remarks. sys.exit(3) else: raise e dxpy.DXJob(job_id).wait_on_done(interval=1) if mode == 'applet': applet_id, _ = dxpy.get_dxlink_ids(dxpy.api.job_describe(job_id)['output']['output_applet']) return applet_id else: # TODO: determine and return the app ID, to allow # running the app if args.run is specified return None finally: if not using_temp_project_for_remote_build: dxpy.DXProject(build_project_id).remove_objects([remote_file.get_id()]) finally: if using_temp_project_for_remote_build: dxpy.api.project_destroy(build_project_id, {"terminateJobs": True}) shutil.rmtree(temp_dir)
def build_and_upload_locally(src_dir, mode, overwrite=False, archive=False, publish=False, destination_override=None, version_override=None, bill_to_override=None, use_temp_build_project=True, do_parallel_build=True, do_version_autonumbering=True, do_try_update=True, dx_toolkit_autodep="stable", do_build_step=True, do_upload_step=True, do_check_syntax=True, dry_run=False, return_object_dump=False, **kwargs): app_json = _parse_app_spec(src_dir) _verify_app_source_dir(src_dir, enforce=do_check_syntax) if mode == "app" and not dry_run: _verify_app_writable(app_json['name']) working_project = None using_temp_project = False override_folder = None override_applet_name = None if mode == "applet" and destination_override: working_project, override_folder, override_applet_name = parse_destination( destination_override) elif mode == "app" and use_temp_build_project and not dry_run: # Create a temp project working_project = dxpy.api.project_new( {"name": "Temporary build project for dx-build-app"})["id"] print >> sys.stderr, "Created temporary project %s to build in" % ( working_project, ) using_temp_project = True try: if mode == "applet" and working_project is None and dxpy.WORKSPACE_ID is None: parser.error( "Can't create an applet without specifying a destination project; please use the -d/--destination flag to explicitly specify a project" ) if "buildOptions" in app_json: if app_json["buildOptions"].get("dx_toolkit_autodep") == False: dx_toolkit_autodep = False del app_json["buildOptions"] if do_build_step: dxpy.app_builder.build(src_dir, parallel_build=do_parallel_build) if not do_upload_step: return bundled_resources = dxpy.app_builder.upload_resources( src_dir, project=working_project, folder=override_folder) if not dry_run else [] try: # TODO: the "auto" setting is vestigial and should be removed. if dx_toolkit_autodep == "auto": dx_toolkit_autodep = "stable" applet_id, applet_spec = dxpy.app_builder.upload_applet( src_dir, bundled_resources, check_name_collisions=(mode == "applet"), overwrite=overwrite and mode == "applet", archive=archive and mode == "applet", project=working_project, override_folder=override_folder, override_name=override_applet_name, dx_toolkit_autodep=dx_toolkit_autodep, dry_run=dry_run, **kwargs) except: # Avoid leaking any bundled_resources files we may have # created, if applet creation fails. Note that if # using_temp_project, the entire project gets destroyed at # the end, so we don't bother. if not using_temp_project: objects_to_delete = [ dxpy.get_dxlink_ids(bundled_resource_obj['id'])[0] for bundled_resource_obj in bundled_resources ] if objects_to_delete: dxpy.api.project_remove_objects( dxpy.app_builder.get_destination_project( src_dir, project=working_project), input_params={"objects": objects_to_delete}) raise if dry_run: return applet_name = applet_spec['name'] print >> sys.stderr, "Created applet " + applet_id + " successfully" if mode == "app": if 'version' not in app_json: parser.error( "dxapp.json contains no \"version\" field, but it is required to build an app" ) version = app_json['version'] try_versions = [version_override or version] if not version_override and do_version_autonumbering: try_versions.append(version + _get_version_suffix(src_dir, version)) app_id = dxpy.app_builder.create_app(applet_id, applet_name, src_dir, publish=publish, set_default=publish, billTo=bill_to_override, try_versions=try_versions, try_update=do_try_update) app_describe = dxpy.api.app_describe(app_id) if publish: print >> sys.stderr, "Uploaded and published app %s/%s (%s) successfully" % ( app_describe["name"], app_describe["version"], app_id) else: print >> sys.stderr, "Uploaded app %s/%s (%s) successfully" % ( app_describe["name"], app_describe["version"], app_id) print >> sys.stderr, "You can publish this app with:" print >> sys.stderr, " dx api app-%s/%s publish \"{\\\"makeDefault\\\": true}\"" % ( app_describe["name"], app_describe["version"]) return app_describe if return_object_dump else {"id": app_id} elif mode == "applet": return dxpy.api.applet_describe( applet_id) if return_object_dump else { "id": applet_id } else: raise dxpy.app_builder.AppBuilderException("Unrecognized mode %r" % (mode, )) finally: # Clean up after ourselves. if using_temp_project: dxpy.api.project_destroy(working_project)
def build_and_upload_locally(src_dir, mode, overwrite=False, archive=False, publish=False, destination_override=None, version_override=None, bill_to_override=None, use_temp_build_project=True, do_parallel_build=True, do_version_autonumbering=True, do_try_update=True, dx_toolkit_autodep="stable", do_check_syntax=True, dry_run=False, return_object_dump=False, confirm=True, ensure_upload=False, **kwargs): dxpy.app_builder.build(src_dir, parallel_build=do_parallel_build) app_json = _parse_app_spec(src_dir) _verify_app_source_dir(src_dir, mode, enforce=do_check_syntax) if mode == "app" and not dry_run: _verify_app_writable(app_json['name']) working_project = None using_temp_project = False override_folder = None override_applet_name = None if mode == "applet" and destination_override: working_project, override_folder, override_applet_name = parse_destination(destination_override) elif mode == "app" and use_temp_build_project and not dry_run: # Create a temp project working_project = dxpy.api.project_new({"name": "Temporary build project for dx-build-app"})["id"] logger.debug("Created temporary project %s to build in" % (working_project,)) using_temp_project = True try: if mode == "applet" and working_project is None and dxpy.WORKSPACE_ID is None: parser.error("Can't create an applet without specifying a destination project; please use the -d/--destination flag to explicitly specify a project") if "buildOptions" in app_json: if app_json["buildOptions"].get("dx_toolkit_autodep") == False: dx_toolkit_autodep = False # Perform check for existence of applet with same name in # destination for case in which neither "-f" nor "-a" is # given BEFORE uploading resources. if mode == "applet" and not overwrite and not archive: try: dest_name = override_applet_name or app_json.get('name') or os.path.basename(os.path.abspath(src_dir)) except: raise dxpy.app_builder.AppBuilderException("Could not determine applet name from specification + " "(dxapp.json) or from working directory (%r)" % (src_dir,)) dest_folder = override_folder or app_json.get('folder') or '/' if not dest_folder.endswith('/'): dest_folder = dest_folder + '/' dest_project = working_project if working_project else dxpy.WORKSPACE_ID for result in dxpy.find_data_objects(classname="applet", name=dest_name, folder=dest_folder, project=dest_project, recurse=False): dest_path = dest_folder + dest_name msg = "An applet already exists at {} (id {}) and neither".format(dest_path, result["id"]) msg += " -f/--overwrite nor -a/--archive were given." raise dxpy.app_builder.AppBuilderException(msg) bundled_resources = dxpy.app_builder.upload_resources(src_dir, project=working_project, folder=override_folder, ensure_upload=ensure_upload) if not dry_run else [] try: # TODO: the "auto" setting is vestigial and should be removed. if dx_toolkit_autodep == "auto": dx_toolkit_autodep = "stable" applet_id, applet_spec = dxpy.app_builder.upload_applet( src_dir, bundled_resources, check_name_collisions=(mode == "applet"), overwrite=overwrite and mode == "applet", archive=archive and mode == "applet", project=working_project, override_folder=override_folder, override_name=override_applet_name, dx_toolkit_autodep=dx_toolkit_autodep, dry_run=dry_run, **kwargs) except: # Avoid leaking any bundled_resources files we may have # created, if applet creation fails. Note that if # using_temp_project, the entire project gets destroyed at # the end, so we don't bother. if not using_temp_project: objects_to_delete = [dxpy.get_dxlink_ids(bundled_resource_obj['id'])[0] for bundled_resource_obj in bundled_resources] if objects_to_delete: dxpy.api.project_remove_objects(dxpy.app_builder.get_destination_project(src_dir, project=working_project), input_params={"objects": objects_to_delete}) raise if dry_run: return applet_name = applet_spec['name'] logger.debug("Created applet " + applet_id + " successfully") if mode == "app": if 'version' not in app_json: parser.error("dxapp.json contains no \"version\" field, but it is required to build an app") version = app_json['version'] try_versions = [version_override or version] if not version_override and do_version_autonumbering: try_versions.append(version + _get_version_suffix(src_dir, version)) app_id = dxpy.app_builder.create_app(applet_id, applet_name, src_dir, publish=publish, set_default=publish, billTo=bill_to_override, try_versions=try_versions, try_update=do_try_update, confirm=confirm) app_describe = dxpy.api.app_describe(app_id) if publish: print("Uploaded and published app %s/%s (%s) successfully" % (app_describe["name"], app_describe["version"], app_id), file=sys.stderr) else: print("Uploaded app %s/%s (%s) successfully" % (app_describe["name"], app_describe["version"], app_id), file=sys.stderr) print("You can publish this app with:", file=sys.stderr) print(" dx api app-%s/%s publish \"{\\\"makeDefault\\\": true}\"" % (app_describe["name"], app_describe["version"]), file=sys.stderr) return app_describe if return_object_dump else {"id": app_id} elif mode == "applet": return dxpy.api.applet_describe(applet_id) if return_object_dump else {"id": applet_id} else: raise dxpy.app_builder.AppBuilderException("Unrecognized mode %r" % (mode,)) finally: # Clean up after ourselves. if using_temp_project: dxpy.api.project_destroy(working_project)
def main(**kwargs): if len(kwargs) == 0: args = parser.parse_args() else: args = parser.parse_args(kwargs) if dxpy.AUTH_HELPER is None: parser.error('Authentication required to build an executable on the platform; please run "dx login" first') if args.src_dir is None: args.src_dir = os.getcwd() if not os.path.isdir(args.src_dir): parser.error("%s is not a directory" % args.src_dir) if not os.path.exists(os.path.join(args.src_dir, "dxapp.json")): parser.error("Directory %s does not contain dxapp.json: not a valid DNAnexus app source directory" % args.src_dir) if args.mode == "app" and args.destination: parser.error("--destination cannot be used when creating an app (only an applet)") if args.remote: # The following flags might be useful in conjunction with # --remote. To enable these, we need to learn how to pass these # options through to the interior call of dx_build_app(let). if args.dry_run: parser.error('--remote cannot be combined with --dry-run') # The following flags are probably not useful in conjunction # with --remote. if not args.build_step: parser.error('--remote cannot be combined with --no-build-step') if not args.upload_step: parser.error('--remote cannot be combined with --no-upload-step') if args.json: parser.error('--remote cannot be combined with --json') if not args.use_temp_build_project: parser.error('--remote cannot be combined with --no-temp-build-project') more_kwargs = {} if args.version_override: more_kwargs['version_override'] = args.version_override if args.bill_to: more_kwargs['bill_to'] = args.bill_to if not args.version_autonumbering: more_kwargs['version_autonumbering'] = False if not args.update: more_kwargs['update'] = False if not args.parallel_build: more_kwargs['parallel_build'] = False return _build_app_remote(args.mode, args.src_dir, destination=args.destination, publish=args.publish, dx_toolkit_autodep=args.dx_toolkit_autodep, **more_kwargs) working_project = None using_temp_project = False override_folder = None override_applet_name = None if args.mode == "applet" and args.destination: working_project, override_folder, override_applet_name = parse_destination(args.destination) elif args.mode == "app" and args.use_temp_build_project and not args.dry_run: # Create a temp project working_project = dxpy.api.projectNew({"name": "Temporary build project for dx-build-app"})["id"] print >> sys.stderr, "Created temporary project %s to build in" % (working_project,) using_temp_project = True if args.mode == "applet" and working_project is None and dxpy.WORKSPACE_ID is None: parser.error("Can't create an applet without specifying a destination project; please use the -d/--destination flag to explicitly specify a project") try: with open(os.path.join(args.src_dir, "dxapp.json")) as app_desc: try: app_json = json.load(app_desc) except: parser.error("Could not parse dxapp.json file as valid JSON") if "buildOptions" in app_json: if app_json["buildOptions"].get("dx_toolkit_autodep") == False: args.dx_toolkit_autodep = False del app_json["buildOptions"] if args.build_step: dxpy.app_builder.build(args.src_dir, parallel_build=args.parallel_build) if not args.upload_step: return bundled_resources = dxpy.app_builder.upload_resources(args.src_dir, project=working_project) if not args.dry_run else [] try: if args.dx_toolkit_autodep == "auto": # "auto" (the default) means dx-toolkit (stable) on preprod and prod, # and dx-toolkit-beta on all other systems. if dxpy.APISERVER_HOST == "preprodapi.dnanexus.com" or dxpy.APISERVER_HOST == "api.dnanexus.com": args.dx_toolkit_autodep = "stable" else: args.dx_toolkit_autodep = "beta" applet_id, applet_spec = dxpy.app_builder.upload_applet( args.src_dir, bundled_resources, check_name_collisions=(args.mode == "applet"), overwrite=args.overwrite and args.mode == "applet", project=working_project, override_folder=override_folder, override_name=override_applet_name, dx_toolkit_autodep=args.dx_toolkit_autodep, dry_run=args.dry_run) except: # Avoid leaking any bundled_resources files we may have # created, if applet creation fails. Note that if # using_temp_project, the entire project gets destroyed at # the end, so we don't bother. if not using_temp_project: objects_to_delete = [dxpy.get_dxlink_ids(bundled_resource_obj['id'])[0] for bundled_resource_obj in bundled_resources] if objects_to_delete: dxpy.api.projectRemoveObjects(dxpy.app_builder.get_destination_project(args.src_dir, project=working_project), input_params={"objects": objects_to_delete}) raise if args.dry_run: return applet_name = applet_spec['name'] print >> sys.stderr, "Created applet " + applet_id + " successfully" if args.mode == "app": if 'version' not in app_json: parser.error("dxapp.json contains no \"version\" field, but it is required to build an app") version = app_json['version'] try_versions = [args.version_override or version] if not args.version_override and args.version_autonumbering: try_versions.append(version + get_version_suffix(args.src_dir)) app_id = dxpy.app_builder.create_app(applet_id, applet_name, args.src_dir, publish=args.publish, set_default=args.publish, billTo=args.bill_to, try_versions=try_versions, try_update=args.update) app_describe = dxpy.api.appDescribe(app_id) if args.json: print json.dumps(app_describe) if args.publish: print >> sys.stderr, "Uploaded and published app %s/%s (%s) successfully" % (app_describe["name"], app_describe["version"], app_id) else: print >> sys.stderr, "Uploaded app %s/%s (%s) successfully" % (app_describe["name"], app_describe["version"], app_id) print >> sys.stderr, "You can publish this app with:" print >> sys.stderr, " dx api app-%s/%s publish \"{\\\"makeDefault\\\": true}\"" % (app_describe["name"], app_describe["version"]) elif args.mode == "applet": if args.json: print json.dumps(dxpy.api.appletDescribe(applet_id)) else: raise dxpy.app_builder.AppBuilderException("Unrecognized mode %r" % (args.mode,)) except dxpy.app_builder.AppBuilderException as e: # AppBuilderException represents errors during app or applet building # that could reasonably have been anticipated by the user. print >> sys.stderr, "Error: %s" % (e.message,) sys.exit(3) finally: # Clean up after ourselves. if using_temp_project: dxpy.api.projectDestroy(working_project)