def build(src_dir, parallel_build=True): """ Runs any build scripts that are found in the specified directory. In particular, runs ``./configure`` if it exists, followed by ``make -jN`` if it exists (building with as many parallel tasks as there are CPUs on the system). """ # TODO: use Gentoo or deb buildsystem config_script = os.path.join(src_dir, "configure") if os.path.isfile(config_script) and os.access(config_script, os.X_OK): logger.debug("Running ./configure in {cwd}".format(cwd=os.path.abspath(src_dir))) try: subprocess.check_call([config_script]) except subprocess.CalledProcessError as e: raise AppBuilderException("./configure in target directory failed with exit code %d" % (e.returncode,)) if os.path.isfile(os.path.join(src_dir, "Makefile")) \ or os.path.isfile(os.path.join(src_dir, "makefile")) \ or os.path.isfile(os.path.join(src_dir, "GNUmakefile")): if parallel_build: make_shortcmd = "make -j%d" % (NUM_CORES,) else: make_shortcmd = "make" logger.debug("Building with {make} in {cwd}".format(make=make_shortcmd, cwd=os.path.abspath(src_dir))) try: make_cmd = ["make", "-C", src_dir] if parallel_build: make_cmd.append("-j" + str(NUM_CORES)) subprocess.check_call(make_cmd) except subprocess.CalledProcessError as e: raise AppBuilderException("%s in target directory failed with exit code %d" % (make_shortcmd, e.returncode))
def upload_resources(src_dir, project=None, folder='/'): """ :returns: A list (possibly empty) of references to the generated archive(s) :rtype: list If it exists, archives and uploads the contents of the ``resources/`` subdirectory of *src_dir* to a new remote file object, and returns a list describing a single bundled dependency in the form expected by the ``bundledDepends`` field of a run specification. Returns an empty list, if no archive was created. """ applet_spec = _get_applet_spec(src_dir) if project is None: dest_project = applet_spec['project'] else: dest_project = project applet_spec['project'] = project resources_dir = os.path.join(src_dir, "resources") if os.path.exists(resources_dir) and len(os.listdir(resources_dir)) > 0: logger.debug("Uploading in " + src_dir) with tempfile.NamedTemporaryFile(suffix=".tar.gz") as tar_fh: subprocess.check_call( ['tar', '-C', resources_dir, '-czf', tar_fh.name, '.']) if 'folder' in applet_spec: try: dxpy.get_handler(dest_project).new_folder( applet_spec['folder'], parents=True) except dxpy.exceptions.DXAPIError: pass # TODO: make this better target_folder = applet_spec[ 'folder'] if 'folder' in applet_spec else folder dx_resource_archive = dxpy.upload_local_file(tar_fh.name, wait_on_close=True, project=dest_project, folder=target_folder, hidden=True) archive_link = dxpy.dxlink(dx_resource_archive.get_id()) return [{'name': 'resources.tar.gz', 'id': archive_link}] else: return []
def upload_resources(src_dir, project=None, folder='/'): """ :returns: A list (possibly empty) of references to the generated archive(s) :rtype: list If it exists, archives and uploads the contents of the ``resources/`` subdirectory of *src_dir* to a new remote file object, and returns a list describing a single bundled dependency in the form expected by the ``bundledDepends`` field of a run specification. Returns an empty list, if no archive was created. """ applet_spec = _get_applet_spec(src_dir) if project is None: dest_project = applet_spec['project'] else: dest_project = project applet_spec['project'] = project resources_dir = os.path.join(src_dir, "resources") if os.path.exists(resources_dir) and len(os.listdir(resources_dir)) > 0: logger.debug("Uploading in " + src_dir) with tempfile.NamedTemporaryFile(suffix=".tar.gz") as tar_fh: subprocess.check_call(['tar', '-C', resources_dir, '-czf', tar_fh.name, '.']) if 'folder' in applet_spec: try: dxpy.get_handler(dest_project).new_folder(applet_spec['folder'], parents=True) except dxpy.exceptions.DXAPIError: pass # TODO: make this better target_folder = applet_spec['folder'] if 'folder' in applet_spec else folder dx_resource_archive = dxpy.upload_local_file(tar_fh.name, wait_on_close=True, project=dest_project, folder=target_folder, hidden=True) archive_link = dxpy.dxlink(dx_resource_archive.get_id()) return [{'name': 'resources.tar.gz', 'id': archive_link}] else: return []
def build(src_dir, parallel_build=True): """ Runs any build scripts that are found in the specified directory. In particular, runs ``./configure`` if it exists, followed by ``make -jN`` if it exists (building with as many parallel tasks as there are CPUs on the system). """ # TODO: use Gentoo or deb buildsystem config_script = os.path.join(src_dir, "configure") if os.path.isfile(config_script) and os.access(config_script, os.X_OK): logger.debug("Running ./configure in {cwd}".format( cwd=os.path.abspath(src_dir))) try: subprocess.check_call([config_script]) except subprocess.CalledProcessError as e: raise AppBuilderException( "./configure in target directory failed with exit code %d" % (e.returncode, )) if os.path.isfile(os.path.join(src_dir, "Makefile")) \ or os.path.isfile(os.path.join(src_dir, "makefile")) \ or os.path.isfile(os.path.join(src_dir, "GNUmakefile")): if parallel_build: make_shortcmd = "make -j%d" % (NUM_CORES, ) else: make_shortcmd = "make" logger.debug("Building with {make} in {cwd}".format( make=make_shortcmd, cwd=os.path.abspath(src_dir))) try: make_cmd = ["make", "-C", src_dir] if parallel_build: make_cmd.append("-j" + str(NUM_CORES)) subprocess.check_call(make_cmd) except subprocess.CalledProcessError as e: raise AppBuilderException( "%s in target directory failed with exit code %d" % (make_shortcmd, e.returncode))
def build_and_upload_locally(src_dir, mode, overwrite=False, archive=False, publish=False, destination_override=None, version_override=None, bill_to_override=None, use_temp_build_project=True, do_parallel_build=True, do_version_autonumbering=True, do_try_update=True, dx_toolkit_autodep="stable", do_check_syntax=True, dry_run=False, return_object_dump=False, confirm=True, **kwargs): app_json = _parse_app_spec(src_dir) _verify_app_source_dir(src_dir, mode, enforce=do_check_syntax) if mode == "app" and not dry_run: _verify_app_writable(app_json['name']) working_project = None using_temp_project = False override_folder = None override_applet_name = None if mode == "applet" and destination_override: working_project, override_folder, override_applet_name = parse_destination(destination_override) elif mode == "app" and use_temp_build_project and not dry_run: # Create a temp project working_project = dxpy.api.project_new({"name": "Temporary build project for dx-build-app"})["id"] logger.debug("Created temporary project %s to build in" % (working_project,)) using_temp_project = True try: if mode == "applet" and working_project is None and dxpy.WORKSPACE_ID is None: parser.error("Can't create an applet without specifying a destination project; please use the -d/--destination flag to explicitly specify a project") if "buildOptions" in app_json: if app_json["buildOptions"].get("dx_toolkit_autodep") == False: dx_toolkit_autodep = False # Perform check for existence of applet with same name in # destination for case in which neither "-f" nor "-a" is # given BEFORE uploading resources. if mode == "applet" and not overwrite and not archive: try: dest_name = override_applet_name or app_json.get('name') or os.path.basename(os.path.abspath(src_dir)) except: raise dxpy.app_builder.AppBuilderException("Could not determine applet name from specification + " "(dxapp.json) or from working directory (%r)" % (src_dir,)) dest_folder = override_folder or app_json.get('folder') or '/' if not dest_folder.endswith('/'): dest_folder = dest_folder + '/' dest_project = working_project if working_project else dxpy.WORKSPACE_ID for result in dxpy.find_data_objects(classname="applet", name=dest_name, folder=dest_folder, project=dest_project, recurse=False): dest_path = dest_folder + dest_name raise dxpy.app_builder.AppBuilderException("An applet already exists at {} (id {}) and the " + "--overwrite (-f) or --archive (-a) options were not " + "given".format(dest_path, result['id'])) dxpy.app_builder.build(src_dir, parallel_build=do_parallel_build) bundled_resources = dxpy.app_builder.upload_resources( src_dir, project=working_project, folder=override_folder) if not dry_run else [] try: # TODO: the "auto" setting is vestigial and should be removed. if dx_toolkit_autodep == "auto": dx_toolkit_autodep = "stable" applet_id, applet_spec = dxpy.app_builder.upload_applet( src_dir, bundled_resources, check_name_collisions=(mode == "applet"), overwrite=overwrite and mode == "applet", archive=archive and mode == "applet", project=working_project, override_folder=override_folder, override_name=override_applet_name, dx_toolkit_autodep=dx_toolkit_autodep, dry_run=dry_run, **kwargs) except: # Avoid leaking any bundled_resources files we may have # created, if applet creation fails. Note that if # using_temp_project, the entire project gets destroyed at # the end, so we don't bother. if not using_temp_project: objects_to_delete = [dxpy.get_dxlink_ids(bundled_resource_obj['id'])[0] for bundled_resource_obj in bundled_resources] if objects_to_delete: dxpy.api.project_remove_objects(dxpy.app_builder.get_destination_project(src_dir, project=working_project), input_params={"objects": objects_to_delete}) raise if dry_run: return applet_name = applet_spec['name'] logger.debug("Created applet " + applet_id + " successfully") if mode == "app": if 'version' not in app_json: parser.error("dxapp.json contains no \"version\" field, but it is required to build an app") version = app_json['version'] try_versions = [version_override or version] if not version_override and do_version_autonumbering: try_versions.append(version + _get_version_suffix(src_dir, version)) app_id = dxpy.app_builder.create_app(applet_id, applet_name, src_dir, publish=publish, set_default=publish, billTo=bill_to_override, try_versions=try_versions, try_update=do_try_update, confirm=confirm) app_describe = dxpy.api.app_describe(app_id) if publish: print("Uploaded and published app %s/%s (%s) successfully" % (app_describe["name"], app_describe["version"], app_id), file=sys.stderr) else: print("Uploaded app %s/%s (%s) successfully" % (app_describe["name"], app_describe["version"], app_id), file=sys.stderr) print("You can publish this app with:", file=sys.stderr) print(" dx api app-%s/%s publish \"{\\\"makeDefault\\\": true}\"" % (app_describe["name"], app_describe["version"]), file=sys.stderr) return app_describe if return_object_dump else {"id": app_id} elif mode == "applet": return dxpy.api.applet_describe(applet_id) if return_object_dump else {"id": applet_id} else: raise dxpy.app_builder.AppBuilderException("Unrecognized mode %r" % (mode,)) finally: # Clean up after ourselves. if using_temp_project: dxpy.api.project_destroy(working_project)
def build_and_upload_locally( src_dir, mode, overwrite=False, archive=False, publish=False, destination_override=None, version_override=None, bill_to_override=None, use_temp_build_project=True, do_parallel_build=True, do_version_autonumbering=True, do_try_update=True, dx_toolkit_autodep="stable", do_build_step=True, do_upload_step=True, do_check_syntax=True, dry_run=False, return_object_dump=False, confirm=True, **kwargs ): app_json = _parse_app_spec(src_dir) _verify_app_source_dir(src_dir, enforce=do_check_syntax) if mode == "app" and not dry_run: _verify_app_writable(app_json["name"]) working_project = None using_temp_project = False override_folder = None override_applet_name = None if mode == "applet" and destination_override: working_project, override_folder, override_applet_name = parse_destination(destination_override) elif mode == "app" and use_temp_build_project and not dry_run: # Create a temp project working_project = dxpy.api.project_new({"name": "Temporary build project for dx-build-app"})["id"] logger.debug("Created temporary project %s to build in" % (working_project,)) using_temp_project = True try: if mode == "applet" and working_project is None and dxpy.WORKSPACE_ID is None: parser.error( "Can't create an applet without specifying a destination project; please use the -d/--destination flag to explicitly specify a project" ) if "buildOptions" in app_json: if app_json["buildOptions"].get("dx_toolkit_autodep") == False: dx_toolkit_autodep = False del app_json["buildOptions"] if do_build_step: dxpy.app_builder.build(src_dir, parallel_build=do_parallel_build) if not do_upload_step: return bundled_resources = ( dxpy.app_builder.upload_resources(src_dir, project=working_project, folder=override_folder) if not dry_run else [] ) try: # TODO: the "auto" setting is vestigial and should be removed. if dx_toolkit_autodep == "auto": dx_toolkit_autodep = "stable" applet_id, applet_spec = dxpy.app_builder.upload_applet( src_dir, bundled_resources, check_name_collisions=(mode == "applet"), overwrite=overwrite and mode == "applet", archive=archive and mode == "applet", project=working_project, override_folder=override_folder, override_name=override_applet_name, dx_toolkit_autodep=dx_toolkit_autodep, dry_run=dry_run, **kwargs ) except: # Avoid leaking any bundled_resources files we may have # created, if applet creation fails. Note that if # using_temp_project, the entire project gets destroyed at # the end, so we don't bother. if not using_temp_project: objects_to_delete = [ dxpy.get_dxlink_ids(bundled_resource_obj["id"])[0] for bundled_resource_obj in bundled_resources ] if objects_to_delete: dxpy.api.project_remove_objects( dxpy.app_builder.get_destination_project(src_dir, project=working_project), input_params={"objects": objects_to_delete}, ) raise if dry_run: return applet_name = applet_spec["name"] logger.debug("Created applet " + applet_id + " successfully") if mode == "app": if "version" not in app_json: parser.error('dxapp.json contains no "version" field, but it is required to build an app') version = app_json["version"] try_versions = [version_override or version] if not version_override and do_version_autonumbering: try_versions.append(version + _get_version_suffix(src_dir, version)) app_id = dxpy.app_builder.create_app( applet_id, applet_name, src_dir, publish=publish, set_default=publish, billTo=bill_to_override, try_versions=try_versions, try_update=do_try_update, confirm=confirm, ) app_describe = dxpy.api.app_describe(app_id) if publish: print( "Uploaded and published app %s/%s (%s) successfully" % (app_describe["name"], app_describe["version"], app_id), file=sys.stderr, ) else: print( "Uploaded app %s/%s (%s) successfully" % (app_describe["name"], app_describe["version"], app_id), file=sys.stderr, ) print("You can publish this app with:", file=sys.stderr) print( ' dx api app-%s/%s publish "{\\"makeDefault\\": true}"' % (app_describe["name"], app_describe["version"]), file=sys.stderr, ) return app_describe if return_object_dump else {"id": app_id} elif mode == "applet": return dxpy.api.applet_describe(applet_id) if return_object_dump else {"id": applet_id} else: raise dxpy.app_builder.AppBuilderException("Unrecognized mode %r" % (mode,)) finally: # Clean up after ourselves. if using_temp_project: dxpy.api.project_destroy(working_project)
def upload_applet(src_dir, uploaded_resources, check_name_collisions=True, overwrite=False, archive=False, project=None, override_folder=None, override_name=None, dx_toolkit_autodep="stable", dry_run=False, **kwargs): """ Creates a new applet object. :param project: ID of container in which to create the applet. :type project: str, or None to use whatever is specified in dxapp.json :param override_folder: folder name for the resulting applet which, if specified, overrides that given in dxapp.json :type override_folder: str :param override_name: name for the resulting applet which, if specified, overrides that given in dxapp.json :type override_name: str :param dx_toolkit_autodep: What type of dx-toolkit dependency to inject if none is present. "stable" for the APT package; "git" for HEAD of dx-toolkit master branch; or False for no dependency. :type dx_toolkit_autodep: boolean or string """ applet_spec = _get_applet_spec(src_dir) if project is None: dest_project = applet_spec['project'] else: dest_project = project applet_spec['project'] = project if 'name' not in applet_spec: try: applet_spec['name'] = os.path.basename(os.path.abspath(src_dir)) except: raise AppBuilderException("Could not determine applet name from the specification (dxapp.json) or from the name of the working directory (%r)" % (src_dir,)) if override_folder: applet_spec['folder'] = override_folder if 'folder' not in applet_spec: applet_spec['folder'] = '/' if override_name: applet_spec['name'] = override_name if 'dxapi' not in applet_spec: applet_spec['dxapi'] = dxpy.API_VERSION archived_applet = None if check_name_collisions and not dry_run: destination_path = applet_spec['folder'] + ('/' if not applet_spec['folder'].endswith('/') else '') + applet_spec['name'] logger.debug("Checking for existing applet at " + destination_path) for result in dxpy.find_data_objects(classname="applet", name=applet_spec["name"], folder=applet_spec['folder'], project=dest_project, recurse=False): if overwrite: logger.info("Deleting applet %s" % (result['id'])) # TODO: test me dxpy.DXProject(dest_project).remove_objects([result['id']]) elif archive: logger.debug("Archiving applet %s" % (result['id'])) proj = dxpy.DXProject(dest_project) archive_folder = '/.Applet_archive' try: proj.list_folder(archive_folder) except dxpy.DXAPIError: proj.new_folder(archive_folder) proj.move(objects=[result['id']], destination=archive_folder) archived_applet = dxpy.DXApplet(result['id'], project=dest_project) now = datetime.datetime.fromtimestamp(archived_applet.created/1000).ctime() new_name = archived_applet.name + " ({d})".format(d=now) archived_applet.rename(new_name) logger.info("Archived applet %s to %s:\"%s/%s\"" % (result['id'], dest_project, archive_folder, new_name)) else: raise AppBuilderException("An applet already exists at %s (id %s) and the --overwrite (-f) or --archive (-a) options were not given" % (destination_path, result['id'])) # ----- # Override various fields from the pristine dxapp.json # Inline Readme.md and Readme.developer.md _inline_documentation_files(applet_spec, src_dir) # Inline the code of the program if "runSpec" in applet_spec and "file" in applet_spec["runSpec"]: # Avoid using runSpec.file for now, it's not fully implemented #code_filename = os.path.join(src_dir, applet_spec["runSpec"]["file"]) #f = dxpy.upload_local_file(code_filename, wait_on_close=True) #applet_spec["runSpec"]["file"] = f.get_id() # Put it into runSpec.code instead with open(os.path.join(src_dir, applet_spec["runSpec"]["file"])) as code_fh: applet_spec["runSpec"]["code"] = code_fh.read() del applet_spec["runSpec"]["file"] # Attach bundled resources to the app if uploaded_resources is not None: applet_spec["runSpec"].setdefault("bundledDepends", []) applet_spec["runSpec"]["bundledDepends"].extend(uploaded_resources) # Include the DNAnexus client libraries as an execution dependency, if they are not already # there if dx_toolkit_autodep == "git": dx_toolkit_dep = {"name": "dx-toolkit", "package_manager": "git", "url": "git://github.com/dnanexus/dx-toolkit.git", "tag": "master", "build_commands": "make install DESTDIR=/ PREFIX=/opt/dnanexus"} # TODO: reject "beta" and "unstable" eventually elif dx_toolkit_autodep in ("stable", "beta", "unstable"): dx_toolkit_dep = {"name": "dx-toolkit", "package_manager": "apt"} elif dx_toolkit_autodep: raise AppBuilderException("dx_toolkit_autodep must be one of 'stable', 'git', or False; got %r instead" % (dx_toolkit_autodep,)) if dx_toolkit_autodep: applet_spec["runSpec"].setdefault("execDepends", []) exec_depends = applet_spec["runSpec"]["execDepends"] if type(exec_depends) is not list or any(type(dep) is not dict for dep in exec_depends): raise AppBuilderException("Expected runSpec.execDepends to be an array of objects") dx_toolkit_dep_found = any(dep.get('name') in DX_TOOLKIT_PKGS or dep.get('url') in DX_TOOLKIT_GIT_URLS for dep in exec_depends) if not dx_toolkit_dep_found: exec_depends.append(dx_toolkit_dep) if dx_toolkit_autodep == "git": applet_spec.setdefault("access", {}) applet_spec["access"].setdefault("network", []) # Note: this can be set to "github.com" instead of "*" if the build doesn't download any deps if "*" not in applet_spec["access"]["network"]: applet_spec["access"]["network"].append("*") merge(applet_spec, kwargs) # ----- # Now actually create the applet if dry_run: print "Would create the following applet:" print json.dumps(applet_spec, indent=2) print "*** DRY-RUN-- no applet was created ***" return None, None applet_id = dxpy.api.applet_new(applet_spec)["id"] if "categories" in applet_spec: dxpy.DXApplet(applet_id, project=dest_project).add_tags(applet_spec["categories"]) if archived_applet: archived_applet.set_properties({'replacedWith': archived_applet.get_id()}) return applet_id, applet_spec
def build_and_upload_locally(src_dir, mode, overwrite=False, archive=False, publish=False, destination_override=None, version_override=None, bill_to_override=None, use_temp_build_project=True, do_parallel_build=True, do_version_autonumbering=True, do_try_update=True, dx_toolkit_autodep="stable", do_check_syntax=True, dry_run=False, return_object_dump=False, confirm=True, ensure_upload=False, **kwargs): dxpy.app_builder.build(src_dir, parallel_build=do_parallel_build) app_json = _parse_app_spec(src_dir) _verify_app_source_dir(src_dir, mode, enforce=do_check_syntax) if mode == "app" and not dry_run: _verify_app_writable(app_json['name']) working_project = None using_temp_project = False override_folder = None override_applet_name = None if mode == "applet" and destination_override: working_project, override_folder, override_applet_name = parse_destination(destination_override) elif mode == "app" and use_temp_build_project and not dry_run: # Create a temp project working_project = dxpy.api.project_new({"name": "Temporary build project for dx-build-app"})["id"] logger.debug("Created temporary project %s to build in" % (working_project,)) using_temp_project = True try: if mode == "applet" and working_project is None and dxpy.WORKSPACE_ID is None: parser.error("Can't create an applet without specifying a destination project; please use the -d/--destination flag to explicitly specify a project") if "buildOptions" in app_json: if app_json["buildOptions"].get("dx_toolkit_autodep") == False: dx_toolkit_autodep = False # Perform check for existence of applet with same name in # destination for case in which neither "-f" nor "-a" is # given BEFORE uploading resources. if mode == "applet" and not overwrite and not archive: try: dest_name = override_applet_name or app_json.get('name') or os.path.basename(os.path.abspath(src_dir)) except: raise dxpy.app_builder.AppBuilderException("Could not determine applet name from specification + " "(dxapp.json) or from working directory (%r)" % (src_dir,)) dest_folder = override_folder or app_json.get('folder') or '/' if not dest_folder.endswith('/'): dest_folder = dest_folder + '/' dest_project = working_project if working_project else dxpy.WORKSPACE_ID for result in dxpy.find_data_objects(classname="applet", name=dest_name, folder=dest_folder, project=dest_project, recurse=False): dest_path = dest_folder + dest_name msg = "An applet already exists at {} (id {}) and neither".format(dest_path, result["id"]) msg += " -f/--overwrite nor -a/--archive were given." raise dxpy.app_builder.AppBuilderException(msg) bundled_resources = dxpy.app_builder.upload_resources(src_dir, project=working_project, folder=override_folder, ensure_upload=ensure_upload) if not dry_run else [] try: # TODO: the "auto" setting is vestigial and should be removed. if dx_toolkit_autodep == "auto": dx_toolkit_autodep = "stable" applet_id, applet_spec = dxpy.app_builder.upload_applet( src_dir, bundled_resources, check_name_collisions=(mode == "applet"), overwrite=overwrite and mode == "applet", archive=archive and mode == "applet", project=working_project, override_folder=override_folder, override_name=override_applet_name, dx_toolkit_autodep=dx_toolkit_autodep, dry_run=dry_run, **kwargs) except: # Avoid leaking any bundled_resources files we may have # created, if applet creation fails. Note that if # using_temp_project, the entire project gets destroyed at # the end, so we don't bother. if not using_temp_project: objects_to_delete = [dxpy.get_dxlink_ids(bundled_resource_obj['id'])[0] for bundled_resource_obj in bundled_resources] if objects_to_delete: dxpy.api.project_remove_objects(dxpy.app_builder.get_destination_project(src_dir, project=working_project), input_params={"objects": objects_to_delete}) raise if dry_run: return applet_name = applet_spec['name'] logger.debug("Created applet " + applet_id + " successfully") if mode == "app": if 'version' not in app_json: parser.error("dxapp.json contains no \"version\" field, but it is required to build an app") version = app_json['version'] try_versions = [version_override or version] if not version_override and do_version_autonumbering: try_versions.append(version + _get_version_suffix(src_dir, version)) app_id = dxpy.app_builder.create_app(applet_id, applet_name, src_dir, publish=publish, set_default=publish, billTo=bill_to_override, try_versions=try_versions, try_update=do_try_update, confirm=confirm) app_describe = dxpy.api.app_describe(app_id) if publish: print("Uploaded and published app %s/%s (%s) successfully" % (app_describe["name"], app_describe["version"], app_id), file=sys.stderr) else: print("Uploaded app %s/%s (%s) successfully" % (app_describe["name"], app_describe["version"], app_id), file=sys.stderr) print("You can publish this app with:", file=sys.stderr) print(" dx api app-%s/%s publish \"{\\\"makeDefault\\\": true}\"" % (app_describe["name"], app_describe["version"]), file=sys.stderr) return app_describe if return_object_dump else {"id": app_id} elif mode == "applet": return dxpy.api.applet_describe(applet_id) if return_object_dump else {"id": applet_id} else: raise dxpy.app_builder.AppBuilderException("Unrecognized mode %r" % (mode,)) finally: # Clean up after ourselves. if using_temp_project: dxpy.api.project_destroy(working_project)
def upload_applet(src_dir, uploaded_resources, check_name_collisions=True, overwrite=False, archive=False, project=None, override_folder=None, override_name=None, dx_toolkit_autodep="stable", dry_run=False, **kwargs): """ Creates a new applet object. :param project: ID of container in which to create the applet. :type project: str, or None to use whatever is specified in dxapp.json :param override_folder: folder name for the resulting applet which, if specified, overrides that given in dxapp.json :type override_folder: str :param override_name: name for the resulting applet which, if specified, overrides that given in dxapp.json :type override_name: str :param dx_toolkit_autodep: What type of dx-toolkit dependency to inject if none is present. "stable" for the APT package; "git" for HEAD of dx-toolkit master branch; or False for no dependency. :type dx_toolkit_autodep: boolean or string """ applet_spec = _get_applet_spec(src_dir) if project is None: dest_project = applet_spec['project'] else: dest_project = project applet_spec['project'] = project if 'name' not in applet_spec: try: applet_spec['name'] = os.path.basename(os.path.abspath(src_dir)) except: raise AppBuilderException( "Could not determine applet name from the specification (dxapp.json) or from the name of the working directory (%r)" % (src_dir, )) if override_folder: applet_spec['folder'] = override_folder if 'folder' not in applet_spec: applet_spec['folder'] = '/' if override_name: applet_spec['name'] = override_name if 'dxapi' not in applet_spec: applet_spec['dxapi'] = dxpy.API_VERSION archived_applet = None if check_name_collisions and not dry_run: destination_path = applet_spec['folder'] + ( '/' if not applet_spec['folder'].endswith('/') else '') + applet_spec['name'] logger.debug("Checking for existing applet at " + destination_path) for result in dxpy.find_data_objects(classname="applet", name=applet_spec["name"], folder=applet_spec['folder'], project=dest_project, recurse=False): if overwrite: logger.info("Deleting applet %s" % (result['id'])) # TODO: test me dxpy.DXProject(dest_project).remove_objects([result['id']]) elif archive: logger.debug("Archiving applet %s" % (result['id'])) proj = dxpy.DXProject(dest_project) archive_folder = '/.Applet_archive' try: proj.list_folder(archive_folder) except dxpy.DXAPIError: proj.new_folder(archive_folder) proj.move(objects=[result['id']], destination=archive_folder) archived_applet = dxpy.DXApplet(result['id'], project=dest_project) now = datetime.datetime.fromtimestamp(archived_applet.created / 1000).ctime() new_name = archived_applet.name + " ({d})".format(d=now) archived_applet.rename(new_name) logger.info( "Archived applet %s to %s:\"%s/%s\"" % (result['id'], dest_project, archive_folder, new_name)) else: raise AppBuilderException( "An applet already exists at %s (id %s) and the --overwrite (-f) or --archive (-a) options were not given" % (destination_path, result['id'])) # ----- # Override various fields from the pristine dxapp.json # Inline Readme.md and Readme.developer.md _inline_documentation_files(applet_spec, src_dir) # Inline the code of the program if "runSpec" in applet_spec and "file" in applet_spec["runSpec"]: # Avoid using runSpec.file for now, it's not fully implemented #code_filename = os.path.join(src_dir, applet_spec["runSpec"]["file"]) #f = dxpy.upload_local_file(code_filename, wait_on_close=True) #applet_spec["runSpec"]["file"] = f.get_id() # Put it into runSpec.code instead with open(os.path.join(src_dir, applet_spec["runSpec"]["file"])) as code_fh: applet_spec["runSpec"]["code"] = code_fh.read() del applet_spec["runSpec"]["file"] # Attach bundled resources to the app if uploaded_resources is not None: applet_spec["runSpec"].setdefault("bundledDepends", []) applet_spec["runSpec"]["bundledDepends"].extend(uploaded_resources) # Include the DNAnexus client libraries as an execution dependency, if they are not already # there if dx_toolkit_autodep == "git": dx_toolkit_dep = { "name": "dx-toolkit", "package_manager": "git", "url": "git://github.com/dnanexus/dx-toolkit.git", "tag": "master", "build_commands": "make install DESTDIR=/ PREFIX=/opt/dnanexus" } # TODO: reject "beta" and "unstable" eventually elif dx_toolkit_autodep in ("stable", "beta", "unstable"): dx_toolkit_dep = {"name": "dx-toolkit", "package_manager": "apt"} elif dx_toolkit_autodep: raise AppBuilderException( "dx_toolkit_autodep must be one of 'stable', 'git', or False; got %r instead" % (dx_toolkit_autodep, )) if dx_toolkit_autodep: applet_spec["runSpec"].setdefault("execDepends", []) exec_depends = applet_spec["runSpec"]["execDepends"] if type(exec_depends) is not list or any( type(dep) is not dict for dep in exec_depends): raise AppBuilderException( "Expected runSpec.execDepends to be an array of objects") dx_toolkit_dep_found = any( dep.get('name') in DX_TOOLKIT_PKGS or dep.get('url') in DX_TOOLKIT_GIT_URLS for dep in exec_depends) if not dx_toolkit_dep_found: exec_depends.append(dx_toolkit_dep) if dx_toolkit_autodep == "git": applet_spec.setdefault("access", {}) applet_spec["access"].setdefault("network", []) # Note: this can be set to "github.com" instead of "*" if the build doesn't download any deps if "*" not in applet_spec["access"]["network"]: applet_spec["access"]["network"].append("*") merge(applet_spec, kwargs) # ----- # Now actually create the applet if dry_run: print "Would create the following applet:" print json.dumps(applet_spec, indent=2) print "*** DRY-RUN-- no applet was created ***" return None, None applet_id = dxpy.api.applet_new(applet_spec)["id"] if "categories" in applet_spec: dxpy.DXApplet(applet_id, project=dest_project).add_tags(applet_spec["categories"]) if archived_applet: archived_applet.set_properties( {'replacedWith': archived_applet.get_id()}) return applet_id, applet_spec