def build(src_dir, parallel_build=True):
"""
Runs any build scripts that are found in the specified directory.
In particular, runs ``./configure`` if it exists, followed by ``make -jN``
if it exists (building with as many parallel tasks as there are CPUs on the
system).
"""
# TODO: use Gentoo or deb buildsystem
config_script = os.path.join(src_dir, "configure")
if os.path.isfile(config_script) and os.access(config_script, os.X_OK):
logger.debug("Running ./configure in {cwd}".format(cwd=os.path.abspath(src_dir)))
try:
subprocess.check_call([config_script])
except subprocess.CalledProcessError as e:
raise AppBuilderException("./configure in target directory failed with exit code %d" % (e.returncode,))
if os.path.isfile(os.path.join(src_dir, "Makefile")) \
or os.path.isfile(os.path.join(src_dir, "makefile")) \
or os.path.isfile(os.path.join(src_dir, "GNUmakefile")):
if parallel_build:
make_shortcmd = "make -j%d" % (NUM_CORES,)
else:
make_shortcmd = "make"
logger.debug("Building with {make} in {cwd}".format(make=make_shortcmd, cwd=os.path.abspath(src_dir)))
try:
make_cmd = ["make", "-C", src_dir]
if parallel_build:
make_cmd.append("-j" + str(NUM_CORES))
subprocess.check_call(make_cmd)
except subprocess.CalledProcessError as e:
raise AppBuilderException("%s in target directory failed with exit code %d" % (make_shortcmd, e.returncode))
def upload_resources(src_dir, project=None, folder='/'):
"""
:returns: A list (possibly empty) of references to the generated archive(s)
:rtype: list
If it exists, archives and uploads the contents of the
``resources/`` subdirectory of *src_dir* to a new remote file
object, and returns a list describing a single bundled dependency in
the form expected by the ``bundledDepends`` field of a run
specification. Returns an empty list, if no archive was created.
"""
applet_spec = _get_applet_spec(src_dir)
if project is None:
dest_project = applet_spec['project']
else:
dest_project = project
applet_spec['project'] = project
resources_dir = os.path.join(src_dir, "resources")
if os.path.exists(resources_dir) and len(os.listdir(resources_dir)) > 0:
logger.debug("Uploading in " + src_dir)
with tempfile.NamedTemporaryFile(suffix=".tar.gz") as tar_fh:
subprocess.check_call(
['tar', '-C', resources_dir, '-czf', tar_fh.name, '.'])
if 'folder' in applet_spec:
try:
dxpy.get_handler(dest_project).new_folder(
applet_spec['folder'], parents=True)
except dxpy.exceptions.DXAPIError:
pass # TODO: make this better
target_folder = applet_spec[
'folder'] if 'folder' in applet_spec else folder
dx_resource_archive = dxpy.upload_local_file(tar_fh.name,
wait_on_close=True,
project=dest_project,
folder=target_folder,
hidden=True)
archive_link = dxpy.dxlink(dx_resource_archive.get_id())
return [{'name': 'resources.tar.gz', 'id': archive_link}]
else:
return []
def upload_resources(src_dir, project=None, folder='/'):
"""
:returns: A list (possibly empty) of references to the generated archive(s)
:rtype: list
If it exists, archives and uploads the contents of the
``resources/`` subdirectory of *src_dir* to a new remote file
object, and returns a list describing a single bundled dependency in
the form expected by the ``bundledDepends`` field of a run
specification. Returns an empty list, if no archive was created.
"""
applet_spec = _get_applet_spec(src_dir)
if project is None:
dest_project = applet_spec['project']
else:
dest_project = project
applet_spec['project'] = project
resources_dir = os.path.join(src_dir, "resources")
if os.path.exists(resources_dir) and len(os.listdir(resources_dir)) > 0:
logger.debug("Uploading in " + src_dir)
with tempfile.NamedTemporaryFile(suffix=".tar.gz") as tar_fh:
subprocess.check_call(['tar', '-C', resources_dir, '-czf', tar_fh.name, '.'])
if 'folder' in applet_spec:
try:
dxpy.get_handler(dest_project).new_folder(applet_spec['folder'], parents=True)
except dxpy.exceptions.DXAPIError:
pass # TODO: make this better
target_folder = applet_spec['folder'] if 'folder' in applet_spec else folder
dx_resource_archive = dxpy.upload_local_file(tar_fh.name, wait_on_close=True,
project=dest_project, folder=target_folder, hidden=True)
archive_link = dxpy.dxlink(dx_resource_archive.get_id())
return [{'name': 'resources.tar.gz', 'id': archive_link}]
else:
return []
def build(src_dir, parallel_build=True):
"""
Runs any build scripts that are found in the specified directory.
In particular, runs ``./configure`` if it exists, followed by ``make -jN``
if it exists (building with as many parallel tasks as there are CPUs on the
system).
"""
# TODO: use Gentoo or deb buildsystem
config_script = os.path.join(src_dir, "configure")
if os.path.isfile(config_script) and os.access(config_script, os.X_OK):
logger.debug("Running ./configure in {cwd}".format(
cwd=os.path.abspath(src_dir)))
try:
subprocess.check_call([config_script])
except subprocess.CalledProcessError as e:
raise AppBuilderException(
"./configure in target directory failed with exit code %d" %
(e.returncode, ))
if os.path.isfile(os.path.join(src_dir, "Makefile")) \
or os.path.isfile(os.path.join(src_dir, "makefile")) \
or os.path.isfile(os.path.join(src_dir, "GNUmakefile")):
if parallel_build:
make_shortcmd = "make -j%d" % (NUM_CORES, )
else:
make_shortcmd = "make"
logger.debug("Building with {make} in {cwd}".format(
make=make_shortcmd, cwd=os.path.abspath(src_dir)))
try:
make_cmd = ["make", "-C", src_dir]
if parallel_build:
make_cmd.append("-j" + str(NUM_CORES))
subprocess.check_call(make_cmd)
except subprocess.CalledProcessError as e:
raise AppBuilderException(
"%s in target directory failed with exit code %d" %
(make_shortcmd, e.returncode))
def build_and_upload_locally(src_dir, mode, overwrite=False, archive=False, publish=False, destination_override=None,
version_override=None, bill_to_override=None, use_temp_build_project=True,
do_parallel_build=True, do_version_autonumbering=True, do_try_update=True,
dx_toolkit_autodep="stable", do_check_syntax=True, dry_run=False,
return_object_dump=False, confirm=True, **kwargs):
app_json = _parse_app_spec(src_dir)
_verify_app_source_dir(src_dir, mode, enforce=do_check_syntax)
if mode == "app" and not dry_run:
_verify_app_writable(app_json['name'])
working_project = None
using_temp_project = False
override_folder = None
override_applet_name = None
if mode == "applet" and destination_override:
working_project, override_folder, override_applet_name = parse_destination(destination_override)
elif mode == "app" and use_temp_build_project and not dry_run:
# Create a temp project
working_project = dxpy.api.project_new({"name": "Temporary build project for dx-build-app"})["id"]
logger.debug("Created temporary project %s to build in" % (working_project,))
using_temp_project = True
try:
if mode == "applet" and working_project is None and dxpy.WORKSPACE_ID is None:
parser.error("Can't create an applet without specifying a destination project; please use the -d/--destination flag to explicitly specify a project")
if "buildOptions" in app_json:
if app_json["buildOptions"].get("dx_toolkit_autodep") == False:
dx_toolkit_autodep = False
# Perform check for existence of applet with same name in
# destination for case in which neither "-f" nor "-a" is
# given BEFORE uploading resources.
if mode == "applet" and not overwrite and not archive:
try:
dest_name = override_applet_name or app_json.get('name') or os.path.basename(os.path.abspath(src_dir))
except:
raise dxpy.app_builder.AppBuilderException("Could not determine applet name from specification + "
"(dxapp.json) or from working directory (%r)" % (src_dir,))
dest_folder = override_folder or app_json.get('folder') or '/'
if not dest_folder.endswith('/'):
dest_folder = dest_folder + '/'
dest_project = working_project if working_project else dxpy.WORKSPACE_ID
for result in dxpy.find_data_objects(classname="applet", name=dest_name, folder=dest_folder,
project=dest_project, recurse=False):
dest_path = dest_folder + dest_name
raise dxpy.app_builder.AppBuilderException("An applet already exists at {} (id {}) and the " +
"--overwrite (-f) or --archive (-a) options were not " +
"given".format(dest_path, result['id']))
dxpy.app_builder.build(src_dir, parallel_build=do_parallel_build)
bundled_resources = dxpy.app_builder.upload_resources(
src_dir, project=working_project, folder=override_folder) if not dry_run else []
try:
# TODO: the "auto" setting is vestigial and should be removed.
if dx_toolkit_autodep == "auto":
dx_toolkit_autodep = "stable"
applet_id, applet_spec = dxpy.app_builder.upload_applet(
src_dir,
bundled_resources,
check_name_collisions=(mode == "applet"),
overwrite=overwrite and mode == "applet",
archive=archive and mode == "applet",
project=working_project,
override_folder=override_folder,
override_name=override_applet_name,
dx_toolkit_autodep=dx_toolkit_autodep,
dry_run=dry_run,
**kwargs)
except:
# Avoid leaking any bundled_resources files we may have
# created, if applet creation fails. Note that if
# using_temp_project, the entire project gets destroyed at
# the end, so we don't bother.
if not using_temp_project:
objects_to_delete = [dxpy.get_dxlink_ids(bundled_resource_obj['id'])[0] for bundled_resource_obj in bundled_resources]
if objects_to_delete:
dxpy.api.project_remove_objects(dxpy.app_builder.get_destination_project(src_dir, project=working_project),
input_params={"objects": objects_to_delete})
raise
if dry_run:
return
applet_name = applet_spec['name']
logger.debug("Created applet " + applet_id + " successfully")
if mode == "app":
if 'version' not in app_json:
parser.error("dxapp.json contains no \"version\" field, but it is required to build an app")
version = app_json['version']
try_versions = [version_override or version]
if not version_override and do_version_autonumbering:
try_versions.append(version + _get_version_suffix(src_dir, version))
app_id = dxpy.app_builder.create_app(applet_id,
applet_name,
src_dir,
publish=publish,
set_default=publish,
billTo=bill_to_override,
try_versions=try_versions,
try_update=do_try_update,
confirm=confirm)
app_describe = dxpy.api.app_describe(app_id)
if publish:
print("Uploaded and published app %s/%s (%s) successfully" % (app_describe["name"], app_describe["version"], app_id), file=sys.stderr)
else:
print("Uploaded app %s/%s (%s) successfully" % (app_describe["name"], app_describe["version"], app_id), file=sys.stderr)
print("You can publish this app with:", file=sys.stderr)
print(" dx api app-%s/%s publish \"{\\\"makeDefault\\\": true}\"" % (app_describe["name"], app_describe["version"]), file=sys.stderr)
return app_describe if return_object_dump else {"id": app_id}
elif mode == "applet":
return dxpy.api.applet_describe(applet_id) if return_object_dump else {"id": applet_id}
else:
raise dxpy.app_builder.AppBuilderException("Unrecognized mode %r" % (mode,))
finally:
# Clean up after ourselves.
if using_temp_project:
dxpy.api.project_destroy(working_project)
def build_and_upload_locally(
src_dir,
mode,
overwrite=False,
archive=False,
publish=False,
destination_override=None,
version_override=None,
bill_to_override=None,
use_temp_build_project=True,
do_parallel_build=True,
do_version_autonumbering=True,
do_try_update=True,
dx_toolkit_autodep="stable",
do_build_step=True,
do_upload_step=True,
do_check_syntax=True,
dry_run=False,
return_object_dump=False,
confirm=True,
**kwargs
):
app_json = _parse_app_spec(src_dir)
_verify_app_source_dir(src_dir, enforce=do_check_syntax)
if mode == "app" and not dry_run:
_verify_app_writable(app_json["name"])
working_project = None
using_temp_project = False
override_folder = None
override_applet_name = None
if mode == "applet" and destination_override:
working_project, override_folder, override_applet_name = parse_destination(destination_override)
elif mode == "app" and use_temp_build_project and not dry_run:
# Create a temp project
working_project = dxpy.api.project_new({"name": "Temporary build project for dx-build-app"})["id"]
logger.debug("Created temporary project %s to build in" % (working_project,))
using_temp_project = True
try:
if mode == "applet" and working_project is None and dxpy.WORKSPACE_ID is None:
parser.error(
"Can't create an applet without specifying a destination project; please use the -d/--destination flag to explicitly specify a project"
)
if "buildOptions" in app_json:
if app_json["buildOptions"].get("dx_toolkit_autodep") == False:
dx_toolkit_autodep = False
del app_json["buildOptions"]
if do_build_step:
dxpy.app_builder.build(src_dir, parallel_build=do_parallel_build)
if not do_upload_step:
return
bundled_resources = (
dxpy.app_builder.upload_resources(src_dir, project=working_project, folder=override_folder)
if not dry_run
else []
)
try:
# TODO: the "auto" setting is vestigial and should be removed.
if dx_toolkit_autodep == "auto":
dx_toolkit_autodep = "stable"
applet_id, applet_spec = dxpy.app_builder.upload_applet(
src_dir,
bundled_resources,
check_name_collisions=(mode == "applet"),
overwrite=overwrite and mode == "applet",
archive=archive and mode == "applet",
project=working_project,
override_folder=override_folder,
override_name=override_applet_name,
dx_toolkit_autodep=dx_toolkit_autodep,
dry_run=dry_run,
**kwargs
)
except:
# Avoid leaking any bundled_resources files we may have
# created, if applet creation fails. Note that if
# using_temp_project, the entire project gets destroyed at
# the end, so we don't bother.
if not using_temp_project:
objects_to_delete = [
dxpy.get_dxlink_ids(bundled_resource_obj["id"])[0] for bundled_resource_obj in bundled_resources
]
if objects_to_delete:
dxpy.api.project_remove_objects(
dxpy.app_builder.get_destination_project(src_dir, project=working_project),
input_params={"objects": objects_to_delete},
)
raise
if dry_run:
return
applet_name = applet_spec["name"]
logger.debug("Created applet " + applet_id + " successfully")
if mode == "app":
if "version" not in app_json:
parser.error('dxapp.json contains no "version" field, but it is required to build an app')
version = app_json["version"]
try_versions = [version_override or version]
if not version_override and do_version_autonumbering:
try_versions.append(version + _get_version_suffix(src_dir, version))
app_id = dxpy.app_builder.create_app(
applet_id,
applet_name,
src_dir,
publish=publish,
set_default=publish,
billTo=bill_to_override,
try_versions=try_versions,
try_update=do_try_update,
confirm=confirm,
)
app_describe = dxpy.api.app_describe(app_id)
if publish:
print(
"Uploaded and published app %s/%s (%s) successfully"
% (app_describe["name"], app_describe["version"], app_id),
file=sys.stderr,
)
else:
print(
"Uploaded app %s/%s (%s) successfully" % (app_describe["name"], app_describe["version"], app_id),
file=sys.stderr,
)
print("You can publish this app with:", file=sys.stderr)
print(
' dx api app-%s/%s publish "{\\"makeDefault\\": true}"'
% (app_describe["name"], app_describe["version"]),
file=sys.stderr,
)
return app_describe if return_object_dump else {"id": app_id}
elif mode == "applet":
return dxpy.api.applet_describe(applet_id) if return_object_dump else {"id": applet_id}
else:
raise dxpy.app_builder.AppBuilderException("Unrecognized mode %r" % (mode,))
finally:
# Clean up after ourselves.
if using_temp_project:
dxpy.api.project_destroy(working_project)
def upload_applet(src_dir, uploaded_resources, check_name_collisions=True, overwrite=False, archive=False, project=None, override_folder=None, override_name=None, dx_toolkit_autodep="stable", dry_run=False, **kwargs):
"""
Creates a new applet object.
:param project: ID of container in which to create the applet.
:type project: str, or None to use whatever is specified in dxapp.json
:param override_folder: folder name for the resulting applet which, if specified, overrides that given in dxapp.json
:type override_folder: str
:param override_name: name for the resulting applet which, if specified, overrides that given in dxapp.json
:type override_name: str
:param dx_toolkit_autodep: What type of dx-toolkit dependency to inject if none is present. "stable" for the APT package; "git" for HEAD of dx-toolkit master branch; or False for no dependency.
:type dx_toolkit_autodep: boolean or string
"""
applet_spec = _get_applet_spec(src_dir)
if project is None:
dest_project = applet_spec['project']
else:
dest_project = project
applet_spec['project'] = project
if 'name' not in applet_spec:
try:
applet_spec['name'] = os.path.basename(os.path.abspath(src_dir))
except:
raise AppBuilderException("Could not determine applet name from the specification (dxapp.json) or from the name of the working directory (%r)" % (src_dir,))
if override_folder:
applet_spec['folder'] = override_folder
if 'folder' not in applet_spec:
applet_spec['folder'] = '/'
if override_name:
applet_spec['name'] = override_name
if 'dxapi' not in applet_spec:
applet_spec['dxapi'] = dxpy.API_VERSION
archived_applet = None
if check_name_collisions and not dry_run:
destination_path = applet_spec['folder'] + ('/' if not applet_spec['folder'].endswith('/') else '') + applet_spec['name']
logger.debug("Checking for existing applet at " + destination_path)
for result in dxpy.find_data_objects(classname="applet", name=applet_spec["name"], folder=applet_spec['folder'], project=dest_project, recurse=False):
if overwrite:
logger.info("Deleting applet %s" % (result['id']))
# TODO: test me
dxpy.DXProject(dest_project).remove_objects([result['id']])
elif archive:
logger.debug("Archiving applet %s" % (result['id']))
proj = dxpy.DXProject(dest_project)
archive_folder = '/.Applet_archive'
try:
proj.list_folder(archive_folder)
except dxpy.DXAPIError:
proj.new_folder(archive_folder)
proj.move(objects=[result['id']], destination=archive_folder)
archived_applet = dxpy.DXApplet(result['id'], project=dest_project)
now = datetime.datetime.fromtimestamp(archived_applet.created/1000).ctime()
new_name = archived_applet.name + " ({d})".format(d=now)
archived_applet.rename(new_name)
logger.info("Archived applet %s to %s:\"%s/%s\"" % (result['id'], dest_project, archive_folder, new_name))
else:
raise AppBuilderException("An applet already exists at %s (id %s) and the --overwrite (-f) or --archive (-a) options were not given" % (destination_path, result['id']))
# -----
# Override various fields from the pristine dxapp.json
# Inline Readme.md and Readme.developer.md
_inline_documentation_files(applet_spec, src_dir)
# Inline the code of the program
if "runSpec" in applet_spec and "file" in applet_spec["runSpec"]:
# Avoid using runSpec.file for now, it's not fully implemented
#code_filename = os.path.join(src_dir, applet_spec["runSpec"]["file"])
#f = dxpy.upload_local_file(code_filename, wait_on_close=True)
#applet_spec["runSpec"]["file"] = f.get_id()
# Put it into runSpec.code instead
with open(os.path.join(src_dir, applet_spec["runSpec"]["file"])) as code_fh:
applet_spec["runSpec"]["code"] = code_fh.read()
del applet_spec["runSpec"]["file"]
# Attach bundled resources to the app
if uploaded_resources is not None:
applet_spec["runSpec"].setdefault("bundledDepends", [])
applet_spec["runSpec"]["bundledDepends"].extend(uploaded_resources)
# Include the DNAnexus client libraries as an execution dependency, if they are not already
# there
if dx_toolkit_autodep == "git":
dx_toolkit_dep = {"name": "dx-toolkit",
"package_manager": "git",
"url": "git://github.com/dnanexus/dx-toolkit.git",
"tag": "master",
"build_commands": "make install DESTDIR=/ PREFIX=/opt/dnanexus"}
# TODO: reject "beta" and "unstable" eventually
elif dx_toolkit_autodep in ("stable", "beta", "unstable"):
dx_toolkit_dep = {"name": "dx-toolkit", "package_manager": "apt"}
elif dx_toolkit_autodep:
raise AppBuilderException("dx_toolkit_autodep must be one of 'stable', 'git', or False; got %r instead" % (dx_toolkit_autodep,))
if dx_toolkit_autodep:
applet_spec["runSpec"].setdefault("execDepends", [])
exec_depends = applet_spec["runSpec"]["execDepends"]
if type(exec_depends) is not list or any(type(dep) is not dict for dep in exec_depends):
raise AppBuilderException("Expected runSpec.execDepends to be an array of objects")
dx_toolkit_dep_found = any(dep.get('name') in DX_TOOLKIT_PKGS or dep.get('url') in DX_TOOLKIT_GIT_URLS for dep in exec_depends)
if not dx_toolkit_dep_found:
exec_depends.append(dx_toolkit_dep)
if dx_toolkit_autodep == "git":
applet_spec.setdefault("access", {})
applet_spec["access"].setdefault("network", [])
# Note: this can be set to "github.com" instead of "*" if the build doesn't download any deps
if "*" not in applet_spec["access"]["network"]:
applet_spec["access"]["network"].append("*")
merge(applet_spec, kwargs)
# -----
# Now actually create the applet
if dry_run:
print "Would create the following applet:"
print json.dumps(applet_spec, indent=2)
print "*** DRY-RUN-- no applet was created ***"
return None, None
applet_id = dxpy.api.applet_new(applet_spec)["id"]
if "categories" in applet_spec:
dxpy.DXApplet(applet_id, project=dest_project).add_tags(applet_spec["categories"])
if archived_applet:
archived_applet.set_properties({'replacedWith': archived_applet.get_id()})
return applet_id, applet_spec
def build_and_upload_locally(src_dir, mode, overwrite=False, archive=False, publish=False, destination_override=None,
version_override=None, bill_to_override=None, use_temp_build_project=True,
do_parallel_build=True, do_version_autonumbering=True, do_try_update=True,
dx_toolkit_autodep="stable", do_check_syntax=True, dry_run=False,
return_object_dump=False, confirm=True, ensure_upload=False, **kwargs):
dxpy.app_builder.build(src_dir, parallel_build=do_parallel_build)
app_json = _parse_app_spec(src_dir)
_verify_app_source_dir(src_dir, mode, enforce=do_check_syntax)
if mode == "app" and not dry_run:
_verify_app_writable(app_json['name'])
working_project = None
using_temp_project = False
override_folder = None
override_applet_name = None
if mode == "applet" and destination_override:
working_project, override_folder, override_applet_name = parse_destination(destination_override)
elif mode == "app" and use_temp_build_project and not dry_run:
# Create a temp project
working_project = dxpy.api.project_new({"name": "Temporary build project for dx-build-app"})["id"]
logger.debug("Created temporary project %s to build in" % (working_project,))
using_temp_project = True
try:
if mode == "applet" and working_project is None and dxpy.WORKSPACE_ID is None:
parser.error("Can't create an applet without specifying a destination project; please use the -d/--destination flag to explicitly specify a project")
if "buildOptions" in app_json:
if app_json["buildOptions"].get("dx_toolkit_autodep") == False:
dx_toolkit_autodep = False
# Perform check for existence of applet with same name in
# destination for case in which neither "-f" nor "-a" is
# given BEFORE uploading resources.
if mode == "applet" and not overwrite and not archive:
try:
dest_name = override_applet_name or app_json.get('name') or os.path.basename(os.path.abspath(src_dir))
except:
raise dxpy.app_builder.AppBuilderException("Could not determine applet name from specification + "
"(dxapp.json) or from working directory (%r)" % (src_dir,))
dest_folder = override_folder or app_json.get('folder') or '/'
if not dest_folder.endswith('/'):
dest_folder = dest_folder + '/'
dest_project = working_project if working_project else dxpy.WORKSPACE_ID
for result in dxpy.find_data_objects(classname="applet", name=dest_name, folder=dest_folder,
project=dest_project, recurse=False):
dest_path = dest_folder + dest_name
msg = "An applet already exists at {} (id {}) and neither".format(dest_path, result["id"])
msg += " -f/--overwrite nor -a/--archive were given."
raise dxpy.app_builder.AppBuilderException(msg)
bundled_resources = dxpy.app_builder.upload_resources(src_dir,
project=working_project,
folder=override_folder,
ensure_upload=ensure_upload) if not dry_run else []
try:
# TODO: the "auto" setting is vestigial and should be removed.
if dx_toolkit_autodep == "auto":
dx_toolkit_autodep = "stable"
applet_id, applet_spec = dxpy.app_builder.upload_applet(
src_dir,
bundled_resources,
check_name_collisions=(mode == "applet"),
overwrite=overwrite and mode == "applet",
archive=archive and mode == "applet",
project=working_project,
override_folder=override_folder,
override_name=override_applet_name,
dx_toolkit_autodep=dx_toolkit_autodep,
dry_run=dry_run,
**kwargs)
except:
# Avoid leaking any bundled_resources files we may have
# created, if applet creation fails. Note that if
# using_temp_project, the entire project gets destroyed at
# the end, so we don't bother.
if not using_temp_project:
objects_to_delete = [dxpy.get_dxlink_ids(bundled_resource_obj['id'])[0] for bundled_resource_obj in bundled_resources]
if objects_to_delete:
dxpy.api.project_remove_objects(dxpy.app_builder.get_destination_project(src_dir, project=working_project),
input_params={"objects": objects_to_delete})
raise
if dry_run:
return
applet_name = applet_spec['name']
logger.debug("Created applet " + applet_id + " successfully")
if mode == "app":
if 'version' not in app_json:
parser.error("dxapp.json contains no \"version\" field, but it is required to build an app")
version = app_json['version']
try_versions = [version_override or version]
if not version_override and do_version_autonumbering:
try_versions.append(version + _get_version_suffix(src_dir, version))
app_id = dxpy.app_builder.create_app(applet_id,
applet_name,
src_dir,
publish=publish,
set_default=publish,
billTo=bill_to_override,
try_versions=try_versions,
try_update=do_try_update,
confirm=confirm)
app_describe = dxpy.api.app_describe(app_id)
if publish:
print("Uploaded and published app %s/%s (%s) successfully" % (app_describe["name"], app_describe["version"], app_id), file=sys.stderr)
else:
print("Uploaded app %s/%s (%s) successfully" % (app_describe["name"], app_describe["version"], app_id), file=sys.stderr)
print("You can publish this app with:", file=sys.stderr)
print(" dx api app-%s/%s publish \"{\\\"makeDefault\\\": true}\"" % (app_describe["name"], app_describe["version"]), file=sys.stderr)
return app_describe if return_object_dump else {"id": app_id}
elif mode == "applet":
return dxpy.api.applet_describe(applet_id) if return_object_dump else {"id": applet_id}
else:
raise dxpy.app_builder.AppBuilderException("Unrecognized mode %r" % (mode,))
finally:
# Clean up after ourselves.
if using_temp_project:
dxpy.api.project_destroy(working_project)
def upload_applet(src_dir,
uploaded_resources,
check_name_collisions=True,
overwrite=False,
archive=False,
project=None,
override_folder=None,
override_name=None,
dx_toolkit_autodep="stable",
dry_run=False,
**kwargs):
"""
Creates a new applet object.
:param project: ID of container in which to create the applet.
:type project: str, or None to use whatever is specified in dxapp.json
:param override_folder: folder name for the resulting applet which, if specified, overrides that given in dxapp.json
:type override_folder: str
:param override_name: name for the resulting applet which, if specified, overrides that given in dxapp.json
:type override_name: str
:param dx_toolkit_autodep: What type of dx-toolkit dependency to inject if none is present. "stable" for the APT package; "git" for HEAD of dx-toolkit master branch; or False for no dependency.
:type dx_toolkit_autodep: boolean or string
"""
applet_spec = _get_applet_spec(src_dir)
if project is None:
dest_project = applet_spec['project']
else:
dest_project = project
applet_spec['project'] = project
if 'name' not in applet_spec:
try:
applet_spec['name'] = os.path.basename(os.path.abspath(src_dir))
except:
raise AppBuilderException(
"Could not determine applet name from the specification (dxapp.json) or from the name of the working directory (%r)"
% (src_dir, ))
if override_folder:
applet_spec['folder'] = override_folder
if 'folder' not in applet_spec:
applet_spec['folder'] = '/'
if override_name:
applet_spec['name'] = override_name
if 'dxapi' not in applet_spec:
applet_spec['dxapi'] = dxpy.API_VERSION
archived_applet = None
if check_name_collisions and not dry_run:
destination_path = applet_spec['folder'] + (
'/' if not applet_spec['folder'].endswith('/') else
'') + applet_spec['name']
logger.debug("Checking for existing applet at " + destination_path)
for result in dxpy.find_data_objects(classname="applet",
name=applet_spec["name"],
folder=applet_spec['folder'],
project=dest_project,
recurse=False):
if overwrite:
logger.info("Deleting applet %s" % (result['id']))
# TODO: test me
dxpy.DXProject(dest_project).remove_objects([result['id']])
elif archive:
logger.debug("Archiving applet %s" % (result['id']))
proj = dxpy.DXProject(dest_project)
archive_folder = '/.Applet_archive'
try:
proj.list_folder(archive_folder)
except dxpy.DXAPIError:
proj.new_folder(archive_folder)
proj.move(objects=[result['id']], destination=archive_folder)
archived_applet = dxpy.DXApplet(result['id'],
project=dest_project)
now = datetime.datetime.fromtimestamp(archived_applet.created /
1000).ctime()
new_name = archived_applet.name + " ({d})".format(d=now)
archived_applet.rename(new_name)
logger.info(
"Archived applet %s to %s:\"%s/%s\"" %
(result['id'], dest_project, archive_folder, new_name))
else:
raise AppBuilderException(
"An applet already exists at %s (id %s) and the --overwrite (-f) or --archive (-a) options were not given"
% (destination_path, result['id']))
# -----
# Override various fields from the pristine dxapp.json
# Inline Readme.md and Readme.developer.md
_inline_documentation_files(applet_spec, src_dir)
# Inline the code of the program
if "runSpec" in applet_spec and "file" in applet_spec["runSpec"]:
# Avoid using runSpec.file for now, it's not fully implemented
#code_filename = os.path.join(src_dir, applet_spec["runSpec"]["file"])
#f = dxpy.upload_local_file(code_filename, wait_on_close=True)
#applet_spec["runSpec"]["file"] = f.get_id()
# Put it into runSpec.code instead
with open(os.path.join(src_dir,
applet_spec["runSpec"]["file"])) as code_fh:
applet_spec["runSpec"]["code"] = code_fh.read()
del applet_spec["runSpec"]["file"]
# Attach bundled resources to the app
if uploaded_resources is not None:
applet_spec["runSpec"].setdefault("bundledDepends", [])
applet_spec["runSpec"]["bundledDepends"].extend(uploaded_resources)
# Include the DNAnexus client libraries as an execution dependency, if they are not already
# there
if dx_toolkit_autodep == "git":
dx_toolkit_dep = {
"name": "dx-toolkit",
"package_manager": "git",
"url": "git://github.com/dnanexus/dx-toolkit.git",
"tag": "master",
"build_commands": "make install DESTDIR=/ PREFIX=/opt/dnanexus"
}
# TODO: reject "beta" and "unstable" eventually
elif dx_toolkit_autodep in ("stable", "beta", "unstable"):
dx_toolkit_dep = {"name": "dx-toolkit", "package_manager": "apt"}
elif dx_toolkit_autodep:
raise AppBuilderException(
"dx_toolkit_autodep must be one of 'stable', 'git', or False; got %r instead"
% (dx_toolkit_autodep, ))
if dx_toolkit_autodep:
applet_spec["runSpec"].setdefault("execDepends", [])
exec_depends = applet_spec["runSpec"]["execDepends"]
if type(exec_depends) is not list or any(
type(dep) is not dict for dep in exec_depends):
raise AppBuilderException(
"Expected runSpec.execDepends to be an array of objects")
dx_toolkit_dep_found = any(
dep.get('name') in DX_TOOLKIT_PKGS
or dep.get('url') in DX_TOOLKIT_GIT_URLS for dep in exec_depends)
if not dx_toolkit_dep_found:
exec_depends.append(dx_toolkit_dep)
if dx_toolkit_autodep == "git":
applet_spec.setdefault("access", {})
applet_spec["access"].setdefault("network", [])
# Note: this can be set to "github.com" instead of "*" if the build doesn't download any deps
if "*" not in applet_spec["access"]["network"]:
applet_spec["access"]["network"].append("*")
merge(applet_spec, kwargs)
# -----
# Now actually create the applet
if dry_run:
print "Would create the following applet:"
print json.dumps(applet_spec, indent=2)
print "*** DRY-RUN-- no applet was created ***"
return None, None
applet_id = dxpy.api.applet_new(applet_spec)["id"]
if "categories" in applet_spec:
dxpy.DXApplet(applet_id,
project=dest_project).add_tags(applet_spec["categories"])
if archived_applet:
archived_applet.set_properties(
{'replacedWith': archived_applet.get_id()})
return applet_id, applet_spec
