def download_url_create_symlink(self, url, sym_name):
print("url = {}".format(url))
tmp_file = "localfile"
# download [url]
cmd = ["wget", "--tries=5", "--quiet", "-O", tmp_file, url]
try:
print("Downloading original link with wget")
subprocess.check_call(cmd, stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as e:
msg = ""
if e and e.output:
msg = e.output.strip()
err_exit("Failed to download with wget: {cmd}\n{msg}\n".format(
cmd=str(cmd), msg=msg))
# calculate its md5 checksum
digest = md5_checksum(tmp_file)
os.remove(tmp_file)
# create a symlink on the platform, with the correct checksum
input_params = {
'name': sym_name,
'project': self.proj_id,
'drive': "drive-PUBLISHED",
'md5sum': digest,
'symlinkPath': {
'object': url
}
}
result = dxpy.api.file_new(input_params=input_params)
return dxpy.DXFile(dxid=result["id"], project=self.proj_id)
def md5_checksum(filename):
md5sum_exe = which("md5sum")
if md5sum_exe is None:
err_exit("md5sum is not installed on this system")
cmd = [md5sum_exe, "-b", filename]
try:
print("Calculating checksum")
cmd_out = subprocess.check_output(cmd)
except subprocess.CalledProcessError:
err_exit("Failed to run md5sum: " + str(cmd))
line = cmd_out.strip().split()
if len(line) != 2:
err_exit("md5sum returned weird results: " + str(line))
result = line[0]
# convert to string
assert (isinstance(result, bytes))
result = result.decode("ascii")
return result
def main(**kwargs):
"""
Entry point for dx-build-app(let).
Don't call this function as a subroutine in your program! It is liable to
sys.exit your program when it detects certain error conditions, so you
can't recover from those as you could if it raised exceptions. Instead,
call dx_build_app.build_and_upload_locally which provides the real
implementation for dx-build-app(let) but is easier to use in your program.
"""
if len(sys.argv) > 0:
if sys.argv[0].endswith('dx-build-app'):
logging.warn('Warning: dx-build-app has been replaced with "dx build --create-app". Please update your scripts.')
elif sys.argv[0].endswith('dx-build-applet'):
logging.warn('Warning: dx-build-applet has been replaced with "dx build". Please update your scripts.')
if len(kwargs) == 0:
args = parser.parse_args()
else:
args = parser.parse_args(**kwargs)
if dxpy.AUTH_HELPER is None and not args.dry_run:
parser.error('Authentication required to build an executable on the platform; please run "dx login" first')
if args.src_dir is None:
args.src_dir = os.getcwd()
if USING_PYTHON2:
args.src_dir = args.src_dir.decode(sys.getfilesystemencoding())
if args.mode == "app" and args.destination != '.':
parser.error("--destination cannot be used when creating an app (only an applet)")
if args.dx_toolkit_autodep in ['beta', 'unstable']:
logging.warn('The --dx-toolkit-beta-autodep and --dx-toolkit-unstable-autodep flags have no effect and will be removed at some date in the future.')
if args.overwrite and args.archive:
parser.error("Options -f/--overwrite and -a/--archive cannot be specified together")
if args.run is not None and args.dry_run:
parser.error("Options --dry-run and --run cannot be specified together")
if args.run and args.remote and args.mode == 'app':
parser.error("Options --remote, --app, and --run cannot all be specified together. Try removing --run and then separately invoking dx run.")
executable_id = _build_app(args,
json.loads(args.extra_args) if args.extra_args else {})
if args.run is not None:
if executable_id is None:
raise AssertionError('Expected executable_id to be set here')
try:
subprocess.check_call(['dx', 'run', executable_id, '--priority', 'high'] + args.run)
except subprocess.CalledProcessError as e:
sys.exit(e.returncode)
except:
err_exit()
return
def main(**kwargs):
"""
Entry point for dx-build-app(let).
Don't call this function as a subroutine in your program! It is liable to
sys.exit your program when it detects certain error conditions, so you
can't recover from those as you could if it raised exceptions. Instead,
call dx_build_app.build_and_upload_locally which provides the real
implementation for dx-build-app(let) but is easier to use in your program.
"""
if len(sys.argv) > 0:
if sys.argv[0].endswith('dx-build-app'):
logging.warn('Warning: dx-build-app has been replaced with "dx build --create-app". Please update your scripts.')
elif sys.argv[0].endswith('dx-build-applet'):
logging.warn('Warning: dx-build-applet has been replaced with "dx build". Please update your scripts.')
if len(kwargs) == 0:
args = parser.parse_args()
else:
args = parser.parse_args(**kwargs)
if dxpy.AUTH_HELPER is None and not args.dry_run:
parser.error('Authentication required to build an executable on the platform; please run "dx login" first')
if args.src_dir is None:
args.src_dir = os.getcwd()
if args.mode == "app" and args.destination != '.':
parser.error("--destination cannot be used when creating an app (only an applet)")
if args.dx_toolkit_autodep in ['beta', 'unstable']:
logging.warn('The --dx-toolkit-beta-autodep and --dx-toolkit-unstable-autodep flags have no effect and will be removed at some date in the future.')
if args.overwrite and args.archive:
parser.error("Options -f/--overwrite and -a/--archive cannot be specified together")
extra_args = json.loads(args.extra_args) if args.extra_args else {}
if not args.remote:
# LOCAL BUILD
try:
output = build_and_upload_locally(
args.src_dir,
args.mode,
overwrite=args.overwrite,
archive=args.archive,
publish=args.publish,
destination_override=args.destination,
version_override=args.version_override,
bill_to_override=args.bill_to,
use_temp_build_project=args.use_temp_build_project,
do_parallel_build=args.parallel_build,
do_version_autonumbering=args.version_autonumbering,
do_try_update=args.update,
dx_toolkit_autodep=args.dx_toolkit_autodep,
do_build_step=args.build_step,
do_upload_step=args.upload_step,
do_check_syntax=args.check_syntax,
dry_run=args.dry_run,
return_object_dump=args.json,
**extra_args
)
if output is not None:
print json.dumps(output)
except dxpy.app_builder.AppBuilderException as e:
# AppBuilderException represents errors during app or applet building
# that could reasonably have been anticipated by the user.
print >> sys.stderr, "Error: %s" % (e.message,)
sys.exit(3)
except dxpy.exceptions.DXAPIError as e:
print >> sys.stderr, "Error: %s" % (e,)
sys.exit(3)
if args.run is not None:
if output is None:
err_exit("The --run option was given, but no executable was created")
cmd = os.path.join(os.environ['DNANEXUS_HOME'], 'bin', 'dx')
os.execv(cmd, ['dx', 'run', output['id']] + args.run)
return
else:
# REMOTE BUILD
try:
app_json = _parse_app_spec(args.src_dir)
_verify_app_source_dir(args.src_dir)
if args.mode == "app" and not args.dry_run:
_verify_app_writable(app_json['name'])
except dxpy.app_builder.AppBuilderException as e:
print >> sys.stderr, "Error: %s" % (e.message,)
sys.exit(3)
# The following flags might be useful in conjunction with
# --remote. To enable these, we need to learn how to pass these
# options through to the interior call of dx_build_app(let).
if args.dry_run:
parser.error('--remote cannot be combined with --dry-run')
if args.overwrite:
parser.error('--remote cannot be combined with --overwrite/-f')
if args.archive:
parser.error('--remote cannot be combined with --archive/-a')
# The following flags are probably not useful in conjunction
# with --remote.
if not args.build_step:
parser.error('--remote cannot be combined with --no-build-step')
if not args.upload_step:
parser.error('--remote cannot be combined with --no-upload-step')
if args.json:
parser.error('--remote cannot be combined with --json')
if not args.use_temp_build_project:
parser.error('--remote cannot be combined with --no-temp-build-project')
more_kwargs = {}
if args.version_override:
more_kwargs['version_override'] = args.version_override
if args.bill_to:
more_kwargs['bill_to_override'] = args.bill_to
if not args.version_autonumbering:
more_kwargs['do_version_autonumbering'] = False
if not args.update:
more_kwargs['do_try_update'] = False
if not args.parallel_build:
more_kwargs['do_parallel_build'] = False
if not args.check_syntax:
more_kwargs['do_check_syntax'] = False
return _build_app_remote(args.mode, args.src_dir, destination_override=args.destination, publish=args.publish, dx_toolkit_autodep=args.dx_toolkit_autodep, **more_kwargs)
def mount_all_inputs(exclude=None, verbose=False):
'''
:param exclude: List of input variables that should not be mounted.
:type exclude: Array of strings
:returns: dict of lists of strings where each key is the input variable
and each list element is the full path to the file that has
been mounted.
:param verbose: Start dxfuse with '-verbose 2' logging
:type verbose: boolean
This function mounts all files that were supplied as inputs to the app.
By convention, if an input parameter "FOO" has value
{"$dnanexus_link": "file-xxxx"}
and filename INPUT.TXT, then the linked file will be mounted into the
path:
$HOME/in/FOO/INPUT.TXT
If an input is an array of files, then all files will be placed into
numbered subdirectories under a parent directory named for the
input. For example, if the input key is FOO, and the inputs are {A, B,
C}.vcf then, the directory structure will be:
$HOME/in/FOO/0/A.vcf
1/B.vcf
2/C.vcf
Zero padding is used to ensure argument order. For example, if there are
12 input files {A, B, C, D, E, F, G, H, I, J, K, L}.txt, the directory
structure will be:
$HOME/in/FOO/00/A.vcf
...
11/L.vcf
This allows using shell globbing (FOO/*/*.vcf) to get all the files in the input
order and prevents issues with files which have the same filename.'''
print("Mounting inputs...")
home_dir = os.environ["HOME"]
mount_dir = os.path.join(home_dir, "in")
mount_manifest_file = os.path.join(home_dir, "mount-manifest.json")
dxfuse_cmd = _which("dxfuse")
if dxfuse_cmd is None:
err_exit("dxfuse is not installed on this system")
subprocess.check_output(["mkdir", mount_dir])
try:
job_input_file = file_load_utils.get_input_json_file()
dirs, inputs, rest = file_load_utils.get_job_input_filenames(job_input_file)
except IOError:
msg = 'Error: Could not find the input json file: {0}.\n'.format(job_input_file)
msg += ' This function should only be called from within a running job.'
print(msg)
raise
# Remove excluded inputs
if exclude:
inputs = file_load_utils.filter_dict(inputs, exclude)
# Convert to a flat list of elements to mount
to_mount = []
for ival_list in inputs.values():
to_mount.extend(ival_list)
files_manifest = _build_mount_manifest(to_mount)
with open(mount_manifest_file, 'w') as mfile:
json.dump(files_manifest, mfile)
dxfuse_version = subprocess.check_output([dxfuse_cmd, "-version"])
print("Using dxfuse version " + str(dxfuse_version))
uid = str(int(subprocess.check_output(["id", "-u"])))
gid = str(int(subprocess.check_output(["id", "-g"])))
cmd = [dxfuse_cmd, "-uid", uid, "-gid", gid, mount_dir, mount_manifest_file]
if verbose:
cmd[1:1] = ["-verbose", "2"]
print(subprocess.check_output(cmd))
print("Done mounting inputs.")
subprocess.call(["find", mount_dir, "-name", "*"])
helper_vars = _gen_helper_dict(inputs)
return helper_vars
def main(**kwargs):
"""
Entry point for dx-build-app(let).
Don't call this function as a subroutine in your program! It is liable to
sys.exit your program when it detects certain error conditions, so you
can't recover from those as you could if it raised exceptions. Instead,
call dx_build_app.build_and_upload_locally which provides the real
implementation for dx-build-app(let) but is easier to use in your program.
"""
if len(sys.argv) > 0:
if sys.argv[0].endswith('dx-build-app'):
logging.warn(
'Warning: dx-build-app has been replaced with "dx build --create-app". Please update your scripts.'
)
elif sys.argv[0].endswith('dx-build-applet'):
logging.warn(
'Warning: dx-build-applet has been replaced with "dx build". Please update your scripts.'
)
if len(kwargs) == 0:
args = parser.parse_args()
else:
args = parser.parse_args(**kwargs)
if dxpy.AUTH_HELPER is None and not args.dry_run:
parser.error(
'Authentication required to build an executable on the platform; please run "dx login" first'
)
if args.src_dir is None:
args.src_dir = os.getcwd()
if args.mode == "app" and args.destination != '.':
parser.error(
"--destination cannot be used when creating an app (only an applet)"
)
if args.dx_toolkit_autodep in ['beta', 'unstable']:
logging.warn(
'The --dx-toolkit-beta-autodep and --dx-toolkit-unstable-autodep flags have no effect and will be removed at some date in the future.'
)
if args.overwrite and args.archive:
parser.error(
"Options -f/--overwrite and -a/--archive cannot be specified together"
)
extra_args = json.loads(args.extra_args) if args.extra_args else {}
if not args.remote:
# LOCAL BUILD
try:
output = build_and_upload_locally(
args.src_dir,
args.mode,
overwrite=args.overwrite,
archive=args.archive,
publish=args.publish,
destination_override=args.destination,
version_override=args.version_override,
bill_to_override=args.bill_to,
use_temp_build_project=args.use_temp_build_project,
do_parallel_build=args.parallel_build,
do_version_autonumbering=args.version_autonumbering,
do_try_update=args.update,
dx_toolkit_autodep=args.dx_toolkit_autodep,
do_build_step=args.build_step,
do_upload_step=args.upload_step,
do_check_syntax=args.check_syntax,
dry_run=args.dry_run,
return_object_dump=args.json,
**extra_args)
if output is not None:
print json.dumps(output)
except dxpy.app_builder.AppBuilderException as e:
# AppBuilderException represents errors during app or applet building
# that could reasonably have been anticipated by the user.
print >> sys.stderr, "Error: %s" % (e.message, )
sys.exit(3)
except dxpy.exceptions.DXAPIError as e:
print >> sys.stderr, "Error: %s" % (e, )
sys.exit(3)
if args.run is not None:
if output is None:
err_exit(
"The --run option was given, but no executable was created"
)
cmd = os.path.join(os.environ['DNANEXUS_HOME'], 'bin', 'dx')
os.execv(cmd, ['dx', 'run', output['id']] + args.run)
return
else:
# REMOTE BUILD
try:
app_json = _parse_app_spec(args.src_dir)
_verify_app_source_dir(args.src_dir)
if args.mode == "app" and not args.dry_run:
_verify_app_writable(app_json['name'])
except dxpy.app_builder.AppBuilderException as e:
print >> sys.stderr, "Error: %s" % (e.message, )
sys.exit(3)
# The following flags might be useful in conjunction with
# --remote. To enable these, we need to learn how to pass these
# options through to the interior call of dx_build_app(let).
if args.dry_run:
parser.error('--remote cannot be combined with --dry-run')
if args.overwrite:
parser.error('--remote cannot be combined with --overwrite/-f')
if args.archive:
parser.error('--remote cannot be combined with --archive/-a')
# The following flags are probably not useful in conjunction
# with --remote.
if not args.build_step:
parser.error('--remote cannot be combined with --no-build-step')
if not args.upload_step:
parser.error('--remote cannot be combined with --no-upload-step')
if args.json:
parser.error('--remote cannot be combined with --json')
if not args.use_temp_build_project:
parser.error(
'--remote cannot be combined with --no-temp-build-project')
more_kwargs = {}
if args.version_override:
more_kwargs['version_override'] = args.version_override
if args.bill_to:
more_kwargs['bill_to_override'] = args.bill_to
if not args.version_autonumbering:
more_kwargs['do_version_autonumbering'] = False
if not args.update:
more_kwargs['do_try_update'] = False
if not args.parallel_build:
more_kwargs['do_parallel_build'] = False
if not args.check_syntax:
more_kwargs['do_check_syntax'] = False
return _build_app_remote(args.mode,
args.src_dir,
destination_override=args.destination,
publish=args.publish,
dx_toolkit_autodep=args.dx_toolkit_autodep,
**more_kwargs)
def main(**kwargs):
"""
Entry point for dx-build-app(let).
Don't call this function as a subroutine in your program! It is liable to
sys.exit your program when it detects certain error conditions, so you
can't recover from those as you could if it raised exceptions. Instead,
call dx_build_app.build_and_upload_locally which provides the real
implementation for dx-build-app(let) but is easier to use in your program.
"""
if len(sys.argv) > 0:
if sys.argv[0].endswith('dx-build-app'):
logging.warn('Warning: dx-build-app has been replaced with "dx build --create-app". Please update your scripts.')
elif sys.argv[0].endswith('dx-build-applet'):
logging.warn('Warning: dx-build-applet has been replaced with "dx build". Please update your scripts.')
if len(kwargs) == 0:
args = parser.parse_args()
else:
args = parser.parse_args(**kwargs)
if dxpy.AUTH_HELPER is None and not args.dry_run:
parser.error('Authentication required to build an executable on the platform; please run "dx login" first')
if args.src_dir is None:
args.src_dir = os.getcwd()
if USING_PYTHON2:
args.src_dir = args.src_dir.decode(sys.getfilesystemencoding())
if args.mode == "app" and args.destination != '.':
parser.error("--destination cannot be used when creating an app (only an applet)")
if args.dx_toolkit_autodep in ['beta', 'unstable']:
logging.warn('The --dx-toolkit-beta-autodep and --dx-toolkit-unstable-autodep flags have no effect and will be removed at some date in the future.')
if args.overwrite and args.archive:
parser.error("Options -f/--overwrite and -a/--archive cannot be specified together")
if args.run is not None and args.dry_run:
parser.error("Options --dry-run and --run cannot be specified together")
if args.run and args.remote and args.mode == 'app':
parser.error("Options --remote, --app, and --run cannot all be specified together. Try removing --run and then separately invoking dx run.")
executable_id = _build_app(args,
json.loads(args.extra_args) if args.extra_args else {})
if args.run is not None:
if executable_id is None:
raise AssertionError('Expected executable_id to be set here')
try:
subprocess.check_call(['dx', 'run', executable_id, '--priority', 'high'] + args.run)
except subprocess.CalledProcessError as e:
sys.exit(e.returncode)
except:
err_exit()
return
