def _populate_matches(self, prefix): try: name_query = None if len(prefix) > 0: if prefix.startswith("app-") and len(prefix) > 4: name_query = prefix[4:] + "*" elif len(prefix) > 4 or not "app-".startswith(prefix): name_query = prefix + "*" appnames = [ result["describe"]["name"] for result in dxpy.find_apps( name=name_query, name_mode="glob", describe={"fields": {"name": True, "installed": (self.installed is not None)}}, ) if self.installed is None or (self.installed == result["describe"]["installed"]) ] except: # This is for (temporary) backwards-compatibility appnames = [ result["describe"]["name"] for result in dxpy.find_apps(describe=True) if self.installed is None or (self.installed == result["describe"]["installed"]) ] self.matches = [name for name in appnames if name.startswith(prefix)] if prefix != "" and prefix.startswith("app-"[: len(prefix)]): appnames_with_prefix = [("app-" + name) for name in appnames] self.matches += [name for name in appnames_with_prefix if name.startswith(prefix)]
def get_notebook_app_versions(): """ Get the valid version numbers of the notebook app. """ notebook_apps = dxpy.find_apps(name=NOTEBOOK_APP, all_versions=True) versions = [str(dxpy.describe(app['id'])['version']) for app in notebook_apps] return versions
def _populate_matches(self, prefix): try: name_query = None if len(prefix) > 0: if prefix.startswith("app-") and len(prefix) > 4: name_query = prefix[4:] + "*" elif len(prefix) > 4 or not "app-".startswith(prefix): name_query = prefix + "*" appnames = [result['describe']['name'] for result in dxpy.find_apps(name=name_query, name_mode="glob", describe={"fields": {"name": True, "installed": (self.installed is not None)}}) if self.installed is None or (self.installed == result['describe']['installed'])] except: # This is for (temporary) backwards-compatibility appnames = [result['describe']['name'] for result in dxpy.find_apps(describe=True) if self.installed is None or (self.installed == result['describe']['installed'])] self.matches = [name for name in appnames if name.startswith(prefix)] if prefix != '' and prefix.startswith('app-'[:len(prefix)]): appnames_with_prefix = [('app-' + name) for name in appnames] self.matches += [name for name in appnames_with_prefix if name.startswith(prefix)]
def get_notebook_app_versions(): """ Get the valid version numbers of the notebook app. """ notebook_apps = dxpy.find_apps(name=NOTEBOOK_APP, all_versions=True) versions = [ str(dxpy.describe(app['id'])['version']) for app in notebook_apps ] return versions
def map_contaminant(Contig, Reads): # get ID of our mapper try: bwa = dxpy.DXApp(dxpy.find_apps(name="bwa").next()['id']) except StopIteration: raise dxpy.AppError("Unable to find app 'bwa'. Please install it to enable contaminant mapping") # TODO: find optimal chunk size so we don't launch too many bwa jobs map_job = bwa.run({"reads":Reads, "reference": Contig, "discard_unmapped_rows":True, "chunk_size":10000000}) total_reads = 0 for r in Reads: desc = dxpy.DXGTable(r).describe() current_reads = desc['length'] if 'sequence2' in desc['columns']: current_reads *= 2 total_reads += current_reads # launch a job to wait for the mapping and will calculate what % has mapped calc_job = dxpy.new_dxjob({"num_reads":total_reads, "mappings":{"job":map_job.get_id(), "field":"mappings"}}, "calc_contam") return calc_job.get_id()
def map_contaminant(Contig, Reads): # get ID of our mapper try: bwa = dxpy.DXApp( dxpy.find_apps(name="bwa_mem_fastq_read_mapper").next()['id']) except StopIteration: raise dxpy.AppError( "Unable to find app 'bwa_mem_fastq_read_mapper'. Please install it to enable contaminant mapping" ) # TODO: find optimal chunk size so we don't launch too many bwa jobs map_job = bwa.run({ "reads": Reads, "reference": Contig, "discard_unmapped_rows": True, "chunk_size": 10000000 }) total_reads = 0 for r in Reads: desc = dxpy.DXGTable(r).describe() current_reads = desc['length'] if 'sequence2' in desc['columns']: current_reads *= 2 total_reads += current_reads # launch a job to wait for the mapping and will calculate what % has mapped calc_job = dxpy.new_dxjob( { "num_reads": total_reads, "mappings": { "job": map_job.get_id(), "field": "mappings" } }, "calc_contam") return calc_job.get_id()
def _create_app(applet_or_regional_options, app_name, src_dir, publish=False, set_default=False, billTo=None, try_versions=None, try_update=True, confirm=True): app_spec = _get_app_spec(src_dir) logger.info("Will create app with spec: %s" % (json.dumps(app_spec), )) app_spec.update(applet_or_regional_options, name=app_name) # Inline Readme.md and Readme.developer.md dxpy.executable_builder.inline_documentation_files(app_spec, src_dir) if billTo: app_spec["billTo"] = billTo if not try_versions: try_versions = [app_spec["version"]] for version in try_versions: logger.debug("Attempting to create version %s..." % (version, )) app_spec['version'] = version app_describe = None try: # 404, which is rather likely in this app_describe request # (the purpose of the request is to find out whether the # version of interest exists), would ordinarily cause this # request to be retried multiple times, introducing a # substantial delay. So we disable retrying here for this # request. app_describe = dxpy.api.app_describe("app-" + app_spec["name"], alias=version, always_retry=False) except dxpy.exceptions.DXAPIError as e: if e.name == 'ResourceNotFound': pass else: raise e # Now app_describe is None if the app didn't exist, OR it contains the # app describe content. # The describe check does not eliminate race conditions since an app # may always have been created, or published, since we last looked at # it. So the describe that happens here is just to save time and avoid # unnecessary API calls, but we always have to be prepared to recover # from API errors. if app_describe is None: logger.debug('App %s/%s does not yet exist' % (app_spec["name"], version)) app_id = _create_or_update_version(app_spec['name'], app_spec['version'], app_spec, try_update=try_update) if app_id is None: continue logger.info("Created app " + app_id) # Success! break elif app_describe.get("published", 0) == 0: logger.debug( 'App %s/%s already exists and has not been published' % (app_spec["name"], version)) app_id = _update_version(app_spec['name'], app_spec['version'], app_spec, try_update=try_update) if app_id is None: continue logger.info("Updated existing app " + app_id) # Success! break else: logger.debug('App %s/%s already exists and has been published' % (app_spec["name"], version)) # App has already been published. Give up on this version. continue else: # All versions requested failed if len(try_versions) != 1: tried_versions = 'any of the requested versions: ' + ', '.join( try_versions) else: tried_versions = 'the requested version: ' + try_versions[0] raise AppBuilderException('Could not create %s' % (tried_versions, )) # Set categories appropriately. categories_to_set = app_spec.get("categories", []) existing_categories = dxpy.api.app_list_categories(app_id)['categories'] categories_to_add = set(categories_to_set).difference( set(existing_categories)) categories_to_remove = set(existing_categories).difference( set(categories_to_set)) if categories_to_add: dxpy.api.app_add_categories( app_id, input_params={'categories': list(categories_to_add)}) if categories_to_remove: dxpy.api.app_remove_categories( app_id, input_params={'categories': list(categories_to_remove)}) # Set developers list appropriately, but only if provided. developers_to_set = app_spec.get("developers") if developers_to_set is not None: existing_developers = dxpy.api.app_list_developers( app_id)['developers'] developers_to_add = set(developers_to_set) - set(existing_developers) developers_to_remove = set(existing_developers) - set( developers_to_set) skip_updating_developers = False if developers_to_add or developers_to_remove: parts = [] if developers_to_add: parts.append('the following developers will be added: ' + ', '.join(sorted(developers_to_add))) if developers_to_remove: parts.append('the following developers will be removed: ' + ', '.join(sorted(developers_to_remove))) developer_change_message = '; and '.join(parts) if confirm: if INTERACTIVE_CLI: try: print('***') print(fill('WARNING: ' + developer_change_message)) print('***') value = input( 'Confirm updating developers list [y/N]: ') except KeyboardInterrupt: value = 'n' if not value.lower().startswith('y'): skip_updating_developers = True else: # Default to NOT updating developers if operating # without a TTY. logger.warn( 'skipping requested change to the developer list. Rerun "dx build" interactively or pass --yes to confirm this change.' ) skip_updating_developers = True else: logger.warn(developer_change_message) if not skip_updating_developers: if developers_to_add: dxpy.api.app_add_developers( app_id, input_params={'developers': list(developers_to_add)}) if developers_to_remove: dxpy.api.app_remove_developers( app_id, input_params={'developers': list(developers_to_remove)}) # Set authorizedUsers list appropriately, but only if provided. authorized_users_to_set = app_spec.get("authorizedUsers") existing_authorized_users = dxpy.api.app_list_authorized_users( app_id)['authorizedUsers'] if authorized_users_to_set is not None: authorized_users_to_add = set(authorized_users_to_set) - set( existing_authorized_users) authorized_users_to_remove = set(existing_authorized_users) - set( authorized_users_to_set) skip_adding_public = False if 'PUBLIC' in authorized_users_to_add: acl_change_message = 'app-%s will be made public. Anyone will be able to view and run all published versions of this app.' % ( app_spec['name'], ) if confirm: if INTERACTIVE_CLI: try: print('***') print(fill('WARNING: ' + acl_change_message)) print('***') value = input('Confirm making this app public [y/N]: ') except KeyboardInterrupt: value = 'n' if not value.lower().startswith('y'): skip_adding_public = True else: # Default to NOT adding PUBLIC if operating # without a TTY. logger.warn( 'skipping requested change to add PUBLIC to the authorized users list. Rerun "dx build" interactively or pass --yes to confirm this change.' ) skip_adding_public = True else: logger.warn(acl_change_message) if skip_adding_public: authorized_users_to_add -= {'PUBLIC'} if authorized_users_to_add: dxpy.api.app_add_authorized_users(app_id, input_params={ 'authorizedUsers': list(authorized_users_to_add) }) if skip_adding_public: logger.warn( 'the app was NOT made public as requested in the dxapp.json. To make it so, run "dx add users app-%s PUBLIC".' % (app_spec["name"], )) if authorized_users_to_remove: dxpy.api.app_remove_authorized_users( app_id, input_params={ 'authorizedUsers': list(authorized_users_to_remove) }) elif not len(existing_authorized_users): # Apps that had authorized users added by any other means will # not have this message printed. logger.warn( 'authorizedUsers is missing from the dxapp.json. No one will be able to view or run the app except the app\'s developers.' ) if publish: dxpy.api.app_publish(app_id, input_params={'makeDefault': set_default}) else: # If no versions of this app have ever been published, then # we'll set the "default" tag to point to the latest # (unpublished) version. no_published_versions = len( list(dxpy.find_apps(name=app_name, published=True, limit=1))) == 0 if no_published_versions: dxpy.api.app_add_tags(app_id, input_params={'tags': ['default']}) return app_id
def create_app(applet_id, applet_name, src_dir, publish=False, set_default=False, billTo=None, try_versions=None, try_update=True, confirm=True): """ Creates a new app object from the specified applet. """ app_spec = _get_app_spec(src_dir) logger.info("Will create app with spec: %s" % (app_spec,)) app_spec["applet"] = applet_id app_spec["name"] = applet_name # Inline Readme.md and Readme.developer.md _inline_documentation_files(app_spec, src_dir) if billTo: app_spec["billTo"] = billTo if not try_versions: try_versions = [app_spec["version"]] for version in try_versions: logger.debug("Attempting to create version %s..." % (version,)) app_spec['version'] = version app_describe = None try: # 404, which is rather likely in this app_describe request # (the purpose of the request is to find out whether the # version of interest exists), would ordinarily cause this # request to be retried multiple times, introducing a # substantial delay. So we disable retrying here for this # request. app_describe = dxpy.api.app_describe("app-" + app_spec["name"], alias=version, always_retry=False) except dxpy.exceptions.DXAPIError as e: if e.name == 'ResourceNotFound': pass else: raise e # Now app_describe is None if the app didn't exist, OR it contains the # app describe content. # The describe check does not eliminate race conditions since an app # may always have been created, or published, since we last looked at # it. So the describe that happens here is just to save time and avoid # unnecessary API calls, but we always have to be prepared to recover # from API errors. if app_describe is None: logger.debug('App %s/%s does not yet exist' % (app_spec["name"], version)) app_id = _create_or_update_version(app_spec['name'], app_spec['version'], app_spec, try_update=try_update) if app_id is None: continue logger.info("Created app " + app_id) # Success! break elif app_describe.get("published", 0) == 0: logger.debug('App %s/%s already exists and has not been published' % (app_spec["name"], version)) app_id = _update_version(app_spec['name'], app_spec['version'], app_spec, try_update=try_update) if app_id is None: continue logger.info("Updated existing app " + app_id) # Success! break else: logger.debug('App %s/%s already exists and has been published' % (app_spec["name"], version)) # App has already been published. Give up on this version. continue else: # All versions requested failed if len(try_versions) != 1: tried_versions = 'any of the requested versions: ' + ', '.join(try_versions) else: tried_versions = 'the requested version: ' + try_versions[0] raise AppBuilderException('Could not create %s' % (tried_versions,)) # Set categories appropriately. categories_to_set = app_spec.get("categories", []) existing_categories = dxpy.api.app_list_categories(app_id)['categories'] categories_to_add = set(categories_to_set).difference(set(existing_categories)) categories_to_remove = set(existing_categories).difference(set(categories_to_set)) if categories_to_add: dxpy.api.app_add_categories(app_id, input_params={'categories': list(categories_to_add)}) if categories_to_remove: dxpy.api.app_remove_categories(app_id, input_params={'categories': list(categories_to_remove)}) # Set authorizedUsers list appropriately, but only if provided. authorized_users_to_set = app_spec.get("authorizedUsers") existing_authorized_users = dxpy.api.app_list_authorized_users(app_id)['authorizedUsers'] if authorized_users_to_set is not None: authorized_users_to_add = set(authorized_users_to_set) - set(existing_authorized_users) authorized_users_to_remove = set(existing_authorized_users) - set(authorized_users_to_set) skip_adding_public = False if 'PUBLIC' in authorized_users_to_add: acl_change_message = 'app-%s will be made public. Anyone will be able to view and run all published versions of this app.' % (app_spec['name'],) if confirm: if sys.stdout.isatty(): try: print('***') print(fill('WARNING: ' + acl_change_message)) print('***') value = input('Confirm making this app public [y/N]: ') except KeyboardInterrupt: value = 'n' if not value.lower().startswith('y'): skip_adding_public = True else: # Default to NOT adding PUBLIC if operating # without a TTY. skip_adding_public = True else: logger.warn(acl_change_message) if skip_adding_public: authorized_users_to_add -= {'PUBLIC'} if authorized_users_to_add: dxpy.api.app_add_authorized_users(app_id, input_params={'authorizedUsers': list(authorized_users_to_add)}) if skip_adding_public: logger.warn('the app was NOT made public as requested in the dxapp.json. To make it so, run "dx add users app-%s PUBLIC".' % (app_spec["name"],)) if authorized_users_to_remove: dxpy.api.app_remove_authorized_users(app_id, input_params={'authorizedUsers': list(authorized_users_to_remove)}) elif not len(existing_authorized_users): # Apps that had authorized users added by any other means will # not have this message printed. logger.warn('authorizedUsers is missing from the dxapp.json. No one will be able to view or run the app except the app\'s developers.') if publish: dxpy.api.app_publish(app_id, input_params={'makeDefault': set_default}) else: # If no versions of this app have ever been published, then # we'll set the "default" tag to point to the latest # (unpublished) version. no_published_versions = len(list(dxpy.find_apps(name=applet_name, published=True, limit=1))) == 0 if no_published_versions: dxpy.api.app_add_tags(app_id, input_params={'tags': ['default']}) return app_id
def create_app(applet_id, applet_name, src_dir, publish=False, set_default=False, billTo=None, try_versions=None, try_update=True): """ Creates a new app object from the specified applet. """ app_spec = _get_app_spec(src_dir) print >> sys.stderr, "Will create app with spec: ", app_spec app_spec["applet"] = applet_id app_spec["name"] = applet_name # Inline Readme.md and Readme.developer.md _inline_documentation_files(app_spec, src_dir) if billTo: app_spec["billTo"] = billTo if not try_versions: try_versions = [app_spec["version"]] for version in try_versions: print >> sys.stderr, "Attempting to create version %s..." % (version,) app_spec['version'] = version app_describe = None try: # 404, which is rather likely in this app_describe request # (the purpose of the request is to find out whether the # version of interest exists), would ordinarily cause this # request to be retried multiple times, introducing a # substantial delay. So we disable retrying here for this # request. app_describe = dxpy.api.app_describe("app-" + app_spec["name"], alias=version, always_retry=False) except dxpy.exceptions.DXAPIError as e: if e.name == 'ResourceNotFound': pass else: raise e # Now app_describe is None if the app didn't exist, OR it contains the # app describe content. # The describe check does not eliminate race conditions since an app # may always have been created, or published, since we last looked at # it. So the describe that happens here is just to save time and avoid # unnecessary API calls, but we always have to be prepared to recover # from API errors. if app_describe is None: print >> sys.stderr, 'App %s/%s does not yet exist' % (app_spec["name"], version) app_id = _create_or_update_version(app_spec['name'], app_spec['version'], app_spec, try_update=try_update) if app_id is None: continue print >> sys.stderr, "Created app " + app_id # Success! break elif app_describe.get("published", 0) == 0: print >> sys.stderr, 'App %s/%s already exists and has not been published' % (app_spec["name"], version) app_id = _update_version(app_spec['name'], app_spec['version'], app_spec, try_update=try_update) if app_id is None: continue print >> sys.stderr, "Updated existing app " + app_id # Success! break else: print >> sys.stderr, 'App %s/%s already exists and has been published' % (app_spec["name"], version) # App has already been published. Give up on this version. continue else: # All versions requested failed if len(try_versions) != 1: tried_versions = 'any of the requested versions: ' + ', '.join(try_versions) else: tried_versions = 'the requested version: ' + try_versions[0] raise AppBuilderException('Could not create %s' % (tried_versions,)) # Set categories appropriately. categories_to_set = app_spec.get("categories", []) existing_categories = dxpy.api.app_list_categories(app_id)['categories'] categories_to_add = set(categories_to_set).difference(set(existing_categories)) categories_to_remove = set(existing_categories).difference(set(categories_to_set)) if categories_to_add: dxpy.api.app_add_categories(app_id, input_params={'categories': list(categories_to_add)}) if categories_to_remove: dxpy.api.app_remove_categories(app_id, input_params={'categories': list(categories_to_remove)}) # Set authorizedUsers list appropriately, but only if provided. authorized_users_to_set = app_spec.get("authorizedUsers") if authorized_users_to_set is not None: existing_authorized_users = dxpy.api.app_list_authorized_users(app_id)['authorizedUsers'] authorized_users_to_add = set(authorized_users_to_set) - set(existing_authorized_users) authorized_users_to_remove = set(existing_authorized_users) - set(authorized_users_to_set) if authorized_users_to_add: dxpy.api.app_add_authorized_users(app_id, input_params={'authorizedUsers': list(authorized_users_to_add)}) if authorized_users_to_remove: dxpy.api.app_remove_authorized_users(app_id, input_params={'authorizedUsers': list(authorized_users_to_remove)}) if publish: dxpy.api.app_publish(app_id, input_params={'makeDefault': set_default}) else: # If no versions of this app have ever been published, then # we'll set the "default" tag to point to the latest # (unpublished) version. no_published_versions = len(list(dxpy.find_apps(name=applet_name, published=True, limit=1))) == 0 if no_published_versions: dxpy.api.app_add_tags(app_id, input_params={'tags': ['default']}) return app_id
def create_app(applet_id, applet_name, src_dir, publish=False, set_default=False, billTo=None, try_versions=None, try_update=True): """ Creates a new app object from the specified applet. """ app_spec = _get_app_spec(src_dir) print >> sys.stderr, "Will create app with spec: ", app_spec app_spec["applet"] = applet_id app_spec["name"] = applet_name # Inline Readme.md and Readme.developer.md _inline_documentation_files(app_spec, src_dir) if billTo: app_spec["billTo"] = billTo if not try_versions: try_versions = [app_spec["version"]] for version in try_versions: print >> sys.stderr, "Attempting to create version %s..." % (version, ) app_spec['version'] = version app_describe = None try: # 404, which is rather likely in this app_describe request # (the purpose of the request is to find out whether the # version of interest exists), would ordinarily cause this # request to be retried multiple times, introducing a # substantial delay. So we disable retrying here for this # request. app_describe = dxpy.api.app_describe("app-" + app_spec["name"], alias=version, always_retry=False) except dxpy.exceptions.DXAPIError as e: if e.name == 'ResourceNotFound': pass else: raise e # Now app_describe is None if the app didn't exist, OR it contains the # app describe content. # The describe check does not eliminate race conditions since an app # may always have been created, or published, since we last looked at # it. So the describe that happens here is just to save time and avoid # unnecessary API calls, but we always have to be prepared to recover # from API errors. if app_describe is None: print >> sys.stderr, 'App %s/%s does not yet exist' % ( app_spec["name"], version) app_id = _create_or_update_version(app_spec['name'], app_spec['version'], app_spec, try_update=try_update) if app_id is None: continue print >> sys.stderr, "Created app " + app_id # Success! break elif app_describe.get("published", 0) == 0: print >> sys.stderr, 'App %s/%s already exists and has not been published' % ( app_spec["name"], version) app_id = _update_version(app_spec['name'], app_spec['version'], app_spec, try_update=try_update) if app_id is None: continue print >> sys.stderr, "Updated existing app " + app_id # Success! break else: print >> sys.stderr, 'App %s/%s already exists and has been published' % ( app_spec["name"], version) # App has already been published. Give up on this version. continue else: # All versions requested failed if len(try_versions) != 1: tried_versions = 'any of the requested versions: ' + ', '.join( try_versions) else: tried_versions = 'the requested version: ' + try_versions[0] raise AppBuilderException('Could not create %s' % (tried_versions, )) # Set categories appropriately. categories_to_set = app_spec.get("categories", []) existing_categories = dxpy.api.app_list_categories(app_id)['categories'] categories_to_add = set(categories_to_set).difference( set(existing_categories)) categories_to_remove = set(existing_categories).difference( set(categories_to_set)) if categories_to_add: dxpy.api.app_add_categories( app_id, input_params={'categories': list(categories_to_add)}) if categories_to_remove: dxpy.api.app_remove_categories( app_id, input_params={'categories': list(categories_to_remove)}) # Set authorizedUsers list appropriately, but only if provided. authorized_users_to_set = app_spec.get("authorizedUsers") if authorized_users_to_set is not None: existing_authorized_users = dxpy.api.app_list_authorized_users( app_id)['authorizedUsers'] authorized_users_to_add = set(authorized_users_to_set) - set( existing_authorized_users) authorized_users_to_remove = set(existing_authorized_users) - set( authorized_users_to_set) if authorized_users_to_add: dxpy.api.app_add_authorized_users(app_id, input_params={ 'authorizedUsers': list(authorized_users_to_add) }) if authorized_users_to_remove: dxpy.api.app_remove_authorized_users( app_id, input_params={ 'authorizedUsers': list(authorized_users_to_remove) }) if publish: dxpy.api.app_publish(app_id, input_params={'makeDefault': set_default}) else: # If no versions of this app have ever been published, then # we'll set the "default" tag to point to the latest # (unpublished) version. no_published_versions = len( list(dxpy.find_apps(name=applet_name, published=True, limit=1))) == 0 if no_published_versions: dxpy.api.app_add_tags(app_id, input_params={'tags': ['default']}) return app_id