def __extract_payload_from_request(trans, func, kwargs): content_type = trans.request.headers['content-type'] if content_type.startswith('application/x-www-form-urlencoded') or content_type.startswith('multipart/form-data'): # If the content type is a standard type such as multipart/form-data, the wsgi framework parses the request body # and loads all field values into kwargs. However, kwargs also contains formal method parameters etc. which # are not a part of the request body. This is a problem because it's not possible to differentiate between values # which are a part of the request body, and therefore should be a part of the payload, and values which should not be # in the payload. Therefore, the decorated method's formal arguments are discovered through reflection and removed from # the payload dictionary. This helps to prevent duplicate argument conflicts in downstream methods. payload = kwargs.copy() named_args, _, _, _ = inspect.getargspec(func) for arg in named_args: payload.pop(arg, None) for k, v in payload.iteritems(): if isinstance(v, (str, unicode)): try: payload[k] = loads(v) except: # may not actually be json, just continue pass payload = util.recursively_stringify_dictionary_keys( payload ) else: # Assume application/json content type and parse request body manually, since wsgi won't do it. However, the order of this check # should ideally be in reverse, with the if clause being a check for application/json and the else clause assuming a standard encoding # such as multipart/form-data. Leaving it as is for backward compatibility, just in case. payload = util.recursively_stringify_dictionary_keys( loads( trans.request.body ) ) return payload
def __extract_payload_from_request(trans, func, kwargs): content_type = trans.request.headers['content-type'] if content_type.startswith( 'application/x-www-form-urlencoded') or content_type.startswith( 'multipart/form-data'): # If the content type is a standard type such as multipart/form-data, the wsgi framework parses the request body # and loads all field values into kwargs. However, kwargs also contains formal method parameters etc. which # are not a part of the request body. This is a problem because it's not possible to differentiate between values # which are a part of the request body, and therefore should be a part of the payload, and values which should not be # in the payload. Therefore, the decorated method's formal arguments are discovered through reflection and removed from # the payload dictionary. This helps to prevent duplicate argument conflicts in downstream methods. payload = kwargs.copy() named_args, _, _, _ = inspect.getargspec(func) for arg in named_args: payload.pop(arg, None) for k, v in payload.iteritems(): if isinstance(v, (str, unicode)): try: payload[k] = loads(v) except: # may not actually be json, just continue pass payload = util.recursively_stringify_dictionary_keys(payload) else: # Assume application/json content type and parse request body manually, since wsgi won't do it. However, the order of this check # should ideally be in reverse, with the if clause being a check for application/json and the else clause assuming a standard encoding # such as multipart/form-data. Leaving it as is for backward compatibility, just in case. payload = util.recursively_stringify_dictionary_keys( loads(trans.request.body)) return payload
def recover_runtime_state( self, runtime_state ): """ Take secure runtime state from persisted invocation and convert it into a DefaultToolState object for use during workflow invocation. """ state = galaxy.tools.DefaultToolState() app = self.trans.app state.decode( runtime_state, self.tool, app, secure=False ) state_dict = loads( runtime_state ) if RUNTIME_STEP_META_STATE_KEY in state_dict: self.__restore_step_meta_runtime_state( loads( state_dict[ RUNTIME_STEP_META_STATE_KEY ] ) ) return state
def get_chrom_info(self, dbkey, trans=None, custom_build_hack_get_len_from_fasta_conversion=True): # FIXME: flag to turn off custom_build_hack_get_len_from_fasta_conversion should not be required chrom_info = None db_dataset = None # Collect chromInfo from custom builds if trans: db_dataset = trans.db_dataset_for(dbkey) if db_dataset: chrom_info = db_dataset.file_name else: # Do Custom Build handling if trans.user and ('dbkeys' in trans.user.preferences) and ( dbkey in loads(trans.user.preferences['dbkeys'])): custom_build_dict = loads( trans.user.preferences['dbkeys'])[dbkey] # HACK: the attempt to get chrom_info below will trigger the # fasta-to-len converter if the dataset is not available or, # which will in turn create a recursive loop when # running the fasta-to-len tool. So, use a hack in the second # condition below to avoid getting chrom_info when running the # fasta-to-len converter. if 'fasta' in custom_build_dict and custom_build_hack_get_len_from_fasta_conversion: # Build is defined by fasta; get len file, which is obtained from converting fasta. build_fasta_dataset = trans.sa_session.query( trans.app.model.HistoryDatasetAssociation).get( custom_build_dict['fasta']) chrom_info = build_fasta_dataset.get_converted_dataset( trans, 'len').file_name elif 'len' in custom_build_dict: # Build is defined by len file, so use it. chrom_info = trans.sa_session.query( trans.app.model.HistoryDatasetAssociation).get( custom_build_dict['len']).file_name # Check Data table if not chrom_info: dbkey_table = self._app.tool_data_tables.get( self._data_table_name, None) if dbkey_table is not None: chrom_info = dbkey_table.get_entry('value', dbkey, 'len_path', default=None) # use configured server len path if not chrom_info: # Default to built-in build. # Since we are using an unverified dbkey, we will sanitize the dbkey before use chrom_info = os.path.join( self._static_chrom_info_path, "%s.len" % sanitize_lists_to_string(dbkey)) chrom_info = os.path.abspath(chrom_info) return (chrom_info, db_dataset)
def get_genome_build_names(self, trans=None): # FIXME: how to deal with key duplicates? rval = [] # load user custom genome builds if trans is not None: if trans.history: # This is a little bit Odd. We are adding every .len file in the current history to dbkey list, # but this is previous behavior from trans.db_names, so we'll continue to do it. # It does allow one-off, history specific dbkeys to be created by a user. But we are not filtering, # so a len file will be listed twice (as the build name and again as dataset name), # if custom dbkey creation/conversion occurred within the current history. datasets = trans.sa_session.query( self._app.model.HistoryDatasetAssociation ) \ .filter_by( deleted=False, history_id=trans.history.id, extension="len" ) for dataset in datasets: rval.append( (dataset.dbkey, "%s (%s) [History]" % (dataset.name, dataset.dbkey))) user = trans.user if user and hasattr( user, 'preferences') and 'dbkeys' in user.preferences: user_keys = loads(user.preferences['dbkeys']) for key, chrom_dict in user_keys.iteritems(): rval.append( (key, "%s (%s) [Custom]" % (chrom_dict['name'], key))) # Load old builds.txt static keys rval.extend(self._static_dbkeys) #load dbkeys from dbkey data table dbkey_table = self._app.tool_data_tables.get(self._data_table_name, None) if dbkey_table is not None: for field_dict in dbkey_table.get_named_fields_list(): rval.append((field_dict['value'], field_dict['name'])) return rval
def sweepster( self, trans, id=None, hda_ldda=None, dataset_id=None, regions=None ): """ Displays a sweepster visualization using the incoming parameters. If id is available, get the visualization with the given id; otherwise, create a new visualization using a given dataset and regions. """ regions = regions or '{}' # Need to create history if necessary in order to create tool form. trans.get_history( create=True ) if id: # Loading a shared visualization. viz = self.get_visualization( trans, id ) viz_config = self.get_visualization_config( trans, viz ) decoded_id = self.decode_id( viz_config[ 'dataset_id' ] ) dataset = self.hda_manager.get_owned( decoded_id, trans.user, current_history=trans.history ) else: # Loading new visualization. dataset = self.get_hda_or_ldda( trans, hda_ldda, dataset_id ) job = self.hda_manager.creating_job( dataset ) viz_config = { 'dataset_id': dataset_id, 'tool_id': job.tool_id, 'regions': loads( regions ) } # Add tool, dataset attributes to config based on id. tool = trans.app.toolbox.get_tool( viz_config[ 'tool_id' ] ) viz_config[ 'tool' ] = tool.to_dict( trans, io_details=True ) viz_config[ 'dataset' ] = trans.security.encode_dict_ids( dataset.to_dict() ) return trans.fill_template_mako( "visualization/sweepster.mako", config=viz_config )
def from_workflow_step( Class, trans, step ): toolbox = trans.app.toolbox tool_id = step.tool_id if toolbox: # See if we have access to a different version of the tool. # TODO: If workflows are ever enhanced to use tool version # in addition to tool id, enhance the selection process here # to retrieve the correct version of the tool. tool_id = toolbox.get_tool_id( tool_id ) if ( toolbox and tool_id ): if step.config: # This step has its state saved in the config field due to the # tool being previously unavailable. return module_factory.from_dict(trans, loads(step.config), secure=False) tool_version = step.tool_version module = Class( trans, tool_id, tool_version=tool_version ) if step.tool_version and (step.tool_version != module.tool.version): message = "%s: using version '%s' instead of version '%s' indicated in this workflow." % (tool_id, module.tool.version, step.tool_version) log.debug(message) module.version_changes.append(message) module.recover_state( step.tool_inputs ) module.errors = step.tool_errors module.workflow_outputs = step.workflow_outputs pjadict = {} for pja in step.post_job_actions: pjadict[pja.action_type] = pja module.post_job_actions = pjadict return module return None
def chroms( self, trans, dbkey=None, num=None, chrom=None, low=None ): """ Returns a naturally sorted list of chroms/contigs for a given dbkey. Use either chrom or low to specify the starting chrom in the return list. """ # If there is no dbkey owner, default to current user. dbkey_owner, dbkey = decode_dbkey( dbkey ) if dbkey_owner: dbkey_user = trans.sa_session.query( trans.app.model.User ).filter_by( username=dbkey_owner ).first() else: dbkey_user = trans.user # # Get/create genome object. # genome = None twobit_file = None # Look first in user's custom builds. if dbkey_user and 'dbkeys' in dbkey_user.preferences: user_keys = loads( dbkey_user.preferences['dbkeys'] ) if dbkey in user_keys: dbkey_attributes = user_keys[ dbkey ] dbkey_name = dbkey_attributes[ 'name' ] # If there's a fasta for genome, convert to 2bit for later use. if 'fasta' in dbkey_attributes: build_fasta = trans.sa_session.query( trans.app.model.HistoryDatasetAssociation ).get( dbkey_attributes[ 'fasta' ] ) len_file = build_fasta.get_converted_dataset( trans, 'len' ).file_name build_fasta.get_converted_dataset( trans, 'twobit' ) # HACK: set twobit_file to True rather than a file name because # get_converted_dataset returns null during conversion even though # there will eventually be a twobit file available for genome. twobit_file = True # Backwards compatibility: look for len file directly. elif 'len' in dbkey_attributes: len_file = trans.sa_session.query( trans.app.model.HistoryDatasetAssociation ).get( user_keys[ dbkey ][ 'len' ] ).file_name if len_file: genome = Genome( dbkey, dbkey_name, len_file=len_file, twobit_file=twobit_file ) # Look in history and system builds. if not genome: # Look in history for chromosome len file. len_ds = trans.db_dataset_for( dbkey ) if len_ds: genome = Genome( dbkey, dbkey_name, len_file=len_ds.file_name ) # Look in system builds. elif dbkey in self.genomes: genome = self.genomes[ dbkey ] # Set up return value or log exception if genome not found for key. rval = None if genome: rval = genome.to_dict( num=num, chrom=chrom, low=low ) else: log.exception( 'genome not found for key %s' % dbkey ) return rval
def handle_result(self, result, param_dict, trans): rval = loads(result.content) return trans.fill_template( '/external_services/generic_jquery_grid.mako', result=rval, param_dict=param_dict, action=self.parent)
def __main__(): if len( sys.argv ) < 4: print >>sys.stderr, 'usage: upload.py <root> <datatypes_conf> <json paramfile> <output spec> ...' sys.exit( 1 ) output_paths = parse_outputs( sys.argv[4:] ) json_file = open( 'galaxy.json', 'w' ) registry = Registry() registry.load_datatypes( root_dir=sys.argv[1], config=sys.argv[2] ) for line in open( sys.argv[3], 'r' ): dataset = loads( line ) dataset = util.bunch.Bunch( **safe_dict( dataset ) ) try: output_path = output_paths[int( dataset.dataset_id )][0] except: print >>sys.stderr, 'Output path for dataset %s not found on command line' % dataset.dataset_id sys.exit( 1 ) if dataset.type == 'composite': files_path = output_paths[int( dataset.dataset_id )][1] add_composite_file( dataset, json_file, output_path, files_path ) else: add_file( dataset, registry, json_file, output_path ) # clean up paramfile # TODO: this will not work when running as the actual user unless the # parent directory is writable by the user. try: os.remove( sys.argv[3] ) except: pass
def check_job( self, job ): if self._missing_params( job.params, [ 'type' ] ): return self.job_states.INVALID if job.params[ 'type' ] == 'init_transfer': if self._missing_params( job.params, [ 'smrt_host', 'smrt_job_id' ] ): return self.job_states.INVALID url = 'http://' + job.params[ 'smrt_host' ] + self.api_path + '/Jobs/' + job.params[ 'smrt_job_id' ] + '/Status' r = urllib2.urlopen( url ) status = json.loads( r.read() ) # TODO: error handling: unexpected json or bad response, bad url, etc. if status[ 'Code' ] == 'Completed': log.debug( "SMRT Portal job '%s' is Completed. Initiating transfer." % job.params[ 'smrt_job_id' ] ) return self.job_states.READY return self.job_states.WAIT if job.params[ 'type' ] == 'finish_transfer': if self._missing_params( job.params, [ 'transfer_job_id' ] ): return self.job_states.INVALID # Get the TransferJob object and add it to the DeferredJob so we only look it up once. if not hasattr( job, 'transfer_job' ): job.transfer_job = self.sa_session.query( self.app.model.TransferJob ).get( int( job.params[ 'transfer_job_id' ] ) ) state = self.app.transfer_manager.get_state( job.transfer_job ) if not state: log.error( 'No state for transfer job id: %s' % job.transfer_job.id ) return self.job_states.WAIT if state[ 'state' ] in self.app.model.TransferJob.terminal_states: return self.job_states.READY log.debug( "Checked on finish transfer job %s, not done yet." % job.id ) return self.job_states.WAIT else: log.error( 'Unknown job type for SMRTPortalPlugin: %s' % str( job.params[ 'type' ] ) ) return self.job_states.INVALID
def upgrade(migrate_engine): metadata.bind = migrate_engine print __doc__ metadata.reflect() Visualization_table = Table( "visualization", metadata, autoload=True ) Visualization_revision_table = Table( "visualization_revision", metadata, autoload=True ) # Create dbkey columns. x = Column( "dbkey", TEXT ) y = Column( "dbkey", TEXT ) x.create( Visualization_table ) y.create( Visualization_revision_table ) # Manually create indexes for compatability w/ mysql_length. xi = Index( "ix_visualization_dbkey", Visualization_table.c.dbkey, mysql_length = 200) xi.create() yi = Index( "ix_visualization_revision_dbkey", Visualization_revision_table.c.dbkey, mysql_length = 200) yi.create() assert x is Visualization_table.c.dbkey assert y is Visualization_revision_table.c.dbkey all_viz = migrate_engine.execute( "SELECT visualization.id as viz_id, visualization_revision.id as viz_rev_id, visualization_revision.config FROM visualization_revision \ LEFT JOIN visualization ON visualization.id=visualization_revision.visualization_id" ) for viz in all_viz: viz_id = viz['viz_id'] viz_rev_id = viz['viz_rev_id'] if viz[Visualization_revision_table.c.config]: dbkey = loads(viz[Visualization_revision_table.c.config]).get('dbkey', "").replace("'", "\\'") migrate_engine.execute("UPDATE visualization_revision SET dbkey='%s' WHERE id=%s" % (dbkey, viz_rev_id)) migrate_engine.execute("UPDATE visualization SET dbkey='%s' WHERE id=%s" % (dbkey, viz_id))
def import_workflow( self, trans, payload, **kwd ): """ POST /api/tool_shed_repositories/import_workflow Import the specified exported workflow contained in the specified installed tool shed repository into Galaxy. :param key: the API key of the Galaxy user with which the imported workflow will be associated. :param id: the encoded id of the ToolShedRepository object The following parameters are included in the payload. :param index: the index location of the workflow tuple in the list of exported workflows stored in the metadata for the specified repository """ api_key = kwd.get( 'key', None ) if api_key is None: raise HTTPBadRequest( detail="Missing required parameter 'key' whose value is the API key for the Galaxy user importing the specified workflow." ) tool_shed_repository_id = kwd.get( 'id', '' ) if not tool_shed_repository_id: raise HTTPBadRequest( detail="Missing required parameter 'id'." ) index = payload.get( 'index', None ) if index is None: raise HTTPBadRequest( detail="Missing required parameter 'index'." ) repository = suc.get_tool_shed_repository_by_id( trans.app, tool_shed_repository_id ) exported_workflows = json.loads( self.exported_workflows( trans, tool_shed_repository_id ) ) # Since we don't have an in-memory object with an id, we'll identify the exported workflow via its location (i.e., index) in the list. exported_workflow = exported_workflows[ int( index ) ] workflow_name = exported_workflow[ 'workflow_name' ] workflow, status, error_message = workflow_util.import_workflow( trans, repository, workflow_name ) if status == 'error': log.debug( error_message ) return {} return workflow.to_dict( view='element' )
def sweepster( self, trans, id=None, hda_ldda=None, dataset_id=None, regions=None ): """ Displays a sweepster visualization using the incoming parameters. If id is available, get the visualization with the given id; otherwise, create a new visualization using a given dataset and regions. """ regions = regions or '{}' # Need to create history if necessary in order to create tool form. trans.get_history( create=True ) if id: # Loading a shared visualization. viz = self.get_visualization( trans, id ) viz_config = self.get_visualization_config( trans, viz ) decoded_id = self.decode_id( viz_config[ 'dataset_id' ] ) dataset = self.hda_manager.get_owned( trans, decoded_id, trans.user ) else: # Loading new visualization. dataset = self.get_hda_or_ldda( trans, hda_ldda, dataset_id ) job = self.hda_manager.creating_job( dataset ) viz_config = { 'dataset_id': dataset_id, 'tool_id': job.tool_id, 'regions': loads( regions ) } # Add tool, dataset attributes to config based on id. tool = trans.app.toolbox.get_tool( viz_config[ 'tool_id' ] ) viz_config[ 'tool' ] = tool.to_dict( trans, io_details=True ) viz_config[ 'dataset' ] = trans.security.encode_dict_ids( dataset.to_dict() ) return trans.fill_template_mako( "visualization/sweepster.mako", config=viz_config )
def import_workflows(self, trans, **kwd): """ POST /api/tool_shed_repositories/import_workflows Import all of the exported workflows contained in the specified installed tool shed repository into Galaxy. :param key: the API key of the Galaxy user with which the imported workflows will be associated. :param id: the encoded id of the ToolShedRepository object """ api_key = kwd.get('key', None) if api_key is None: raise HTTPBadRequest( detail= "Missing required parameter 'key' whose value is the API key for the Galaxy user importing the specified workflow." ) tool_shed_repository_id = kwd.get('id', '') if not tool_shed_repository_id: raise HTTPBadRequest(detail="Missing required parameter 'id'.") repository = suc.get_tool_shed_repository_by_id( trans.app, tool_shed_repository_id) exported_workflows = json.loads( self.exported_workflows(trans, tool_shed_repository_id)) imported_workflow_dicts = [] for exported_workflow_dict in exported_workflows: workflow_name = exported_workflow_dict['workflow_name'] workflow, status, error_message = workflow_util.import_workflow( trans, repository, workflow_name) if status == 'error': log.debug(error_message) else: imported_workflow_dicts.append( workflow.to_dict(view='element')) return imported_workflow_dicts
def _POST_to_saved( self, trans, id=None, revision=None, type=None, config=None, title=None, **kwargs ): """ Save the visualiztion info (revision, type, config, title, etc.) to the Visualization at `id` or to a new Visualization if `id` is None. Uses POST/redirect/GET after a successful save, redirecting to GET. """ DEFAULT_VISUALIZATION_NAME = 'Unnamed Visualization' # post to saved in order to save a visualization if type is None or config is None: return HTTPBadRequest( 'A visualization type and config are required to save a visualization' ) if isinstance( config, basestring ): config = loads( config ) title = title or DEFAULT_VISUALIZATION_NAME # TODO: allow saving to (updating) a specific revision - should be part of UsesVisualization # TODO: would be easier if this returned the visualization directly # check security if posting to existing visualization if id is not None: self.get_visualization( trans, id, check_ownership=True, check_accessible=False ) # ??: on not owner: error raised, but not returned (status = 200) # TODO: there's no security check in save visualization (if passed an id) returned = self.save_visualization( trans, config, type, id, title ) # redirect to GET to prevent annoying 'Do you want to post again?' dialog on page reload render_url = web.url_for( controller='visualization', action='saved', id=returned.get( 'vis_id' ) ) return trans.response.send_redirect( render_url )
def import_workflows( self, trans, **kwd ): """ POST /api/tool_shed_repositories/import_workflows Import all of the exported workflows contained in the specified installed tool shed repository into Galaxy. :param key: the API key of the Galaxy user with which the imported workflows will be associated. :param id: the encoded id of the ToolShedRepository object """ api_key = kwd.get( 'key', None ) if api_key is None: raise HTTPBadRequest( detail="Missing required parameter 'key' whose value is the API key for the Galaxy user importing the specified workflow." ) tool_shed_repository_id = kwd.get( 'id', '' ) if not tool_shed_repository_id: raise HTTPBadRequest( detail="Missing required parameter 'id'." ) repository = suc.get_tool_shed_repository_by_id( trans.app, tool_shed_repository_id ) exported_workflows = json.loads( self.exported_workflows( trans, tool_shed_repository_id ) ) imported_workflow_dicts = [] for exported_workflow_dict in exported_workflows: workflow_name = exported_workflow_dict[ 'workflow_name' ] workflow, status, error_message = workflow_util.import_workflow( trans, repository, workflow_name ) if status == 'error': log.debug( error_message ) else: imported_workflow_dicts.append( workflow.to_dict( view='element' ) ) return imported_workflow_dicts
def get_dbkeys(self, trans, chrom_info=False, **kwd): """ Returns all known dbkeys. If chrom_info is True, only dbkeys with chromosome lengths are returned. """ self.check_and_reload() dbkeys = [] # Add user's custom keys to dbkeys. user_keys_dict = {} user = trans.get_user() if user: if 'dbkeys' in user.preferences: user_keys_dict = loads(user.preferences['dbkeys']) dbkeys.extend([(attributes['name'], key) for key, attributes in user_keys_dict.items()]) # Add app keys to dbkeys. # If chrom_info is True, only include keys with len files (which contain chromosome info). filter_fn = lambda b: True if chrom_info: filter_fn = lambda b: b.len_file is not None dbkeys.extend([(genome.description, genome.key) for key, genome in self.genomes.items() if filter_fn(genome)]) return dbkeys
def __main__(): if len(sys.argv) < 4: print >> sys.stderr, 'usage: upload.py <root> <datatypes_conf> <json paramfile> <output spec> ...' sys.exit(1) output_paths = parse_outputs(sys.argv[4:]) json_file = open('galaxy.json', 'w') registry = Registry() registry.load_datatypes(root_dir=sys.argv[1], config=sys.argv[2]) for line in open(sys.argv[3], 'r'): dataset = loads(line) dataset = util.bunch.Bunch(**safe_dict(dataset)) try: output_path = output_paths[int(dataset.dataset_id)][0] except: print >> sys.stderr, 'Output path for dataset %s not found on command line' % dataset.dataset_id sys.exit(1) if dataset.type == 'composite': files_path = output_paths[int(dataset.dataset_id)][1] add_composite_file(dataset, registry, json_file, output_path, files_path) else: add_file(dataset, registry, json_file, output_path) # clean up paramfile # TODO: this will not work when running as the actual user unless the # parent directory is writable by the user. try: os.remove(sys.argv[3]) except: pass
def get_genome_build_names( self, trans=None ): # FIXME: how to deal with key duplicates? rval = [] # load user custom genome builds if trans is not None: if trans.history: # This is a little bit Odd. We are adding every .len file in the current history to dbkey list, # but this is previous behavior from trans.db_names, so we'll continue to do it. # It does allow one-off, history specific dbkeys to be created by a user. But we are not filtering, # so a len file will be listed twice (as the build name and again as dataset name), # if custom dbkey creation/conversion occurred within the current history. datasets = trans.sa_session.query( self._app.model.HistoryDatasetAssociation ) \ .filter_by( deleted=False, history_id=trans.history.id, extension="len" ) for dataset in datasets: rval.append( ( dataset.dbkey, "%s (%s) [History]" % ( dataset.name, dataset.dbkey ) ) ) user = trans.user if user and hasattr( user, 'preferences' ) and 'dbkeys' in user.preferences: user_keys = loads( user.preferences['dbkeys'] ) for key, chrom_dict in user_keys.iteritems(): rval.append( ( key, "%s (%s) [Custom]" % ( chrom_dict['name'], key ) ) ) # Load old builds.txt static keys rval.extend( self._static_dbkeys ) # load dbkeys from dbkey data table dbkey_table = self._app.tool_data_tables.get( self._data_table_name, None ) if dbkey_table is not None: for field_dict in dbkey_table.get_named_fields_list(): rval.append( ( field_dict[ 'value' ], field_dict[ 'name' ] ) ) return rval
def import_workflow(trans, repository, workflow_name): """Import a workflow contained in an installed tool shed repository into Galaxy (this method is called only from Galaxy).""" status = 'done' message = '' changeset_revision = repository.changeset_revision metadata = repository.metadata workflows = metadata.get('workflows', []) tools_metadata = metadata.get('tools', []) workflow_dict = None for workflow_data_tuple in workflows: # The value of workflow_data_tuple is ( relative_path_to_workflow_file, exported_workflow_dict ). relative_path_to_workflow_file, exported_workflow_dict = workflow_data_tuple if exported_workflow_dict['name'] == workflow_name: # If the exported workflow is available on disk, import it. if os.path.exists(relative_path_to_workflow_file): workflow_file = open(relative_path_to_workflow_file, 'rb') workflow_data = workflow_file.read() workflow_file.close() workflow_dict = json.loads(workflow_data) else: # Use the current exported_workflow_dict. workflow_dict = exported_workflow_dict break if workflow_dict: # Create workflow if possible. workflow, missing_tool_tups = get_workflow_from_dict( trans=trans, workflow_dict=workflow_dict, tools_metadata=tools_metadata, repository_id=repository.id, changeset_revision=changeset_revision) # Save the workflow in the Galaxy database. Pass workflow_dict along to create annotation at this point. stored_workflow = save_workflow(trans, workflow, workflow_dict) # Use the latest version of the saved workflow. workflow = stored_workflow.latest_workflow if workflow_name: workflow.name = workflow_name # Provide user feedback and show workflow list. if workflow.has_errors: message += "Imported, but some steps in this workflow have validation errors. " status = "error" if workflow.has_cycles: message += "Imported, but this workflow contains cycles. " status = "error" else: message += "Workflow <b>%s</b> imported successfully. " % workflow.name if missing_tool_tups: name_and_id_str = '' for missing_tool_tup in missing_tool_tups: tool_id, tool_name, other = missing_tool_tup name_and_id_str += 'name: %s, id: %s' % (str(tool_id), str(tool_name)) message += "The following tools required by this workflow are missing from this Galaxy instance: %s. " % name_and_id_str else: workflow = None message += 'The workflow named %s is not included in the metadata for revision %s of repository %s' % \ ( str( workflow_name ), str( changeset_revision ), str( repository.name ) ) status = 'error' return workflow, status, message
def import_workflow( trans, repository, workflow_name ): """Import a workflow contained in an installed tool shed repository into Galaxy (this method is called only from Galaxy).""" status = 'done' message = '' changeset_revision = repository.changeset_revision metadata = repository.metadata workflows = metadata.get( 'workflows', [] ) tools_metadata = metadata.get( 'tools', [] ) workflow_dict = None for workflow_data_tuple in workflows: # The value of workflow_data_tuple is ( relative_path_to_workflow_file, exported_workflow_dict ). relative_path_to_workflow_file, exported_workflow_dict = workflow_data_tuple if exported_workflow_dict[ 'name' ] == workflow_name: # If the exported workflow is available on disk, import it. if os.path.exists( relative_path_to_workflow_file ): workflow_file = open( relative_path_to_workflow_file, 'rb' ) workflow_data = workflow_file.read() workflow_file.close() workflow_dict = json.loads( workflow_data ) else: # Use the current exported_workflow_dict. workflow_dict = exported_workflow_dict break if workflow_dict: # Create workflow if possible. workflow, missing_tool_tups = get_workflow_from_dict( trans=trans, workflow_dict=workflow_dict, tools_metadata=tools_metadata, repository_id=repository.id, changeset_revision=changeset_revision ) # Save the workflow in the Galaxy database. Pass workflow_dict along to create annotation at this point. stored_workflow = save_workflow( trans, workflow, workflow_dict ) # Use the latest version of the saved workflow. workflow = stored_workflow.latest_workflow if workflow_name: workflow.name = workflow_name # Provide user feedback and show workflow list. if workflow.has_errors: message += "Imported, but some steps in this workflow have validation errors. " status = "error" if workflow.has_cycles: message += "Imported, but this workflow contains cycles. " status = "error" else: message += "Workflow <b>%s</b> imported successfully. " % workflow.name if missing_tool_tups: name_and_id_str = '' for missing_tool_tup in missing_tool_tups: tool_id, tool_name, other = missing_tool_tup name_and_id_str += 'name: %s, id: %s' % ( str( tool_id ), str( tool_name ) ) message += "The following tools required by this workflow are missing from this Galaxy instance: %s. " % name_and_id_str else: workflow = None message += 'The workflow named %s is not included in the metadata for revision %s of repository %s' % \ ( str( workflow_name ), str( changeset_revision ), str( repository.name ) ) status = 'error' return workflow, status, message
def save(self, trans, id, content, annotations): id = trans.security.decode_id(id) page = trans.sa_session.query(model.Page).get(id) assert page.user == trans.user # Sanitize content content = sanitize_html(content, 'utf-8', 'text/html') # Add a new revision to the page with the provided content. page_revision = model.PageRevision() page_revision.title = page.title page_revision.page = page page.latest_revision = page_revision page_revision.content = content # Save annotations. annotations = loads(annotations) for annotation_dict in annotations: item_id = trans.security.decode_id(annotation_dict['item_id']) item_class = self.get_class(annotation_dict['item_class']) item = trans.sa_session.query(item_class).filter_by( id=item_id).first() if not item: raise RuntimeError("cannot find annotated item") text = sanitize_html(annotation_dict['text'], 'utf-8', 'text/html') # Add/update annotation. if item_id and item_class and text: # Get annotation association. annotation_assoc_class = eval("model.%sAnnotationAssociation" % item_class.__name__) annotation_assoc = trans.sa_session.query( annotation_assoc_class).filter_by(user=trans.get_user()) if item_class == model.History.__class__: annotation_assoc = annotation_assoc.filter_by(history=item) elif item_class == model.HistoryDatasetAssociation.__class__: annotation_assoc = annotation_assoc.filter_by(hda=item) elif item_class == model.StoredWorkflow.__class__: annotation_assoc = annotation_assoc.filter_by( stored_workflow=item) elif item_class == model.WorkflowStep.__class__: annotation_assoc = annotation_assoc.filter_by( workflow_step=item) annotation_assoc = annotation_assoc.first() if not annotation_assoc: # Create association. annotation_assoc = annotation_assoc_class() item.annotations.append(annotation_assoc) annotation_assoc.user = trans.get_user() # Set annotation user text. annotation_assoc.annotation = text trans.sa_session.flush()
def save( self, trans, vis_json=None, type=None, id=None, title=None, dbkey=None, annotation=None ): """ Save a visualization; if visualization does not have an ID, a new visualization is created. Returns JSON of visualization. """ # Get visualization attributes from kwargs or from config. vis_config = loads( vis_json ) vis_type = type or vis_config[ 'type' ] vis_id = id or vis_config.get( 'id', None ) vis_title = title or vis_config.get( 'title', None ) vis_dbkey = dbkey or vis_config.get( 'dbkey', None ) vis_annotation = annotation or vis_config.get( 'annotation', None ) return self.save_visualization( trans, vis_config, vis_type, vis_id, vis_title, vis_dbkey, vis_annotation )
def create_archive( history_attrs_file, datasets_attrs_file, jobs_attrs_file, out_file, gzip=False ): """ Create archive from the given attribute/metadata files and save it to out_file. """ tarfile_mode = "w" if gzip: tarfile_mode += ":gz" try: history_archive = tarfile.open( out_file, tarfile_mode ) # Read datasets attributes from file. datasets_attr_in = open( datasets_attrs_file, 'rb' ) datasets_attr_str = '' buffsize = 1048576 try: while True: datasets_attr_str += datasets_attr_in.read( buffsize ) if not datasets_attr_str or len( datasets_attr_str ) % buffsize != 0: break except OverflowError: pass datasets_attr_in.close() datasets_attrs = loads( datasets_attr_str ) # Add datasets to archive and update dataset attributes. # TODO: security check to ensure that files added are in Galaxy dataset directory? for dataset_attrs in datasets_attrs: if dataset_attrs['exported']: dataset_file_name = dataset_attrs[ 'file_name' ] # Full file name. dataset_archive_name = os.path.join( 'datasets', get_dataset_filename( dataset_attrs[ 'name' ], dataset_attrs[ 'extension' ] ) ) history_archive.add( dataset_file_name, arcname=dataset_archive_name ) # Update dataset filename to be archive name. dataset_attrs[ 'file_name' ] = dataset_archive_name # Rewrite dataset attributes file. datasets_attrs_out = open( datasets_attrs_file, 'w' ) datasets_attrs_out.write( dumps( datasets_attrs ) ) datasets_attrs_out.close() # Finish archive. history_archive.add( history_attrs_file, arcname="history_attrs.txt" ) history_archive.add( datasets_attrs_file, arcname="datasets_attrs.txt" ) if os.path.exists( datasets_attrs_file + ".provenance" ): history_archive.add( datasets_attrs_file + ".provenance", arcname="datasets_attrs.txt.provenance" ) history_archive.add( jobs_attrs_file, arcname="jobs_attrs.txt" ) history_archive.close() # Status. return 'Created history archive.' except Exception, e: return 'Error creating history archive: %s' % str( e ), sys.stderr
def params_from_strings(params, param_values, app, ignore_errors=False): """ Convert a dictionary of strings as produced by `params_to_strings` back into parameter values (decode the json representation and then allow each parameter to convert the basic types into the parameters preferred form). """ rval = dict() for key, value in param_values.iteritems(): value = json_fix(loads(value)) if key in params: value = params[key].value_from_basic(value, app, ignore_errors) rval[key] = value return rval
def reference(self, trans, dbkey, chrom, low, high): """ Return reference data for a build. """ self.check_and_reload() # If there is no dbkey owner, default to current user. dbkey_owner, dbkey = decode_dbkey(dbkey) if dbkey_owner: dbkey_user = trans.sa_session.query( trans.app.model.User).filter_by(username=dbkey_owner).first() else: dbkey_user = trans.user if not self.has_reference_data(dbkey, dbkey_user): return None # # Get twobit file with reference data. # twobit_file_name = None if dbkey in self.genomes: # Built-in twobit. twobit_file_name = self.genomes[dbkey].twobit_file else: user_keys = loads(dbkey_user.preferences['dbkeys']) dbkey_attributes = user_keys[dbkey] fasta_dataset = trans.sa_session.query( trans.app.model.HistoryDatasetAssociation).get( dbkey_attributes['fasta']) msg = fasta_dataset.convert_dataset(trans, 'twobit') if msg: return msg else: twobit_dataset = fasta_dataset.get_converted_dataset( trans, 'twobit') twobit_file_name = twobit_dataset.file_name # Read and return reference data. try: twobit = TwoBitFile(open(twobit_file_name)) if chrom in twobit: seq_data = twobit[chrom].get(int(low), int(high)) return GenomeRegion(chrom=chrom, start=low, end=high, sequence=seq_data) except IOError: return None
def get_readme_files_dict_for_display( app, tool_shed_url, repo_info_dict ): """ Return a dictionary of README files contained in the single repository being installed so they can be displayed on the tool panel section selection page. """ name = repo_info_dict.keys()[ 0 ] repo_info_tuple = repo_info_dict[ name ] description, repository_clone_url, changeset_revision, ctx_rev, repository_owner, repository_dependencies, installed_td = \ suc.get_repo_info_tuple_contents( repo_info_tuple ) # Handle changing HTTP protocols over time. tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( app, tool_shed_url ) params = dict( name=name, owner=repository_owner, changeset_revision=changeset_revision ) pathspec = [ 'repository', 'get_readme_files' ] raw_text = common_util.tool_shed_get( app, tool_shed_url, pathspec=pathspec, params=params ) readme_files_dict = json.loads( raw_text ) return readme_files_dict
def save( self, trans, id, content, annotations ): id = self.decode_id( id ) page = trans.sa_session.query( model.Page ).get( id ) assert page.user == trans.user # Sanitize content content = sanitize_html( content, 'utf-8', 'text/html' ) # Add a new revision to the page with the provided content. page_revision = model.PageRevision() page_revision.title = page.title page_revision.page = page page.latest_revision = page_revision page_revision.content = content # Save annotations. annotations = loads( annotations ) for annotation_dict in annotations: item_id = self.decode_id( annotation_dict[ 'item_id' ] ) item_class = self.get_class( annotation_dict[ 'item_class' ] ) item = trans.sa_session.query( item_class ).filter_by( id=item_id ).first() if not item: raise RuntimeError( "cannot find annotated item" ) text = sanitize_html( annotation_dict[ 'text' ], 'utf-8', 'text/html' ) # Add/update annotation. if item_id and item_class and text: # Get annotation association. annotation_assoc_class = eval( "model.%sAnnotationAssociation" % item_class.__name__ ) annotation_assoc = trans.sa_session.query( annotation_assoc_class ).filter_by( user=trans.get_user() ) if item_class == model.History.__class__: annotation_assoc = annotation_assoc.filter_by( history=item ) elif item_class == model.HistoryDatasetAssociation.__class__: annotation_assoc = annotation_assoc.filter_by( hda=item ) elif item_class == model.StoredWorkflow.__class__: annotation_assoc = annotation_assoc.filter_by( stored_workflow=item ) elif item_class == model.WorkflowStep.__class__: annotation_assoc = annotation_assoc.filter_by( workflow_step=item ) annotation_assoc = annotation_assoc.first() if not annotation_assoc: # Create association. annotation_assoc = annotation_assoc_class() item.annotations.append( annotation_assoc ) annotation_assoc.user = trans.get_user() # Set annotation user text. annotation_assoc.annotation = text trans.sa_session.flush()
def load_input_parameters( filename, erase_file = True ): datasource_params = {} try: json_params = loads( open( filename, 'r' ).read() ) datasource_params = json_params.get( 'param_dict' ) except: json_params = None for line in open( filename, 'r' ): try: line = line.strip() fields = line.split( '\t' ) datasource_params[ fields[0] ] = fields[1] except: continue if erase_file: open( filename, 'w' ).close() #open file for writing, then close, removes params from file return json_params, datasource_params
def loads_recurse(item): decoded_list = [] if isinstance( item, basestring): try: # Not clear what we're decoding, so recurse to ensure that we catch everything. decoded_item = loads( item ) if isinstance( decoded_item, list): decoded_list = loads_recurse( decoded_item ) else: decoded_list = [ unicode( decoded_item ) ] except ValueError: decoded_list = [ unicode( item ) ] elif isinstance( item, list): for element in item: a_list = loads_recurse( element ) decoded_list = decoded_list + a_list return decoded_list
def upgrade(migrate_engine): metadata.bind = migrate_engine print __doc__ metadata.reflect() Visualization_table = Table("visualization", metadata, autoload=True) Visualization_revision_table = Table("visualization_revision", metadata, autoload=True) # Create dbkey columns. x = Column("dbkey", TEXT) y = Column("dbkey", TEXT) x.create(Visualization_table) y.create(Visualization_revision_table) # Manually create indexes for compatability w/ mysql_length. xi = Index("ix_visualization_dbkey", Visualization_table.c.dbkey, mysql_length=200) xi.create() yi = Index("ix_visualization_revision_dbkey", Visualization_revision_table.c.dbkey, mysql_length=200) yi.create() assert x is Visualization_table.c.dbkey assert y is Visualization_revision_table.c.dbkey all_viz = migrate_engine.execute( "SELECT visualization.id as viz_id, visualization_revision.id as viz_rev_id, visualization_revision.config FROM visualization_revision \ LEFT JOIN visualization ON visualization.id=visualization_revision.visualization_id" ) for viz in all_viz: viz_id = viz['viz_id'] viz_rev_id = viz['viz_rev_id'] if viz[Visualization_revision_table.c.config]: dbkey = loads(viz[Visualization_revision_table.c.config]).get( 'dbkey', "").replace("'", "\\'") migrate_engine.execute( "UPDATE visualization_revision SET dbkey='%s' WHERE id=%s" % (dbkey, viz_rev_id)) migrate_engine.execute( "UPDATE visualization SET dbkey='%s' WHERE id=%s" % (dbkey, viz_id))
def get_readme_files_dict_for_display(app, tool_shed_url, repo_info_dict): """ Return a dictionary of README files contained in the single repository being installed so they can be displayed on the tool panel section selection page. """ name = repo_info_dict.keys()[0] repo_info_tuple = repo_info_dict[name] description, repository_clone_url, changeset_revision, ctx_rev, repository_owner, repository_dependencies, installed_td = \ suc.get_repo_info_tuple_contents( repo_info_tuple ) # Handle changing HTTP protocols over time. tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( app, tool_shed_url) params = '?name=%s&owner=%s&changeset_revision=%s' % ( name, repository_owner, changeset_revision) url = common_util.url_join(tool_shed_url, 'repository/get_readme_files%s' % params) raw_text = common_util.tool_shed_get(app, tool_shed_url, url) readme_files_dict = json.loads(raw_text) return readme_files_dict
def has_reference_data( self, dbkey, dbkey_owner=None ): """ Returns true if there is reference data for the specified dbkey. If dbkey is custom, dbkey_owner is needed to determine if there is reference data. """ # Look for key in built-in builds. if dbkey in self.genomes and self.genomes[ dbkey ].twobit_file: # There is built-in reference data. return True # Look for key in owner's custom builds. if dbkey_owner and 'dbkeys' in dbkey_owner.preferences: user_keys = loads( dbkey_owner.preferences[ 'dbkeys' ] ) if dbkey in user_keys: dbkey_attributes = user_keys[ dbkey ] if 'fasta' in dbkey_attributes: # Fasta + converted datasets can provide reference data. return True return False
def saved( self, trans, id=None, revision=None, type=None, config=None, title=None, **kwargs ): """ """ DEFAULT_VISUALIZATION_NAME = 'Unnamed Visualization' # post to saved in order to save a visualization #TODO: re-route this one to clear up signature if trans.request.method == 'POST': if type is None or config is None: return HTTPBadRequest( 'A visualization type and config are required to save a visualization' ) if isinstance( config, basestring ): config = loads( config ) title = title or DEFAULT_VISUALIZATION_NAME #TODO: allow saving to (updating) a specific revision - should be part of UsesVisualization #TODO: would be easier if this returned the visualization directly # check security if posting to existing visualization if id is not None: visualization = self.get_visualization( trans, id, check_ownership=True, check_accessible=False ) #??: on not owner: error raised, but not returned (status = 200) #TODO: there's no security check in save visualization (if passed an id) returned = self.save_visualization( trans, config, type, id, title ) # redirect to GET to prevent annoying 'Do you want to post again?' dialog on page reload render_url = web.url_for( controller='visualization', action='saved', id=returned.get( 'vis_id' ) ) return trans.response.send_redirect( render_url ) if id is None: return HTTPBadRequest( 'A valid visualization id is required to load a visualization' ) # render the saved visualization by passing to render, sending latest revision config #TODO: allow loading a specific revision - should be part of UsesVisualization #visualization = self.get_visualization( trans, id, check_ownership=True, check_accessible=False ) visualization = self.get_visualization( trans, id, check_ownership=False, check_accessible=True ) config = copy.copy( visualization.latest_revision.config ) # re-add title to kwargs for passing to render if title: kwargs[ 'title' ] = title return self.render( trans, visualization.type, visualization, config=config, **kwargs )
def reference( self, trans, dbkey, chrom, low, high ): """ Return reference data for a build. """ # If there is no dbkey owner, default to current user. dbkey_owner, dbkey = decode_dbkey( dbkey ) if dbkey_owner: dbkey_user = trans.sa_session.query( trans.app.model.User ).filter_by( username=dbkey_owner ).first() else: dbkey_user = trans.user if not self.has_reference_data( dbkey, dbkey_user ): return None # # Get twobit file with reference data. # twobit_file_name = None if dbkey in self.genomes: # Built-in twobit. twobit_file_name = self.genomes[dbkey].twobit_file else: user_keys = loads( dbkey_user.preferences['dbkeys'] ) dbkey_attributes = user_keys[ dbkey ] fasta_dataset = trans.sa_session.query( trans.app.model.HistoryDatasetAssociation ).get( dbkey_attributes[ 'fasta' ] ) msg = fasta_dataset.convert_dataset( trans, 'twobit' ) if msg: return msg else: twobit_dataset = fasta_dataset.get_converted_dataset( trans, 'twobit' ) twobit_file_name = twobit_dataset.file_name # Read and return reference data. try: twobit = TwoBitFile( open( twobit_file_name ) ) if chrom in twobit: seq_data = twobit[chrom].get( int(low), int(high) ) return GenomeRegion( chrom=chrom, start=low, end=high, sequence=seq_data ) except IOError: return None
def get_sequencer_id(sequencer_info): '''Get the sequencer id corresponding to the sequencer information''' # Check if there is any existing sequencer which have the same sequencer # information fields & values cmd = "SELECT sequencer.id, form_values.content FROM sequencer, form_values WHERE sequencer.form_values_id=form_values.id" result = migrate_engine.execute(cmd) for row in result: sequencer_id = row[0] values = str(row[1]) if not values.strip(): continue values = loads(values) # proceed only if sequencer_info is a valid list if values and type(values) == type(dict()): if sequencer_info.get( 'host', '' ) == values.get( 'field_0', '' ) \ and sequencer_info.get( 'username', '' ) == values.get( 'field_1', '' ) \ and sequencer_info.get( 'password', '' ) == values.get( 'field_2', '' ) \ and sequencer_info.get( 'data_dir', '' ) == values.get( 'field_3', '' ) \ and sequencer_info.get( 'rename_dataset', '' ) == values.get( 'field_4', '' ): return sequencer_id return None
def get_sequencer_id( sequencer_info ): '''Get the sequencer id corresponding to the sequencer information''' # Check if there is any existing sequencer which have the same sequencer # information fields & values cmd = "SELECT sequencer.id, form_values.content FROM sequencer, form_values WHERE sequencer.form_values_id=form_values.id" result = migrate_engine.execute( cmd ) for row in result: sequencer_id = row[0] values = str( row[1] ) if not values.strip(): continue values = loads( values ) # proceed only if sequencer_info is a valid list if values and type( values ) == type( dict() ): if sequencer_info.get( 'host', '' ) == values.get( 'field_0', '' ) \ and sequencer_info.get( 'username', '' ) == values.get( 'field_1', '' ) \ and sequencer_info.get( 'password', '' ) == values.get( 'field_2', '' ) \ and sequencer_info.get( 'data_dir', '' ) == values.get( 'field_3', '' ) \ and sequencer_info.get( 'rename_dataset', '' ) == values.get( 'field_4', '' ): return sequencer_id return None
def check_job(self, job): if self._missing_params(job.params, ['type']): return self.job_states.INVALID if job.params['type'] == 'init_transfer': if self._missing_params(job.params, ['smrt_host', 'smrt_job_id']): return self.job_states.INVALID url = 'http://' + job.params[ 'smrt_host'] + self.api_path + '/Jobs/' + job.params[ 'smrt_job_id'] + '/Status' r = urllib2.urlopen(url) status = json.loads(r.read()) # TODO: error handling: unexpected json or bad response, bad url, etc. if status['Code'] == 'Completed': log.debug( "SMRT Portal job '%s' is Completed. Initiating transfer." % job.params['smrt_job_id']) return self.job_states.READY return self.job_states.WAIT if job.params['type'] == 'finish_transfer': if self._missing_params(job.params, ['transfer_job_id']): return self.job_states.INVALID # Get the TransferJob object and add it to the DeferredJob so we only look it up once. if not hasattr(job, 'transfer_job'): job.transfer_job = self.sa_session.query( self.app.model.TransferJob).get( int(job.params['transfer_job_id'])) state = self.app.transfer_manager.get_state(job.transfer_job) if not state: log.error('No state for transfer job id: %s' % job.transfer_job.id) return self.job_states.WAIT if state['state'] in self.app.model.TransferJob.terminal_states: return self.job_states.READY log.debug("Checked on finish transfer job %s, not done yet." % job.id) return self.job_states.WAIT else: log.error('Unknown job type for SMRTPortalPlugin: %s' % str(job.params['type'])) return self.job_states.INVALID
def get_chrom_info( self, dbkey, trans=None, custom_build_hack_get_len_from_fasta_conversion=True ): # FIXME: flag to turn off custom_build_hack_get_len_from_fasta_conversion should not be required chrom_info = None db_dataset = None # Collect chromInfo from custom builds if trans: db_dataset = trans.db_dataset_for( dbkey ) if db_dataset: chrom_info = db_dataset.file_name else: # Do Custom Build handling if trans.user and ( 'dbkeys' in trans.user.preferences ) and ( dbkey in loads( trans.user.preferences[ 'dbkeys' ] ) ): custom_build_dict = loads( trans.user.preferences[ 'dbkeys' ] )[ dbkey ] # HACK: the attempt to get chrom_info below will trigger the # fasta-to-len converter if the dataset is not available or, # which will in turn create a recursive loop when # running the fasta-to-len tool. So, use a hack in the second # condition below to avoid getting chrom_info when running the # fasta-to-len converter. if 'fasta' in custom_build_dict and custom_build_hack_get_len_from_fasta_conversion: # Build is defined by fasta; get len file, which is obtained from converting fasta. build_fasta_dataset = trans.sa_session.query( trans.app.model.HistoryDatasetAssociation ).get( custom_build_dict[ 'fasta' ] ) chrom_info = build_fasta_dataset.get_converted_dataset( trans, 'len' ).file_name elif 'len' in custom_build_dict: # Build is defined by len file, so use it. chrom_info = trans.sa_session.query( trans.app.model.HistoryDatasetAssociation ).get( custom_build_dict[ 'len' ] ).file_name # Check Data table if not chrom_info: dbkey_table = self._app.tool_data_tables.get( self._data_table_name, None ) if dbkey_table is not None: chrom_info = dbkey_table.get_entry( 'value', dbkey, 'len_path', default=None ) # use configured server len path if not chrom_info: # Default to built-in build. # Since we are using an unverified dbkey, we will sanitize the dbkey before use chrom_info = os.path.join( self._static_chrom_info_path, "%s.len" % sanitize_lists_to_string( dbkey ) ) chrom_info = os.path.abspath( chrom_info ) return ( chrom_info, db_dataset )
def get_dbkeys( self, trans, chrom_info=False, **kwd ): """ Returns all known dbkeys. If chrom_info is True, only dbkeys with chromosome lengths are returned. """ dbkeys = [] # Add user's custom keys to dbkeys. user_keys_dict = {} user = trans.get_user() if user: if 'dbkeys' in user.preferences: user_keys_dict = loads( user.preferences[ 'dbkeys' ] ) dbkeys.extend( [ (attributes[ 'name' ], key ) for key, attributes in user_keys_dict.items() ] ) # Add app keys to dbkeys. # If chrom_info is True, only include keys with len files (which contain chromosome info). filter_fn = lambda b: True if chrom_info: filter_fn = lambda b: b.len_file is not None dbkeys.extend( [ ( genome.description, genome.key ) for key, genome in self.genomes.items() if filter_fn( genome ) ] ) return dbkeys
def __call__( self, trans, **kwargs ): # Get basics. # FIXME: pretty sure this is only here to pass along, can likely be eliminated status = kwargs.get( 'status', None ) message = kwargs.get( 'message', None ) # Build a base filter and sort key that is the combination of the saved state and defaults. # Saved state takes preference over defaults. base_filter = {} if self.default_filter: # default_filter is a dictionary that provides a default set of filters based on the grid's columns. base_filter = self.default_filter.copy() base_sort_key = self.default_sort_key if self.preserve_state: pref_name = unicode( self.__class__.__name__ + self.cur_filter_pref_name ) if pref_name in trans.get_user().preferences: saved_filter = loads( trans.get_user().preferences[pref_name] ) base_filter.update( saved_filter ) pref_name = unicode( self.__class__.__name__ + self.cur_sort_key_pref_name ) if pref_name in trans.get_user().preferences: base_sort_key = loads( trans.get_user().preferences[pref_name] ) # Build initial query query = self.build_initial_query( trans, **kwargs ) query = self.apply_query_filter( trans, query, **kwargs ) # Maintain sort state in generated urls extra_url_args = {} # Determine whether use_default_filter flag is set. use_default_filter_str = kwargs.get( 'use_default_filter' ) use_default_filter = False if use_default_filter_str: use_default_filter = ( use_default_filter_str.lower() == 'true' ) # Process filtering arguments to (a) build a query that represents the filter and (b) build a # dictionary that denotes the current filter. cur_filter_dict = {} for column in self.columns: if column.key: # Get the filter criterion for the column. Precedence is (a) if using default filter, only look there; otherwise, # (b) look in kwargs; and (c) look in base filter. column_filter = None if use_default_filter: if self.default_filter: column_filter = self.default_filter.get( column.key ) elif "f-" + column.model_class.__name__ + ".%s" % column.key in kwargs: # Queries that include table joins cannot guarantee unique column names. This problem is # handled by setting the column_filter value to <TableName>.<ColumnName>. column_filter = kwargs.get( "f-" + column.model_class.__name__ + ".%s" % column.key ) elif "f-" + column.key in kwargs: column_filter = kwargs.get( "f-" + column.key ) elif column.key in base_filter: column_filter = base_filter.get( column.key ) # Method (1) combines a mix of strings and lists of strings into a single string and (2) attempts to de-jsonify all strings. def loads_recurse(item): decoded_list = [] if isinstance( item, basestring): try: # Not clear what we're decoding, so recurse to ensure that we catch everything. decoded_item = loads( item ) if isinstance( decoded_item, list): decoded_list = loads_recurse( decoded_item ) else: decoded_list = [ unicode( decoded_item ) ] except ValueError: decoded_list = [ unicode( item ) ] elif isinstance( item, list): for element in item: a_list = loads_recurse( element ) decoded_list = decoded_list + a_list return decoded_list # If column filter found, apply it. if column_filter is not None: # TextColumns may have a mix of json and strings. if isinstance( column, TextColumn ): column_filter = loads_recurse( column_filter ) if len( column_filter ) == 1: column_filter = column_filter[0] # Interpret ',' as a separator for multiple terms. if isinstance( column_filter, basestring ) and column_filter.find(',') != -1: column_filter = column_filter.split(',') # Check if filter is empty if isinstance( column_filter, list ): # Remove empty strings from filter list column_filter = [x for x in column_filter if x != ''] if len(column_filter) == 0: continue elif isinstance(column_filter, basestring): # If filter criterion is empty, do nothing. if column_filter == '': continue # Update query. query = column.filter( trans, trans.user, query, column_filter ) # Upate current filter dict. # Column filters are rendered in various places, sanitize them all here. cur_filter_dict[ column.key ] = sanitize_text(column_filter) # Carry filter along to newly generated urls; make sure filter is a string so # that we can encode to UTF-8 and thus handle user input to filters. if isinstance( column_filter, list ): # Filter is a list; process each item. for filter in column_filter: if not isinstance( filter, basestring ): filter = unicode( filter ).encode("utf-8") extra_url_args[ "f-" + column.key ] = dumps( column_filter ) else: # Process singleton filter. if not isinstance( column_filter, basestring ): column_filter = unicode(column_filter) extra_url_args[ "f-" + column.key ] = column_filter.encode("utf-8") # Process sort arguments. sort_key = None if 'sort' in kwargs: sort_key = kwargs['sort'] elif base_sort_key: sort_key = base_sort_key if sort_key: ascending = not( sort_key.startswith( "-" ) ) # Queries that include table joins cannot guarantee unique column names. This problem is # handled by setting the column_filter value to <TableName>.<ColumnName>. table_name = None if sort_key.find( '.' ) > -1: a_list = sort_key.split( '.' ) if ascending: table_name = a_list[0] else: table_name = a_list[0][1:] column_name = a_list[1] elif ascending: column_name = sort_key else: column_name = sort_key[1:] # Sort key is a column key. for column in self.columns: if column.key and column.key.find( '.' ) > -1: column_key = column.key.split( '.' )[1] else: column_key = column.key if ( table_name is None or table_name == column.model_class.__name__ ) and column_key == column_name: query = column.sort( trans, query, ascending, column_name=column_name ) break extra_url_args['sort'] = sort_key # There might be a current row current_item = self.get_current_item( trans, **kwargs ) # Process page number. if self.use_paging: if 'page' in kwargs: if kwargs['page'] == 'all': page_num = 0 else: page_num = int( kwargs['page'] ) else: page_num = 1 if page_num == 0: # Show all rows in page. total_num_rows = query.count() page_num = 1 num_pages = 1 else: # Show a limited number of rows. Before modifying query, get the total number of rows that query # returns so that the total number of pages can be computed. total_num_rows = query.count() query = query.limit( self.num_rows_per_page ).offset( ( page_num - 1 ) * self.num_rows_per_page ) num_pages = int( math.ceil( float( total_num_rows ) / self.num_rows_per_page ) ) else: # Defaults. page_num = 1 num_pages = 1 # There are some places in grid templates where it's useful for a grid # to have its current filter. self.cur_filter_dict = cur_filter_dict # Preserve grid state: save current filter and sort key. if self.preserve_state: pref_name = unicode( self.__class__.__name__ + self.cur_filter_pref_name ) trans.get_user().preferences[pref_name] = unicode( dumps( cur_filter_dict ) ) if sort_key: pref_name = unicode( self.__class__.__name__ + self.cur_sort_key_pref_name ) trans.get_user().preferences[pref_name] = unicode( dumps( sort_key ) ) trans.sa_session.flush() # Log grid view. context = unicode( self.__class__.__name__ ) params = cur_filter_dict.copy() params['sort'] = sort_key params['async'] = ( 'async' in kwargs ) # TODO:?? # commenting this out; when this fn calls session.add( action ) and session.flush the query from this fn # is effectively 'wiped' out. Nate believes it has something to do with our use of session( autocommit=True ) # in mapping.py. If you change that to False, the log_action doesn't affect the query # Below, I'm rendering the template first (that uses query), then calling log_action, then returning the page # trans.log_action( trans.get_user(), unicode( "grid.view" ), context, params ) # Render grid. def url( *args, **kwargs ): # Only include sort/filter arguments if not linking to another # page. This is a bit of a hack. if 'action' in kwargs: new_kwargs = dict() else: new_kwargs = dict( extra_url_args ) # Extend new_kwargs with first argument if found if len(args) > 0: new_kwargs.update( args[0] ) new_kwargs.update( kwargs ) # We need to encode item ids if 'id' in new_kwargs: id = new_kwargs[ 'id' ] if isinstance( id, list ): new_kwargs[ 'id' ] = [ trans.security.encode_id( i ) for i in id ] else: new_kwargs[ 'id' ] = trans.security.encode_id( id ) # The url_for invocation *must* include a controller and action. if 'controller' not in new_kwargs: new_kwargs['controller'] = trans.controller if 'action' not in new_kwargs: new_kwargs['action'] = trans.action return url_for( **new_kwargs) self.use_panels = ( kwargs.get( 'use_panels', False ) in [ True, 'True', 'true' ] ) self.advanced_search = ( kwargs.get( 'advanced_search', False ) in [ True, 'True', 'true' ] ) async_request = ( ( self.use_async ) and ( kwargs.get( 'async', False ) in [ True, 'True', 'true'] ) ) # Currently, filling the template returns a str object; this requires decoding the string into a # unicode object within mako templates. What probably should be done is to return the template as # utf-8 unicode; however, this would require encoding the object as utf-8 before returning the grid # results via a controller method, which is require substantial changes. Hence, for now, return grid # as str. page = trans.fill_template( iff( async_request, self.async_template, self.template ), grid=self, query=query, cur_page_num=page_num, num_pages=num_pages, num_page_links=self.num_page_links, default_filter_dict=self.default_filter, cur_filter_dict=cur_filter_dict, sort_key=sort_key, current_item=current_item, ids=kwargs.get( 'id', [] ), url=url, status=status, message=message, info_text=self.info_text, use_panels=self.use_panels, use_hide_message=self.use_hide_message, advanced_search=self.advanced_search, show_item_checkboxes=( self.show_item_checkboxes or kwargs.get( 'show_item_checkboxes', '' ) in [ 'True', 'true' ] ), # Pass back kwargs so that grid template can set and use args without # grid explicitly having to pass them. kwargs=kwargs ) trans.log_action( trans.get_user(), unicode( "grid.view" ), context, params ) return page
class DataManager(BaseUIController): @web.expose def index(self, trans, **kwd): not_is_admin = not trans.user_is_admin() if not_is_admin and not trans.app.config.enable_data_manager_user_view: raise paste.httpexceptions.HTTPUnauthorized( "This Galaxy instance is not configured to allow non-admins to view the data manager." ) message = escape(kwd.get('message', '')) status = escape(kwd.get('status', 'info')) return trans.fill_template("data_manager/index.mako", data_managers=trans.app.data_managers, tool_data_tables=trans.app.tool_data_tables, view_only=not_is_admin, message=message, status=status) @web.expose def manage_data_manager(self, trans, **kwd): not_is_admin = not trans.user_is_admin() if not_is_admin and not trans.app.config.enable_data_manager_user_view: raise paste.httpexceptions.HTTPUnauthorized( "This Galaxy instance is not configured to allow non-admins to view the data manager." ) message = escape(kwd.get('message', '')) status = escape(kwd.get('status', 'info')) data_manager_id = kwd.get('id', None) data_manager = trans.app.data_managers.get_manager(data_manager_id) if data_manager is None: return trans.response.send_redirect( web.url_for(controller="data_manager", action="index", message="Invalid Data Manager (%s) was requested" % data_manager_id, status="error")) jobs = list( reversed([ assoc.job for assoc in trans.sa_session.query( trans.app.model.DataManagerJobAssociation).filter_by( data_manager_id=data_manager_id) ])) return trans.fill_template("data_manager/manage_data_manager.mako", data_manager=data_manager, jobs=jobs, view_only=not_is_admin, message=message, status=status) @web.expose def view_job(self, trans, **kwd): not_is_admin = not trans.user_is_admin() if not_is_admin and not trans.app.config.enable_data_manager_user_view: raise paste.httpexceptions.HTTPUnauthorized( "This Galaxy instance is not configured to allow non-admins to view the data manager." ) message = escape(kwd.get('message', '')) status = escape(kwd.get('status', 'info')) job_id = kwd.get('id', None) try: job_id = trans.security.decode_id(job_id) job = trans.sa_session.query(trans.app.model.Job).get(job_id) except Exception, e: job = None log.error("Bad job id (%s) passed to view_job: %s" % (job_id, e)) if not job: return trans.response.send_redirect( web.url_for(controller="data_manager", action="index", message="Invalid job (%s) was requested" % job_id, status="error")) data_manager_id = job.data_manager_association.data_manager_id data_manager = trans.app.data_managers.get_manager(data_manager_id) hdas = [assoc.dataset for assoc in job.get_output_datasets()] data_manager_output = [] error_messages = [] for hda in hdas: try: data_manager_json = loads(open(hda.get_file_name()).read()) except Exception, e: data_manager_json = {} error_messages.append( escape( "Unable to obtain data_table info for hda (%s): %s" % (hda.id, e))) values = [] for key, value in data_manager_json.get('data_tables', {}).iteritems(): values.append((key, value)) data_manager_output.append(values)
def from_dict( Class, trans, d, secure=True ): module = Class( trans ) state = loads( d["tool_state"] ) module.recover_state( state ) return module
def pretty_print_json(json_data, is_json_string=False): if is_json_string: json_data = json.loads(json_data) return json.dumps(json_data, sort_keys=True, indent=4)
def cleanup_after_job(self): """ Set history, datasets, and jobs' attributes and clean up archive directory. """ # # Helper methods. # def file_in_dir(file_path, a_dir): """ Returns true if file is in directory. """ abs_file_path = os.path.abspath(file_path) return os.path.split(abs_file_path)[0] == a_dir def read_file_contents(file_path): """ Read contents of a file. """ fp = open(file_path, 'rb') buffsize = 1048576 file_contents = '' try: while True: file_contents += fp.read(buffsize) if not file_contents or len(file_contents) % buffsize != 0: break except OverflowError: pass fp.close() return file_contents def get_tag_str(tag, value): """ Builds a tag string for a tag, value pair. """ if not value: return tag else: return tag + ":" + value # # Import history. # jiha = self.sa_session.query(model.JobImportHistoryArchive).filter_by( job_id=self.job_id).first() if jiha: try: archive_dir = jiha.archive_dir user = jiha.job.user # # Create history. # history_attr_file_name = os.path.join(archive_dir, 'history_attrs.txt') history_attr_str = read_file_contents(history_attr_file_name) history_attrs = loads(history_attr_str) # Create history. new_history = model.History( name='imported from archive: %s' % history_attrs['name'].encode('utf-8'), user=user) new_history.importing = True new_history.hid_counter = history_attrs['hid_counter'] new_history.genome_build = history_attrs['genome_build'] self.sa_session.add(new_history) jiha.history = new_history self.sa_session.flush() # Add annotation, tags. if user: self.add_item_annotation(self.sa_session, user, new_history, history_attrs['annotation']) """ TODO: figure out to how add tags to item. for tag, value in history_attrs[ 'tags' ].items(): trans.app.tag_handler.apply_item_tags( trans, trans.user, new_history, get_tag_str( tag, value ) ) """ # # Create datasets. # datasets_attrs_file_name = os.path.join( archive_dir, 'datasets_attrs.txt') datasets_attr_str = read_file_contents( datasets_attrs_file_name) datasets_attrs = loads(datasets_attr_str) if os.path.exists(datasets_attrs_file_name + ".provenance"): provenance_attr_str = read_file_contents( datasets_attrs_file_name + ".provenance") provenance_attrs = loads(provenance_attr_str) datasets_attrs += provenance_attrs # Get counts of how often each dataset file is used; a file can # be linked to multiple dataset objects (HDAs). datasets_usage_counts = {} for dataset_attrs in datasets_attrs: temp_dataset_file_name = \ os.path.abspath( os.path.join( archive_dir, dataset_attrs['file_name'] ) ) if (temp_dataset_file_name not in datasets_usage_counts): datasets_usage_counts[temp_dataset_file_name] = 0 datasets_usage_counts[temp_dataset_file_name] += 1 # Create datasets. for dataset_attrs in datasets_attrs: metadata = dataset_attrs['metadata'] # Create dataset and HDA. hda = model.HistoryDatasetAssociation( name=dataset_attrs['name'].encode('utf-8'), extension=dataset_attrs['extension'], info=dataset_attrs['info'].encode('utf-8'), blurb=dataset_attrs['blurb'], peek=dataset_attrs['peek'], designation=dataset_attrs['designation'], visible=dataset_attrs['visible'], dbkey=metadata['dbkey'], metadata=metadata, history=new_history, create_dataset=True, sa_session=self.sa_session) if 'uuid' in dataset_attrs: hda.dataset.uuid = dataset_attrs["uuid"] if dataset_attrs.get('exported', True) is False: hda.state = hda.states.DISCARDED hda.deleted = True hda.purged = True else: hda.state = hda.states.OK self.sa_session.add(hda) self.sa_session.flush() new_history.add_dataset(hda, genome_build=None) hda.hid = dataset_attrs[ 'hid'] # Overwrite default hid set when HDA added to history. # TODO: Is there a way to recover permissions? Is this needed? # permissions = trans.app.security_agent.history_get_default_permissions( new_history ) # trans.app.security_agent.set_all_dataset_permissions( hda.dataset, permissions ) self.sa_session.flush() if dataset_attrs.get('exported', True) is True: # Do security check and move/copy dataset data. temp_dataset_file_name = \ os.path.abspath( os.path.join( archive_dir, dataset_attrs['file_name'] ) ) if not file_in_dir( temp_dataset_file_name, os.path.join(archive_dir, "datasets")): raise Exception("Invalid dataset path: %s" % temp_dataset_file_name) if datasets_usage_counts[temp_dataset_file_name] == 1: self.app.object_store.update_from_file( hda.dataset, file_name=temp_dataset_file_name, create=True) # Import additional files if present. Histories exported previously might not have this attribute set. dataset_extra_files_path = dataset_attrs.get( 'extra_files_path', None) if dataset_extra_files_path: try: file_list = os.listdir( os.path.join(archive_dir, dataset_extra_files_path)) except OSError: file_list = [] if file_list: for extra_file in file_list: self.app.object_store.update_from_file( hda.dataset, extra_dir='dataset_%s_files' % hda.dataset.id, alt_name=extra_file, file_name=os.path.join( archive_dir, dataset_extra_files_path, extra_file), create=True) else: datasets_usage_counts[temp_dataset_file_name] -= 1 shutil.copyfile(temp_dataset_file_name, hda.file_name) hda.dataset.set_total_size( ) # update the filesize record in the database # Set tags, annotations. if user: self.add_item_annotation(self.sa_session, user, hda, dataset_attrs['annotation']) # TODO: Set tags. """ for tag, value in dataset_attrs[ 'tags' ].items(): trans.app.tag_handler.apply_item_tags( trans, trans.user, hda, get_tag_str( tag, value ) ) self.sa_session.flush() """ # Although metadata is set above, need to set metadata to recover BAI for BAMs. if hda.extension == 'bam': self.app.datatypes_registry.set_external_metadata_tool.tool_action.execute_via_app( self.app.datatypes_registry. set_external_metadata_tool, self.app, jiha.job.session_id, new_history.id, jiha.job.user, incoming={'input1': hda}, overwrite=False) # # Create jobs. # # Read jobs attributes. jobs_attr_file_name = os.path.join(archive_dir, 'jobs_attrs.txt') jobs_attr_str = read_file_contents(jobs_attr_file_name) # Decode jobs attributes. def as_hda(obj_dct): """ Hook to 'decode' an HDA; method uses history and HID to get the HDA represented by the encoded object. This only works because HDAs are created above. """ if obj_dct.get('__HistoryDatasetAssociation__', False): return self.sa_session.query( model.HistoryDatasetAssociation).filter_by( history=new_history, hid=obj_dct['hid']).first() return obj_dct jobs_attrs = loads(jobs_attr_str, object_hook=as_hda) # Create each job. for job_attrs in jobs_attrs: imported_job = model.Job() imported_job.user = user # TODO: set session? # imported_job.session = trans.get_galaxy_session().id imported_job.history = new_history imported_job.imported = True imported_job.tool_id = job_attrs['tool_id'] imported_job.tool_version = job_attrs['tool_version'] imported_job.set_state(job_attrs['state']) imported_job.info = job_attrs.get('info', None) imported_job.exit_code = job_attrs.get('exit_code', None) imported_job.traceback = job_attrs.get('traceback', None) imported_job.stdout = job_attrs.get('stdout', None) imported_job.stderr = job_attrs.get('stderr', None) imported_job.command_line = job_attrs.get( 'command_line', None) try: imported_job.create_time = datetime.datetime.strptime( job_attrs["create_time"], "%Y-%m-%dT%H:%M:%S.%f") imported_job.update_time = datetime.datetime.strptime( job_attrs["update_time"], "%Y-%m-%dT%H:%M:%S.%f") except: pass self.sa_session.add(imported_job) self.sa_session.flush() class HistoryDatasetAssociationIDEncoder(json.JSONEncoder): """ Custom JSONEncoder for a HistoryDatasetAssociation that encodes an HDA as its ID. """ def default(self, obj): """ Encode an HDA, default encoding for everything else. """ if isinstance(obj, model.HistoryDatasetAssociation): return obj.id return json.JSONEncoder.default(self, obj) # Set parameters. May be useful to look at metadata.py for creating parameters. # TODO: there may be a better way to set parameters, e.g.: # for name, value in tool.params_to_strings( incoming, trans.app ).iteritems(): # job.add_parameter( name, value ) # to make this work, we'd need to flesh out the HDA objects. The code below is # relatively similar. for name, value in job_attrs['params'].items(): # Transform parameter values when necessary. if isinstance(value, model.HistoryDatasetAssociation): # HDA input: use hid to find input. input_hda = self.sa_session.query( model.HistoryDatasetAssociation ) \ .filter_by( history=new_history, hid=value.hid ).first() value = input_hda.id # print "added parameter %s-->%s to job %i" % ( name, value, imported_job.id ) imported_job.add_parameter( name, dumps(value, cls=HistoryDatasetAssociationIDEncoder)) # TODO: Connect jobs to input datasets. # Connect jobs to output datasets. for output_hid in job_attrs['output_datasets']: # print "%s job has output dataset %i" % (imported_job.id, output_hid) output_hda = self.sa_session.query( model.HistoryDatasetAssociation).filter_by( history=new_history, hid=output_hid).first() if output_hda: imported_job.add_output_dataset( output_hda.name, output_hda) # Connect jobs to input datasets. if 'input_mapping' in job_attrs: for input_name, input_hid in job_attrs[ 'input_mapping'].items(): input_hda = self.sa_session.query( model.HistoryDatasetAssociation ) \ .filter_by( history=new_history, hid=input_hid ).first() if input_hda: imported_job.add_input_dataset( input_name, input_hda) self.sa_session.flush() # Done importing. new_history.importing = False self.sa_session.flush() # Cleanup. if os.path.exists(archive_dir): shutil.rmtree(archive_dir) except Exception, e: jiha.job.stderr += "Error cleaning up history import job: %s" % e self.sa_session.flush()
assert col is Request_table.c.notification except Exception, e: log.debug( "Creating column 'notification' in the 'request' table failed: %s" % ( str( e ) ) ) cmd = "SELECT id, user_id, notify FROM request" result = migrate_engine.execute( cmd ) for r in result: id = int(r[0]) notify_old = r[1] notify_new = dict(email=[], sample_states=[], body='', subject='') cmd = "update request set notification='%s' where id=%i" % (dumps(notify_new), id) migrate_engine.execute( cmd ) cmd = "SELECT id, notification FROM request" result = migrate_engine.execute( cmd ) for r in result: rr = loads(str(r[1])) # remove the 'notify' column for non-sqlite databases. if migrate_engine.name != 'sqlite': try: Request_table.c.notify.drop() except Exception, e: log.debug( "Deleting column 'notify' from the 'request' table failed: %s" % ( str( e ) ) ) def downgrade(migrate_engine): metadata.bind = migrate_engine pass