def __extract_payload_from_request(trans, func, kwargs):
content_type = trans.request.headers['content-type']
if content_type.startswith('application/x-www-form-urlencoded') or content_type.startswith('multipart/form-data'):
# If the content type is a standard type such as multipart/form-data, the wsgi framework parses the request body
# and loads all field values into kwargs. However, kwargs also contains formal method parameters etc. which
# are not a part of the request body. This is a problem because it's not possible to differentiate between values
# which are a part of the request body, and therefore should be a part of the payload, and values which should not be
# in the payload. Therefore, the decorated method's formal arguments are discovered through reflection and removed from
# the payload dictionary. This helps to prevent duplicate argument conflicts in downstream methods.
payload = kwargs.copy()
named_args, _, _, _ = inspect.getargspec(func)
for arg in named_args:
payload.pop(arg, None)
for k, v in payload.iteritems():
if isinstance(v, (str, unicode)):
try:
payload[k] = loads(v)
except:
# may not actually be json, just continue
pass
payload = util.recursively_stringify_dictionary_keys( payload )
else:
# Assume application/json content type and parse request body manually, since wsgi won't do it. However, the order of this check
# should ideally be in reverse, with the if clause being a check for application/json and the else clause assuming a standard encoding
# such as multipart/form-data. Leaving it as is for backward compatibility, just in case.
payload = util.recursively_stringify_dictionary_keys( loads( trans.request.body ) )
return payload
def __extract_payload_from_request(trans, func, kwargs):
content_type = trans.request.headers['content-type']
if content_type.startswith(
'application/x-www-form-urlencoded') or content_type.startswith(
'multipart/form-data'):
# If the content type is a standard type such as multipart/form-data, the wsgi framework parses the request body
# and loads all field values into kwargs. However, kwargs also contains formal method parameters etc. which
# are not a part of the request body. This is a problem because it's not possible to differentiate between values
# which are a part of the request body, and therefore should be a part of the payload, and values which should not be
# in the payload. Therefore, the decorated method's formal arguments are discovered through reflection and removed from
# the payload dictionary. This helps to prevent duplicate argument conflicts in downstream methods.
payload = kwargs.copy()
named_args, _, _, _ = inspect.getargspec(func)
for arg in named_args:
payload.pop(arg, None)
for k, v in payload.iteritems():
if isinstance(v, (str, unicode)):
try:
payload[k] = loads(v)
except:
# may not actually be json, just continue
pass
payload = util.recursively_stringify_dictionary_keys(payload)
else:
# Assume application/json content type and parse request body manually, since wsgi won't do it. However, the order of this check
# should ideally be in reverse, with the if clause being a check for application/json and the else clause assuming a standard encoding
# such as multipart/form-data. Leaving it as is for backward compatibility, just in case.
payload = util.recursively_stringify_dictionary_keys(
loads(trans.request.body))
return payload
def recover_runtime_state( self, runtime_state ):
""" Take secure runtime state from persisted invocation and convert it
into a DefaultToolState object for use during workflow invocation.
"""
state = galaxy.tools.DefaultToolState()
app = self.trans.app
state.decode( runtime_state, self.tool, app, secure=False )
state_dict = loads( runtime_state )
if RUNTIME_STEP_META_STATE_KEY in state_dict:
self.__restore_step_meta_runtime_state( loads( state_dict[ RUNTIME_STEP_META_STATE_KEY ] ) )
return state
def get_chrom_info(self,
dbkey,
trans=None,
custom_build_hack_get_len_from_fasta_conversion=True):
# FIXME: flag to turn off custom_build_hack_get_len_from_fasta_conversion should not be required
chrom_info = None
db_dataset = None
# Collect chromInfo from custom builds
if trans:
db_dataset = trans.db_dataset_for(dbkey)
if db_dataset:
chrom_info = db_dataset.file_name
else:
# Do Custom Build handling
if trans.user and ('dbkeys' in trans.user.preferences) and (
dbkey in loads(trans.user.preferences['dbkeys'])):
custom_build_dict = loads(
trans.user.preferences['dbkeys'])[dbkey]
# HACK: the attempt to get chrom_info below will trigger the
# fasta-to-len converter if the dataset is not available or,
# which will in turn create a recursive loop when
# running the fasta-to-len tool. So, use a hack in the second
# condition below to avoid getting chrom_info when running the
# fasta-to-len converter.
if 'fasta' in custom_build_dict and custom_build_hack_get_len_from_fasta_conversion:
# Build is defined by fasta; get len file, which is obtained from converting fasta.
build_fasta_dataset = trans.sa_session.query(
trans.app.model.HistoryDatasetAssociation).get(
custom_build_dict['fasta'])
chrom_info = build_fasta_dataset.get_converted_dataset(
trans, 'len').file_name
elif 'len' in custom_build_dict:
# Build is defined by len file, so use it.
chrom_info = trans.sa_session.query(
trans.app.model.HistoryDatasetAssociation).get(
custom_build_dict['len']).file_name
# Check Data table
if not chrom_info:
dbkey_table = self._app.tool_data_tables.get(
self._data_table_name, None)
if dbkey_table is not None:
chrom_info = dbkey_table.get_entry('value',
dbkey,
'len_path',
default=None)
# use configured server len path
if not chrom_info:
# Default to built-in build.
# Since we are using an unverified dbkey, we will sanitize the dbkey before use
chrom_info = os.path.join(
self._static_chrom_info_path,
"%s.len" % sanitize_lists_to_string(dbkey))
chrom_info = os.path.abspath(chrom_info)
return (chrom_info, db_dataset)
def get_genome_build_names(self, trans=None):
# FIXME: how to deal with key duplicates?
rval = []
# load user custom genome builds
if trans is not None:
if trans.history:
# This is a little bit Odd. We are adding every .len file in the current history to dbkey list,
# but this is previous behavior from trans.db_names, so we'll continue to do it.
# It does allow one-off, history specific dbkeys to be created by a user. But we are not filtering,
# so a len file will be listed twice (as the build name and again as dataset name),
# if custom dbkey creation/conversion occurred within the current history.
datasets = trans.sa_session.query( self._app.model.HistoryDatasetAssociation ) \
.filter_by( deleted=False, history_id=trans.history.id, extension="len" )
for dataset in datasets:
rval.append(
(dataset.dbkey,
"%s (%s) [History]" % (dataset.name, dataset.dbkey)))
user = trans.user
if user and hasattr(
user, 'preferences') and 'dbkeys' in user.preferences:
user_keys = loads(user.preferences['dbkeys'])
for key, chrom_dict in user_keys.iteritems():
rval.append(
(key, "%s (%s) [Custom]" % (chrom_dict['name'], key)))
# Load old builds.txt static keys
rval.extend(self._static_dbkeys)
#load dbkeys from dbkey data table
dbkey_table = self._app.tool_data_tables.get(self._data_table_name,
None)
if dbkey_table is not None:
for field_dict in dbkey_table.get_named_fields_list():
rval.append((field_dict['value'], field_dict['name']))
return rval
def sweepster( self, trans, id=None, hda_ldda=None, dataset_id=None, regions=None ):
"""
Displays a sweepster visualization using the incoming parameters. If id is available,
get the visualization with the given id; otherwise, create a new visualization using
a given dataset and regions.
"""
regions = regions or '{}'
# Need to create history if necessary in order to create tool form.
trans.get_history( create=True )
if id:
# Loading a shared visualization.
viz = self.get_visualization( trans, id )
viz_config = self.get_visualization_config( trans, viz )
decoded_id = self.decode_id( viz_config[ 'dataset_id' ] )
dataset = self.hda_manager.get_owned( decoded_id, trans.user, current_history=trans.history )
else:
# Loading new visualization.
dataset = self.get_hda_or_ldda( trans, hda_ldda, dataset_id )
job = self.hda_manager.creating_job( dataset )
viz_config = {
'dataset_id': dataset_id,
'tool_id': job.tool_id,
'regions': loads( regions )
}
# Add tool, dataset attributes to config based on id.
tool = trans.app.toolbox.get_tool( viz_config[ 'tool_id' ] )
viz_config[ 'tool' ] = tool.to_dict( trans, io_details=True )
viz_config[ 'dataset' ] = trans.security.encode_dict_ids( dataset.to_dict() )
return trans.fill_template_mako( "visualization/sweepster.mako", config=viz_config )
def from_workflow_step( Class, trans, step ):
toolbox = trans.app.toolbox
tool_id = step.tool_id
if toolbox:
# See if we have access to a different version of the tool.
# TODO: If workflows are ever enhanced to use tool version
# in addition to tool id, enhance the selection process here
# to retrieve the correct version of the tool.
tool_id = toolbox.get_tool_id( tool_id )
if ( toolbox and tool_id ):
if step.config:
# This step has its state saved in the config field due to the
# tool being previously unavailable.
return module_factory.from_dict(trans, loads(step.config), secure=False)
tool_version = step.tool_version
module = Class( trans, tool_id, tool_version=tool_version )
if step.tool_version and (step.tool_version != module.tool.version):
message = "%s: using version '%s' instead of version '%s' indicated in this workflow." % (tool_id, module.tool.version, step.tool_version)
log.debug(message)
module.version_changes.append(message)
module.recover_state( step.tool_inputs )
module.errors = step.tool_errors
module.workflow_outputs = step.workflow_outputs
pjadict = {}
for pja in step.post_job_actions:
pjadict[pja.action_type] = pja
module.post_job_actions = pjadict
return module
return None
def chroms( self, trans, dbkey=None, num=None, chrom=None, low=None ):
"""
Returns a naturally sorted list of chroms/contigs for a given dbkey.
Use either chrom or low to specify the starting chrom in the return list.
"""
# If there is no dbkey owner, default to current user.
dbkey_owner, dbkey = decode_dbkey( dbkey )
if dbkey_owner:
dbkey_user = trans.sa_session.query( trans.app.model.User ).filter_by( username=dbkey_owner ).first()
else:
dbkey_user = trans.user
#
# Get/create genome object.
#
genome = None
twobit_file = None
# Look first in user's custom builds.
if dbkey_user and 'dbkeys' in dbkey_user.preferences:
user_keys = loads( dbkey_user.preferences['dbkeys'] )
if dbkey in user_keys:
dbkey_attributes = user_keys[ dbkey ]
dbkey_name = dbkey_attributes[ 'name' ]
# If there's a fasta for genome, convert to 2bit for later use.
if 'fasta' in dbkey_attributes:
build_fasta = trans.sa_session.query( trans.app.model.HistoryDatasetAssociation ).get( dbkey_attributes[ 'fasta' ] )
len_file = build_fasta.get_converted_dataset( trans, 'len' ).file_name
build_fasta.get_converted_dataset( trans, 'twobit' )
# HACK: set twobit_file to True rather than a file name because
# get_converted_dataset returns null during conversion even though
# there will eventually be a twobit file available for genome.
twobit_file = True
# Backwards compatibility: look for len file directly.
elif 'len' in dbkey_attributes:
len_file = trans.sa_session.query( trans.app.model.HistoryDatasetAssociation ).get( user_keys[ dbkey ][ 'len' ] ).file_name
if len_file:
genome = Genome( dbkey, dbkey_name, len_file=len_file, twobit_file=twobit_file )
# Look in history and system builds.
if not genome:
# Look in history for chromosome len file.
len_ds = trans.db_dataset_for( dbkey )
if len_ds:
genome = Genome( dbkey, dbkey_name, len_file=len_ds.file_name )
# Look in system builds.
elif dbkey in self.genomes:
genome = self.genomes[ dbkey ]
# Set up return value or log exception if genome not found for key.
rval = None
if genome:
rval = genome.to_dict( num=num, chrom=chrom, low=low )
else:
log.exception( 'genome not found for key %s' % dbkey )
return rval
def handle_result(self, result, param_dict, trans):
rval = loads(result.content)
return trans.fill_template(
'/external_services/generic_jquery_grid.mako',
result=rval,
param_dict=param_dict,
action=self.parent)
def __main__():
if len( sys.argv ) < 4:
print >>sys.stderr, 'usage: upload.py <root> <datatypes_conf> <json paramfile> <output spec> ...'
sys.exit( 1 )
output_paths = parse_outputs( sys.argv[4:] )
json_file = open( 'galaxy.json', 'w' )
registry = Registry()
registry.load_datatypes( root_dir=sys.argv[1], config=sys.argv[2] )
for line in open( sys.argv[3], 'r' ):
dataset = loads( line )
dataset = util.bunch.Bunch( **safe_dict( dataset ) )
try:
output_path = output_paths[int( dataset.dataset_id )][0]
except:
print >>sys.stderr, 'Output path for dataset %s not found on command line' % dataset.dataset_id
sys.exit( 1 )
if dataset.type == 'composite':
files_path = output_paths[int( dataset.dataset_id )][1]
add_composite_file( dataset, json_file, output_path, files_path )
else:
add_file( dataset, registry, json_file, output_path )
# clean up paramfile
# TODO: this will not work when running as the actual user unless the
# parent directory is writable by the user.
try:
os.remove( sys.argv[3] )
except:
pass
def check_job( self, job ):
if self._missing_params( job.params, [ 'type' ] ):
return self.job_states.INVALID
if job.params[ 'type' ] == 'init_transfer':
if self._missing_params( job.params, [ 'smrt_host', 'smrt_job_id' ] ):
return self.job_states.INVALID
url = 'http://' + job.params[ 'smrt_host' ] + self.api_path + '/Jobs/' + job.params[ 'smrt_job_id' ] + '/Status'
r = urllib2.urlopen( url )
status = json.loads( r.read() )
# TODO: error handling: unexpected json or bad response, bad url, etc.
if status[ 'Code' ] == 'Completed':
log.debug( "SMRT Portal job '%s' is Completed. Initiating transfer." % job.params[ 'smrt_job_id' ] )
return self.job_states.READY
return self.job_states.WAIT
if job.params[ 'type' ] == 'finish_transfer':
if self._missing_params( job.params, [ 'transfer_job_id' ] ):
return self.job_states.INVALID
# Get the TransferJob object and add it to the DeferredJob so we only look it up once.
if not hasattr( job, 'transfer_job' ):
job.transfer_job = self.sa_session.query( self.app.model.TransferJob ).get( int( job.params[ 'transfer_job_id' ] ) )
state = self.app.transfer_manager.get_state( job.transfer_job )
if not state:
log.error( 'No state for transfer job id: %s' % job.transfer_job.id )
return self.job_states.WAIT
if state[ 'state' ] in self.app.model.TransferJob.terminal_states:
return self.job_states.READY
log.debug( "Checked on finish transfer job %s, not done yet." % job.id )
return self.job_states.WAIT
else:
log.error( 'Unknown job type for SMRTPortalPlugin: %s' % str( job.params[ 'type' ] ) )
return self.job_states.INVALID
def upgrade(migrate_engine):
metadata.bind = migrate_engine
print __doc__
metadata.reflect()
Visualization_table = Table( "visualization", metadata, autoload=True )
Visualization_revision_table = Table( "visualization_revision", metadata, autoload=True )
# Create dbkey columns.
x = Column( "dbkey", TEXT )
y = Column( "dbkey", TEXT )
x.create( Visualization_table )
y.create( Visualization_revision_table )
# Manually create indexes for compatability w/ mysql_length.
xi = Index( "ix_visualization_dbkey", Visualization_table.c.dbkey, mysql_length = 200)
xi.create()
yi = Index( "ix_visualization_revision_dbkey", Visualization_revision_table.c.dbkey, mysql_length = 200)
yi.create()
assert x is Visualization_table.c.dbkey
assert y is Visualization_revision_table.c.dbkey
all_viz = migrate_engine.execute( "SELECT visualization.id as viz_id, visualization_revision.id as viz_rev_id, visualization_revision.config FROM visualization_revision \
LEFT JOIN visualization ON visualization.id=visualization_revision.visualization_id" )
for viz in all_viz:
viz_id = viz['viz_id']
viz_rev_id = viz['viz_rev_id']
if viz[Visualization_revision_table.c.config]:
dbkey = loads(viz[Visualization_revision_table.c.config]).get('dbkey', "").replace("'", "\\'")
migrate_engine.execute("UPDATE visualization_revision SET dbkey='%s' WHERE id=%s" % (dbkey, viz_rev_id))
migrate_engine.execute("UPDATE visualization SET dbkey='%s' WHERE id=%s" % (dbkey, viz_id))
def import_workflow( self, trans, payload, **kwd ):
"""
POST /api/tool_shed_repositories/import_workflow
Import the specified exported workflow contained in the specified installed tool shed repository into Galaxy.
:param key: the API key of the Galaxy user with which the imported workflow will be associated.
:param id: the encoded id of the ToolShedRepository object
The following parameters are included in the payload.
:param index: the index location of the workflow tuple in the list of exported workflows stored in the metadata for the specified repository
"""
api_key = kwd.get( 'key', None )
if api_key is None:
raise HTTPBadRequest( detail="Missing required parameter 'key' whose value is the API key for the Galaxy user importing the specified workflow." )
tool_shed_repository_id = kwd.get( 'id', '' )
if not tool_shed_repository_id:
raise HTTPBadRequest( detail="Missing required parameter 'id'." )
index = payload.get( 'index', None )
if index is None:
raise HTTPBadRequest( detail="Missing required parameter 'index'." )
repository = suc.get_tool_shed_repository_by_id( trans.app, tool_shed_repository_id )
exported_workflows = json.loads( self.exported_workflows( trans, tool_shed_repository_id ) )
# Since we don't have an in-memory object with an id, we'll identify the exported workflow via its location (i.e., index) in the list.
exported_workflow = exported_workflows[ int( index ) ]
workflow_name = exported_workflow[ 'workflow_name' ]
workflow, status, error_message = workflow_util.import_workflow( trans, repository, workflow_name )
if status == 'error':
log.debug( error_message )
return {}
return workflow.to_dict( view='element' )
def sweepster( self, trans, id=None, hda_ldda=None, dataset_id=None, regions=None ):
"""
Displays a sweepster visualization using the incoming parameters. If id is available,
get the visualization with the given id; otherwise, create a new visualization using
a given dataset and regions.
"""
regions = regions or '{}'
# Need to create history if necessary in order to create tool form.
trans.get_history( create=True )
if id:
# Loading a shared visualization.
viz = self.get_visualization( trans, id )
viz_config = self.get_visualization_config( trans, viz )
decoded_id = self.decode_id( viz_config[ 'dataset_id' ] )
dataset = self.hda_manager.get_owned( trans, decoded_id, trans.user )
else:
# Loading new visualization.
dataset = self.get_hda_or_ldda( trans, hda_ldda, dataset_id )
job = self.hda_manager.creating_job( dataset )
viz_config = {
'dataset_id': dataset_id,
'tool_id': job.tool_id,
'regions': loads( regions )
}
# Add tool, dataset attributes to config based on id.
tool = trans.app.toolbox.get_tool( viz_config[ 'tool_id' ] )
viz_config[ 'tool' ] = tool.to_dict( trans, io_details=True )
viz_config[ 'dataset' ] = trans.security.encode_dict_ids( dataset.to_dict() )
return trans.fill_template_mako( "visualization/sweepster.mako", config=viz_config )
def import_workflows(self, trans, **kwd):
"""
POST /api/tool_shed_repositories/import_workflows
Import all of the exported workflows contained in the specified installed tool shed repository into Galaxy.
:param key: the API key of the Galaxy user with which the imported workflows will be associated.
:param id: the encoded id of the ToolShedRepository object
"""
api_key = kwd.get('key', None)
if api_key is None:
raise HTTPBadRequest(
detail=
"Missing required parameter 'key' whose value is the API key for the Galaxy user importing the specified workflow."
)
tool_shed_repository_id = kwd.get('id', '')
if not tool_shed_repository_id:
raise HTTPBadRequest(detail="Missing required parameter 'id'.")
repository = suc.get_tool_shed_repository_by_id(
trans.app, tool_shed_repository_id)
exported_workflows = json.loads(
self.exported_workflows(trans, tool_shed_repository_id))
imported_workflow_dicts = []
for exported_workflow_dict in exported_workflows:
workflow_name = exported_workflow_dict['workflow_name']
workflow, status, error_message = workflow_util.import_workflow(
trans, repository, workflow_name)
if status == 'error':
log.debug(error_message)
else:
imported_workflow_dicts.append(
workflow.to_dict(view='element'))
return imported_workflow_dicts
def _POST_to_saved( self, trans, id=None, revision=None, type=None, config=None, title=None, **kwargs ):
"""
Save the visualiztion info (revision, type, config, title, etc.) to
the Visualization at `id` or to a new Visualization if `id` is None.
Uses POST/redirect/GET after a successful save, redirecting to GET.
"""
DEFAULT_VISUALIZATION_NAME = 'Unnamed Visualization'
# post to saved in order to save a visualization
if type is None or config is None:
return HTTPBadRequest( 'A visualization type and config are required to save a visualization' )
if isinstance( config, basestring ):
config = loads( config )
title = title or DEFAULT_VISUALIZATION_NAME
# TODO: allow saving to (updating) a specific revision - should be part of UsesVisualization
# TODO: would be easier if this returned the visualization directly
# check security if posting to existing visualization
if id is not None:
self.get_visualization( trans, id, check_ownership=True, check_accessible=False )
# ??: on not owner: error raised, but not returned (status = 200)
# TODO: there's no security check in save visualization (if passed an id)
returned = self.save_visualization( trans, config, type, id, title )
# redirect to GET to prevent annoying 'Do you want to post again?' dialog on page reload
render_url = web.url_for( controller='visualization', action='saved', id=returned.get( 'vis_id' ) )
return trans.response.send_redirect( render_url )
def import_workflows( self, trans, **kwd ):
"""
POST /api/tool_shed_repositories/import_workflows
Import all of the exported workflows contained in the specified installed tool shed repository into Galaxy.
:param key: the API key of the Galaxy user with which the imported workflows will be associated.
:param id: the encoded id of the ToolShedRepository object
"""
api_key = kwd.get( 'key', None )
if api_key is None:
raise HTTPBadRequest( detail="Missing required parameter 'key' whose value is the API key for the Galaxy user importing the specified workflow." )
tool_shed_repository_id = kwd.get( 'id', '' )
if not tool_shed_repository_id:
raise HTTPBadRequest( detail="Missing required parameter 'id'." )
repository = suc.get_tool_shed_repository_by_id( trans.app, tool_shed_repository_id )
exported_workflows = json.loads( self.exported_workflows( trans, tool_shed_repository_id ) )
imported_workflow_dicts = []
for exported_workflow_dict in exported_workflows:
workflow_name = exported_workflow_dict[ 'workflow_name' ]
workflow, status, error_message = workflow_util.import_workflow( trans, repository, workflow_name )
if status == 'error':
log.debug( error_message )
else:
imported_workflow_dicts.append( workflow.to_dict( view='element' ) )
return imported_workflow_dicts
def import_workflow( self, trans, payload, **kwd ):
"""
POST /api/tool_shed_repositories/import_workflow
Import the specified exported workflow contained in the specified installed tool shed repository into Galaxy.
:param key: the API key of the Galaxy user with which the imported workflow will be associated.
:param id: the encoded id of the ToolShedRepository object
The following parameters are included in the payload.
:param index: the index location of the workflow tuple in the list of exported workflows stored in the metadata for the specified repository
"""
api_key = kwd.get( 'key', None )
if api_key is None:
raise HTTPBadRequest( detail="Missing required parameter 'key' whose value is the API key for the Galaxy user importing the specified workflow." )
tool_shed_repository_id = kwd.get( 'id', '' )
if not tool_shed_repository_id:
raise HTTPBadRequest( detail="Missing required parameter 'id'." )
index = payload.get( 'index', None )
if index is None:
raise HTTPBadRequest( detail="Missing required parameter 'index'." )
repository = suc.get_tool_shed_repository_by_id( trans.app, tool_shed_repository_id )
exported_workflows = json.loads( self.exported_workflows( trans, tool_shed_repository_id ) )
# Since we don't have an in-memory object with an id, we'll identify the exported workflow via its location (i.e., index) in the list.
exported_workflow = exported_workflows[ int( index ) ]
workflow_name = exported_workflow[ 'workflow_name' ]
workflow, status, error_message = workflow_util.import_workflow( trans, repository, workflow_name )
if status == 'error':
log.debug( error_message )
return {}
return workflow.to_dict( view='element' )
def get_dbkeys(self, trans, chrom_info=False, **kwd):
""" Returns all known dbkeys. If chrom_info is True, only dbkeys with
chromosome lengths are returned. """
self.check_and_reload()
dbkeys = []
# Add user's custom keys to dbkeys.
user_keys_dict = {}
user = trans.get_user()
if user:
if 'dbkeys' in user.preferences:
user_keys_dict = loads(user.preferences['dbkeys'])
dbkeys.extend([(attributes['name'], key)
for key, attributes in user_keys_dict.items()])
# Add app keys to dbkeys.
# If chrom_info is True, only include keys with len files (which contain chromosome info).
filter_fn = lambda b: True
if chrom_info:
filter_fn = lambda b: b.len_file is not None
dbkeys.extend([(genome.description, genome.key)
for key, genome in self.genomes.items()
if filter_fn(genome)])
return dbkeys
def __main__():
if len(sys.argv) < 4:
print >> sys.stderr, 'usage: upload.py <root> <datatypes_conf> <json paramfile> <output spec> ...'
sys.exit(1)
output_paths = parse_outputs(sys.argv[4:])
json_file = open('galaxy.json', 'w')
registry = Registry()
registry.load_datatypes(root_dir=sys.argv[1], config=sys.argv[2])
for line in open(sys.argv[3], 'r'):
dataset = loads(line)
dataset = util.bunch.Bunch(**safe_dict(dataset))
try:
output_path = output_paths[int(dataset.dataset_id)][0]
except:
print >> sys.stderr, 'Output path for dataset %s not found on command line' % dataset.dataset_id
sys.exit(1)
if dataset.type == 'composite':
files_path = output_paths[int(dataset.dataset_id)][1]
add_composite_file(dataset, registry, json_file, output_path,
files_path)
else:
add_file(dataset, registry, json_file, output_path)
# clean up paramfile
# TODO: this will not work when running as the actual user unless the
# parent directory is writable by the user.
try:
os.remove(sys.argv[3])
except:
pass
def get_genome_build_names( self, trans=None ):
# FIXME: how to deal with key duplicates?
rval = []
# load user custom genome builds
if trans is not None:
if trans.history:
# This is a little bit Odd. We are adding every .len file in the current history to dbkey list,
# but this is previous behavior from trans.db_names, so we'll continue to do it.
# It does allow one-off, history specific dbkeys to be created by a user. But we are not filtering,
# so a len file will be listed twice (as the build name and again as dataset name),
# if custom dbkey creation/conversion occurred within the current history.
datasets = trans.sa_session.query( self._app.model.HistoryDatasetAssociation ) \
.filter_by( deleted=False, history_id=trans.history.id, extension="len" )
for dataset in datasets:
rval.append( ( dataset.dbkey, "%s (%s) [History]" % ( dataset.name, dataset.dbkey ) ) )
user = trans.user
if user and hasattr( user, 'preferences' ) and 'dbkeys' in user.preferences:
user_keys = loads( user.preferences['dbkeys'] )
for key, chrom_dict in user_keys.iteritems():
rval.append( ( key, "%s (%s) [Custom]" % ( chrom_dict['name'], key ) ) )
# Load old builds.txt static keys
rval.extend( self._static_dbkeys )
# load dbkeys from dbkey data table
dbkey_table = self._app.tool_data_tables.get( self._data_table_name, None )
if dbkey_table is not None:
for field_dict in dbkey_table.get_named_fields_list():
rval.append( ( field_dict[ 'value' ], field_dict[ 'name' ] ) )
return rval
def import_workflow(trans, repository, workflow_name):
"""Import a workflow contained in an installed tool shed repository into Galaxy (this method is called only from Galaxy)."""
status = 'done'
message = ''
changeset_revision = repository.changeset_revision
metadata = repository.metadata
workflows = metadata.get('workflows', [])
tools_metadata = metadata.get('tools', [])
workflow_dict = None
for workflow_data_tuple in workflows:
# The value of workflow_data_tuple is ( relative_path_to_workflow_file, exported_workflow_dict ).
relative_path_to_workflow_file, exported_workflow_dict = workflow_data_tuple
if exported_workflow_dict['name'] == workflow_name:
# If the exported workflow is available on disk, import it.
if os.path.exists(relative_path_to_workflow_file):
workflow_file = open(relative_path_to_workflow_file, 'rb')
workflow_data = workflow_file.read()
workflow_file.close()
workflow_dict = json.loads(workflow_data)
else:
# Use the current exported_workflow_dict.
workflow_dict = exported_workflow_dict
break
if workflow_dict:
# Create workflow if possible.
workflow, missing_tool_tups = get_workflow_from_dict(
trans=trans,
workflow_dict=workflow_dict,
tools_metadata=tools_metadata,
repository_id=repository.id,
changeset_revision=changeset_revision)
# Save the workflow in the Galaxy database. Pass workflow_dict along to create annotation at this point.
stored_workflow = save_workflow(trans, workflow, workflow_dict)
# Use the latest version of the saved workflow.
workflow = stored_workflow.latest_workflow
if workflow_name:
workflow.name = workflow_name
# Provide user feedback and show workflow list.
if workflow.has_errors:
message += "Imported, but some steps in this workflow have validation errors. "
status = "error"
if workflow.has_cycles:
message += "Imported, but this workflow contains cycles. "
status = "error"
else:
message += "Workflow <b>%s</b> imported successfully. " % workflow.name
if missing_tool_tups:
name_and_id_str = ''
for missing_tool_tup in missing_tool_tups:
tool_id, tool_name, other = missing_tool_tup
name_and_id_str += 'name: %s, id: %s' % (str(tool_id),
str(tool_name))
message += "The following tools required by this workflow are missing from this Galaxy instance: %s. " % name_and_id_str
else:
workflow = None
message += 'The workflow named %s is not included in the metadata for revision %s of repository %s' % \
( str( workflow_name ), str( changeset_revision ), str( repository.name ) )
status = 'error'
return workflow, status, message
def import_workflow( trans, repository, workflow_name ):
"""Import a workflow contained in an installed tool shed repository into Galaxy (this method is called only from Galaxy)."""
status = 'done'
message = ''
changeset_revision = repository.changeset_revision
metadata = repository.metadata
workflows = metadata.get( 'workflows', [] )
tools_metadata = metadata.get( 'tools', [] )
workflow_dict = None
for workflow_data_tuple in workflows:
# The value of workflow_data_tuple is ( relative_path_to_workflow_file, exported_workflow_dict ).
relative_path_to_workflow_file, exported_workflow_dict = workflow_data_tuple
if exported_workflow_dict[ 'name' ] == workflow_name:
# If the exported workflow is available on disk, import it.
if os.path.exists( relative_path_to_workflow_file ):
workflow_file = open( relative_path_to_workflow_file, 'rb' )
workflow_data = workflow_file.read()
workflow_file.close()
workflow_dict = json.loads( workflow_data )
else:
# Use the current exported_workflow_dict.
workflow_dict = exported_workflow_dict
break
if workflow_dict:
# Create workflow if possible.
workflow, missing_tool_tups = get_workflow_from_dict( trans=trans,
workflow_dict=workflow_dict,
tools_metadata=tools_metadata,
repository_id=repository.id,
changeset_revision=changeset_revision )
# Save the workflow in the Galaxy database. Pass workflow_dict along to create annotation at this point.
stored_workflow = save_workflow( trans, workflow, workflow_dict )
# Use the latest version of the saved workflow.
workflow = stored_workflow.latest_workflow
if workflow_name:
workflow.name = workflow_name
# Provide user feedback and show workflow list.
if workflow.has_errors:
message += "Imported, but some steps in this workflow have validation errors. "
status = "error"
if workflow.has_cycles:
message += "Imported, but this workflow contains cycles. "
status = "error"
else:
message += "Workflow <b>%s</b> imported successfully. " % workflow.name
if missing_tool_tups:
name_and_id_str = ''
for missing_tool_tup in missing_tool_tups:
tool_id, tool_name, other = missing_tool_tup
name_and_id_str += 'name: %s, id: %s' % ( str( tool_id ), str( tool_name ) )
message += "The following tools required by this workflow are missing from this Galaxy instance: %s. " % name_and_id_str
else:
workflow = None
message += 'The workflow named %s is not included in the metadata for revision %s of repository %s' % \
( str( workflow_name ), str( changeset_revision ), str( repository.name ) )
status = 'error'
return workflow, status, message
def save(self, trans, id, content, annotations):
id = trans.security.decode_id(id)
page = trans.sa_session.query(model.Page).get(id)
assert page.user == trans.user
# Sanitize content
content = sanitize_html(content, 'utf-8', 'text/html')
# Add a new revision to the page with the provided content.
page_revision = model.PageRevision()
page_revision.title = page.title
page_revision.page = page
page.latest_revision = page_revision
page_revision.content = content
# Save annotations.
annotations = loads(annotations)
for annotation_dict in annotations:
item_id = trans.security.decode_id(annotation_dict['item_id'])
item_class = self.get_class(annotation_dict['item_class'])
item = trans.sa_session.query(item_class).filter_by(
id=item_id).first()
if not item:
raise RuntimeError("cannot find annotated item")
text = sanitize_html(annotation_dict['text'], 'utf-8', 'text/html')
# Add/update annotation.
if item_id and item_class and text:
# Get annotation association.
annotation_assoc_class = eval("model.%sAnnotationAssociation" %
item_class.__name__)
annotation_assoc = trans.sa_session.query(
annotation_assoc_class).filter_by(user=trans.get_user())
if item_class == model.History.__class__:
annotation_assoc = annotation_assoc.filter_by(history=item)
elif item_class == model.HistoryDatasetAssociation.__class__:
annotation_assoc = annotation_assoc.filter_by(hda=item)
elif item_class == model.StoredWorkflow.__class__:
annotation_assoc = annotation_assoc.filter_by(
stored_workflow=item)
elif item_class == model.WorkflowStep.__class__:
annotation_assoc = annotation_assoc.filter_by(
workflow_step=item)
annotation_assoc = annotation_assoc.first()
if not annotation_assoc:
# Create association.
annotation_assoc = annotation_assoc_class()
item.annotations.append(annotation_assoc)
annotation_assoc.user = trans.get_user()
# Set annotation user text.
annotation_assoc.annotation = text
trans.sa_session.flush()
def save( self, trans, vis_json=None, type=None, id=None, title=None, dbkey=None, annotation=None ):
"""
Save a visualization; if visualization does not have an ID, a new
visualization is created. Returns JSON of visualization.
"""
# Get visualization attributes from kwargs or from config.
vis_config = loads( vis_json )
vis_type = type or vis_config[ 'type' ]
vis_id = id or vis_config.get( 'id', None )
vis_title = title or vis_config.get( 'title', None )
vis_dbkey = dbkey or vis_config.get( 'dbkey', None )
vis_annotation = annotation or vis_config.get( 'annotation', None )
return self.save_visualization( trans, vis_config, vis_type, vis_id, vis_title, vis_dbkey, vis_annotation )
def save( self, trans, vis_json=None, type=None, id=None, title=None, dbkey=None, annotation=None ):
"""
Save a visualization; if visualization does not have an ID, a new
visualization is created. Returns JSON of visualization.
"""
# Get visualization attributes from kwargs or from config.
vis_config = loads( vis_json )
vis_type = type or vis_config[ 'type' ]
vis_id = id or vis_config.get( 'id', None )
vis_title = title or vis_config.get( 'title', None )
vis_dbkey = dbkey or vis_config.get( 'dbkey', None )
vis_annotation = annotation or vis_config.get( 'annotation', None )
return self.save_visualization( trans, vis_config, vis_type, vis_id, vis_title, vis_dbkey, vis_annotation )
def create_archive( history_attrs_file, datasets_attrs_file, jobs_attrs_file, out_file, gzip=False ):
""" Create archive from the given attribute/metadata files and save it to out_file. """
tarfile_mode = "w"
if gzip:
tarfile_mode += ":gz"
try:
history_archive = tarfile.open( out_file, tarfile_mode )
# Read datasets attributes from file.
datasets_attr_in = open( datasets_attrs_file, 'rb' )
datasets_attr_str = ''
buffsize = 1048576
try:
while True:
datasets_attr_str += datasets_attr_in.read( buffsize )
if not datasets_attr_str or len( datasets_attr_str ) % buffsize != 0:
break
except OverflowError:
pass
datasets_attr_in.close()
datasets_attrs = loads( datasets_attr_str )
# Add datasets to archive and update dataset attributes.
# TODO: security check to ensure that files added are in Galaxy dataset directory?
for dataset_attrs in datasets_attrs:
if dataset_attrs['exported']:
dataset_file_name = dataset_attrs[ 'file_name' ] # Full file name.
dataset_archive_name = os.path.join( 'datasets',
get_dataset_filename( dataset_attrs[ 'name' ], dataset_attrs[ 'extension' ] ) )
history_archive.add( dataset_file_name, arcname=dataset_archive_name )
# Update dataset filename to be archive name.
dataset_attrs[ 'file_name' ] = dataset_archive_name
# Rewrite dataset attributes file.
datasets_attrs_out = open( datasets_attrs_file, 'w' )
datasets_attrs_out.write( dumps( datasets_attrs ) )
datasets_attrs_out.close()
# Finish archive.
history_archive.add( history_attrs_file, arcname="history_attrs.txt" )
history_archive.add( datasets_attrs_file, arcname="datasets_attrs.txt" )
if os.path.exists( datasets_attrs_file + ".provenance" ):
history_archive.add( datasets_attrs_file + ".provenance", arcname="datasets_attrs.txt.provenance" )
history_archive.add( jobs_attrs_file, arcname="jobs_attrs.txt" )
history_archive.close()
# Status.
return 'Created history archive.'
except Exception, e:
return 'Error creating history archive: %s' % str( e ), sys.stderr
def params_from_strings(params, param_values, app, ignore_errors=False):
"""
Convert a dictionary of strings as produced by `params_to_strings`
back into parameter values (decode the json representation and then
allow each parameter to convert the basic types into the parameters
preferred form).
"""
rval = dict()
for key, value in param_values.iteritems():
value = json_fix(loads(value))
if key in params:
value = params[key].value_from_basic(value, app, ignore_errors)
rval[key] = value
return rval
def params_from_strings(params, param_values, app, ignore_errors=False):
"""
Convert a dictionary of strings as produced by `params_to_strings`
back into parameter values (decode the json representation and then
allow each parameter to convert the basic types into the parameters
preferred form).
"""
rval = dict()
for key, value in param_values.iteritems():
value = json_fix(loads(value))
if key in params:
value = params[key].value_from_basic(value, app, ignore_errors)
rval[key] = value
return rval
def reference(self, trans, dbkey, chrom, low, high):
"""
Return reference data for a build.
"""
self.check_and_reload()
# If there is no dbkey owner, default to current user.
dbkey_owner, dbkey = decode_dbkey(dbkey)
if dbkey_owner:
dbkey_user = trans.sa_session.query(
trans.app.model.User).filter_by(username=dbkey_owner).first()
else:
dbkey_user = trans.user
if not self.has_reference_data(dbkey, dbkey_user):
return None
#
# Get twobit file with reference data.
#
twobit_file_name = None
if dbkey in self.genomes:
# Built-in twobit.
twobit_file_name = self.genomes[dbkey].twobit_file
else:
user_keys = loads(dbkey_user.preferences['dbkeys'])
dbkey_attributes = user_keys[dbkey]
fasta_dataset = trans.sa_session.query(
trans.app.model.HistoryDatasetAssociation).get(
dbkey_attributes['fasta'])
msg = fasta_dataset.convert_dataset(trans, 'twobit')
if msg:
return msg
else:
twobit_dataset = fasta_dataset.get_converted_dataset(
trans, 'twobit')
twobit_file_name = twobit_dataset.file_name
# Read and return reference data.
try:
twobit = TwoBitFile(open(twobit_file_name))
if chrom in twobit:
seq_data = twobit[chrom].get(int(low), int(high))
return GenomeRegion(chrom=chrom,
start=low,
end=high,
sequence=seq_data)
except IOError:
return None
def get_readme_files_dict_for_display( app, tool_shed_url, repo_info_dict ):
"""
Return a dictionary of README files contained in the single repository being installed so they can be displayed on the tool panel section
selection page.
"""
name = repo_info_dict.keys()[ 0 ]
repo_info_tuple = repo_info_dict[ name ]
description, repository_clone_url, changeset_revision, ctx_rev, repository_owner, repository_dependencies, installed_td = \
suc.get_repo_info_tuple_contents( repo_info_tuple )
# Handle changing HTTP protocols over time.
tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( app, tool_shed_url )
params = dict( name=name, owner=repository_owner, changeset_revision=changeset_revision )
pathspec = [ 'repository', 'get_readme_files' ]
raw_text = common_util.tool_shed_get( app, tool_shed_url, pathspec=pathspec, params=params )
readme_files_dict = json.loads( raw_text )
return readme_files_dict
def save( self, trans, id, content, annotations ):
id = self.decode_id( id )
page = trans.sa_session.query( model.Page ).get( id )
assert page.user == trans.user
# Sanitize content
content = sanitize_html( content, 'utf-8', 'text/html' )
# Add a new revision to the page with the provided content.
page_revision = model.PageRevision()
page_revision.title = page.title
page_revision.page = page
page.latest_revision = page_revision
page_revision.content = content
# Save annotations.
annotations = loads( annotations )
for annotation_dict in annotations:
item_id = self.decode_id( annotation_dict[ 'item_id' ] )
item_class = self.get_class( annotation_dict[ 'item_class' ] )
item = trans.sa_session.query( item_class ).filter_by( id=item_id ).first()
if not item:
raise RuntimeError( "cannot find annotated item" )
text = sanitize_html( annotation_dict[ 'text' ], 'utf-8', 'text/html' )
# Add/update annotation.
if item_id and item_class and text:
# Get annotation association.
annotation_assoc_class = eval( "model.%sAnnotationAssociation" % item_class.__name__ )
annotation_assoc = trans.sa_session.query( annotation_assoc_class ).filter_by( user=trans.get_user() )
if item_class == model.History.__class__:
annotation_assoc = annotation_assoc.filter_by( history=item )
elif item_class == model.HistoryDatasetAssociation.__class__:
annotation_assoc = annotation_assoc.filter_by( hda=item )
elif item_class == model.StoredWorkflow.__class__:
annotation_assoc = annotation_assoc.filter_by( stored_workflow=item )
elif item_class == model.WorkflowStep.__class__:
annotation_assoc = annotation_assoc.filter_by( workflow_step=item )
annotation_assoc = annotation_assoc.first()
if not annotation_assoc:
# Create association.
annotation_assoc = annotation_assoc_class()
item.annotations.append( annotation_assoc )
annotation_assoc.user = trans.get_user()
# Set annotation user text.
annotation_assoc.annotation = text
trans.sa_session.flush()
def load_input_parameters( filename, erase_file = True ):
datasource_params = {}
try:
json_params = loads( open( filename, 'r' ).read() )
datasource_params = json_params.get( 'param_dict' )
except:
json_params = None
for line in open( filename, 'r' ):
try:
line = line.strip()
fields = line.split( '\t' )
datasource_params[ fields[0] ] = fields[1]
except:
continue
if erase_file:
open( filename, 'w' ).close() #open file for writing, then close, removes params from file
return json_params, datasource_params
def loads_recurse(item):
decoded_list = []
if isinstance( item, basestring):
try:
# Not clear what we're decoding, so recurse to ensure that we catch everything.
decoded_item = loads( item )
if isinstance( decoded_item, list):
decoded_list = loads_recurse( decoded_item )
else:
decoded_list = [ unicode( decoded_item ) ]
except ValueError:
decoded_list = [ unicode( item ) ]
elif isinstance( item, list):
for element in item:
a_list = loads_recurse( element )
decoded_list = decoded_list + a_list
return decoded_list
def loads_recurse(item):
decoded_list = []
if isinstance( item, basestring):
try:
# Not clear what we're decoding, so recurse to ensure that we catch everything.
decoded_item = loads( item )
if isinstance( decoded_item, list):
decoded_list = loads_recurse( decoded_item )
else:
decoded_list = [ unicode( decoded_item ) ]
except ValueError:
decoded_list = [ unicode( item ) ]
elif isinstance( item, list):
for element in item:
a_list = loads_recurse( element )
decoded_list = decoded_list + a_list
return decoded_list
def upgrade(migrate_engine):
metadata.bind = migrate_engine
print __doc__
metadata.reflect()
Visualization_table = Table("visualization", metadata, autoload=True)
Visualization_revision_table = Table("visualization_revision",
metadata,
autoload=True)
# Create dbkey columns.
x = Column("dbkey", TEXT)
y = Column("dbkey", TEXT)
x.create(Visualization_table)
y.create(Visualization_revision_table)
# Manually create indexes for compatability w/ mysql_length.
xi = Index("ix_visualization_dbkey",
Visualization_table.c.dbkey,
mysql_length=200)
xi.create()
yi = Index("ix_visualization_revision_dbkey",
Visualization_revision_table.c.dbkey,
mysql_length=200)
yi.create()
assert x is Visualization_table.c.dbkey
assert y is Visualization_revision_table.c.dbkey
all_viz = migrate_engine.execute(
"SELECT visualization.id as viz_id, visualization_revision.id as viz_rev_id, visualization_revision.config FROM visualization_revision \
LEFT JOIN visualization ON visualization.id=visualization_revision.visualization_id"
)
for viz in all_viz:
viz_id = viz['viz_id']
viz_rev_id = viz['viz_rev_id']
if viz[Visualization_revision_table.c.config]:
dbkey = loads(viz[Visualization_revision_table.c.config]).get(
'dbkey', "").replace("'", "\\'")
migrate_engine.execute(
"UPDATE visualization_revision SET dbkey='%s' WHERE id=%s" %
(dbkey, viz_rev_id))
migrate_engine.execute(
"UPDATE visualization SET dbkey='%s' WHERE id=%s" %
(dbkey, viz_id))
def get_readme_files_dict_for_display(app, tool_shed_url, repo_info_dict):
"""
Return a dictionary of README files contained in the single repository being installed so they can be displayed on the tool panel section
selection page.
"""
name = repo_info_dict.keys()[0]
repo_info_tuple = repo_info_dict[name]
description, repository_clone_url, changeset_revision, ctx_rev, repository_owner, repository_dependencies, installed_td = \
suc.get_repo_info_tuple_contents( repo_info_tuple )
# Handle changing HTTP protocols over time.
tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry(
app, tool_shed_url)
params = '?name=%s&owner=%s&changeset_revision=%s' % (
name, repository_owner, changeset_revision)
url = common_util.url_join(tool_shed_url,
'repository/get_readme_files%s' % params)
raw_text = common_util.tool_shed_get(app, tool_shed_url, url)
readme_files_dict = json.loads(raw_text)
return readme_files_dict
def has_reference_data( self, dbkey, dbkey_owner=None ):
"""
Returns true if there is reference data for the specified dbkey. If dbkey is custom,
dbkey_owner is needed to determine if there is reference data.
"""
# Look for key in built-in builds.
if dbkey in self.genomes and self.genomes[ dbkey ].twobit_file:
# There is built-in reference data.
return True
# Look for key in owner's custom builds.
if dbkey_owner and 'dbkeys' in dbkey_owner.preferences:
user_keys = loads( dbkey_owner.preferences[ 'dbkeys' ] )
if dbkey in user_keys:
dbkey_attributes = user_keys[ dbkey ]
if 'fasta' in dbkey_attributes:
# Fasta + converted datasets can provide reference data.
return True
return False
def saved( self, trans, id=None, revision=None, type=None, config=None, title=None, **kwargs ):
"""
"""
DEFAULT_VISUALIZATION_NAME = 'Unnamed Visualization'
# post to saved in order to save a visualization
#TODO: re-route this one to clear up signature
if trans.request.method == 'POST':
if type is None or config is None:
return HTTPBadRequest( 'A visualization type and config are required to save a visualization' )
if isinstance( config, basestring ):
config = loads( config )
title = title or DEFAULT_VISUALIZATION_NAME
#TODO: allow saving to (updating) a specific revision - should be part of UsesVisualization
#TODO: would be easier if this returned the visualization directly
# check security if posting to existing visualization
if id is not None:
visualization = self.get_visualization( trans, id, check_ownership=True, check_accessible=False )
#??: on not owner: error raised, but not returned (status = 200)
#TODO: there's no security check in save visualization (if passed an id)
returned = self.save_visualization( trans, config, type, id, title )
# redirect to GET to prevent annoying 'Do you want to post again?' dialog on page reload
render_url = web.url_for( controller='visualization', action='saved', id=returned.get( 'vis_id' ) )
return trans.response.send_redirect( render_url )
if id is None:
return HTTPBadRequest( 'A valid visualization id is required to load a visualization' )
# render the saved visualization by passing to render, sending latest revision config
#TODO: allow loading a specific revision - should be part of UsesVisualization
#visualization = self.get_visualization( trans, id, check_ownership=True, check_accessible=False )
visualization = self.get_visualization( trans, id, check_ownership=False, check_accessible=True )
config = copy.copy( visualization.latest_revision.config )
# re-add title to kwargs for passing to render
if title:
kwargs[ 'title' ] = title
return self.render( trans, visualization.type, visualization, config=config, **kwargs )
def saved( self, trans, id=None, revision=None, type=None, config=None, title=None, **kwargs ):
"""
"""
DEFAULT_VISUALIZATION_NAME = 'Unnamed Visualization'
# post to saved in order to save a visualization
#TODO: re-route this one to clear up signature
if trans.request.method == 'POST':
if type is None or config is None:
return HTTPBadRequest( 'A visualization type and config are required to save a visualization' )
if isinstance( config, basestring ):
config = loads( config )
title = title or DEFAULT_VISUALIZATION_NAME
#TODO: allow saving to (updating) a specific revision - should be part of UsesVisualization
#TODO: would be easier if this returned the visualization directly
# check security if posting to existing visualization
if id is not None:
visualization = self.get_visualization( trans, id, check_ownership=True, check_accessible=False )
#??: on not owner: error raised, but not returned (status = 200)
#TODO: there's no security check in save visualization (if passed an id)
returned = self.save_visualization( trans, config, type, id, title )
# redirect to GET to prevent annoying 'Do you want to post again?' dialog on page reload
render_url = web.url_for( controller='visualization', action='saved', id=returned.get( 'vis_id' ) )
return trans.response.send_redirect( render_url )
if id is None:
return HTTPBadRequest( 'A valid visualization id is required to load a visualization' )
# render the saved visualization by passing to render, sending latest revision config
#TODO: allow loading a specific revision - should be part of UsesVisualization
#visualization = self.get_visualization( trans, id, check_ownership=True, check_accessible=False )
visualization = self.get_visualization( trans, id, check_ownership=False, check_accessible=True )
config = copy.copy( visualization.latest_revision.config )
# re-add title to kwargs for passing to render
if title:
kwargs[ 'title' ] = title
return self.render( trans, visualization.type, visualization, config=config, **kwargs )
def reference( self, trans, dbkey, chrom, low, high ):
"""
Return reference data for a build.
"""
# If there is no dbkey owner, default to current user.
dbkey_owner, dbkey = decode_dbkey( dbkey )
if dbkey_owner:
dbkey_user = trans.sa_session.query( trans.app.model.User ).filter_by( username=dbkey_owner ).first()
else:
dbkey_user = trans.user
if not self.has_reference_data( dbkey, dbkey_user ):
return None
#
# Get twobit file with reference data.
#
twobit_file_name = None
if dbkey in self.genomes:
# Built-in twobit.
twobit_file_name = self.genomes[dbkey].twobit_file
else:
user_keys = loads( dbkey_user.preferences['dbkeys'] )
dbkey_attributes = user_keys[ dbkey ]
fasta_dataset = trans.sa_session.query( trans.app.model.HistoryDatasetAssociation ).get( dbkey_attributes[ 'fasta' ] )
msg = fasta_dataset.convert_dataset( trans, 'twobit' )
if msg:
return msg
else:
twobit_dataset = fasta_dataset.get_converted_dataset( trans, 'twobit' )
twobit_file_name = twobit_dataset.file_name
# Read and return reference data.
try:
twobit = TwoBitFile( open( twobit_file_name ) )
if chrom in twobit:
seq_data = twobit[chrom].get( int(low), int(high) )
return GenomeRegion( chrom=chrom, start=low, end=high, sequence=seq_data )
except IOError:
return None
def get_sequencer_id(sequencer_info):
'''Get the sequencer id corresponding to the sequencer information'''
# Check if there is any existing sequencer which have the same sequencer
# information fields & values
cmd = "SELECT sequencer.id, form_values.content FROM sequencer, form_values WHERE sequencer.form_values_id=form_values.id"
result = migrate_engine.execute(cmd)
for row in result:
sequencer_id = row[0]
values = str(row[1])
if not values.strip():
continue
values = loads(values)
# proceed only if sequencer_info is a valid list
if values and type(values) == type(dict()):
if sequencer_info.get( 'host', '' ) == values.get( 'field_0', '' ) \
and sequencer_info.get( 'username', '' ) == values.get( 'field_1', '' ) \
and sequencer_info.get( 'password', '' ) == values.get( 'field_2', '' ) \
and sequencer_info.get( 'data_dir', '' ) == values.get( 'field_3', '' ) \
and sequencer_info.get( 'rename_dataset', '' ) == values.get( 'field_4', '' ):
return sequencer_id
return None
def get_sequencer_id( sequencer_info ):
'''Get the sequencer id corresponding to the sequencer information'''
# Check if there is any existing sequencer which have the same sequencer
# information fields & values
cmd = "SELECT sequencer.id, form_values.content FROM sequencer, form_values WHERE sequencer.form_values_id=form_values.id"
result = migrate_engine.execute( cmd )
for row in result:
sequencer_id = row[0]
values = str( row[1] )
if not values.strip():
continue
values = loads( values )
# proceed only if sequencer_info is a valid list
if values and type( values ) == type( dict() ):
if sequencer_info.get( 'host', '' ) == values.get( 'field_0', '' ) \
and sequencer_info.get( 'username', '' ) == values.get( 'field_1', '' ) \
and sequencer_info.get( 'password', '' ) == values.get( 'field_2', '' ) \
and sequencer_info.get( 'data_dir', '' ) == values.get( 'field_3', '' ) \
and sequencer_info.get( 'rename_dataset', '' ) == values.get( 'field_4', '' ):
return sequencer_id
return None
def check_job(self, job):
if self._missing_params(job.params, ['type']):
return self.job_states.INVALID
if job.params['type'] == 'init_transfer':
if self._missing_params(job.params, ['smrt_host', 'smrt_job_id']):
return self.job_states.INVALID
url = 'http://' + job.params[
'smrt_host'] + self.api_path + '/Jobs/' + job.params[
'smrt_job_id'] + '/Status'
r = urllib2.urlopen(url)
status = json.loads(r.read())
# TODO: error handling: unexpected json or bad response, bad url, etc.
if status['Code'] == 'Completed':
log.debug(
"SMRT Portal job '%s' is Completed. Initiating transfer."
% job.params['smrt_job_id'])
return self.job_states.READY
return self.job_states.WAIT
if job.params['type'] == 'finish_transfer':
if self._missing_params(job.params, ['transfer_job_id']):
return self.job_states.INVALID
# Get the TransferJob object and add it to the DeferredJob so we only look it up once.
if not hasattr(job, 'transfer_job'):
job.transfer_job = self.sa_session.query(
self.app.model.TransferJob).get(
int(job.params['transfer_job_id']))
state = self.app.transfer_manager.get_state(job.transfer_job)
if not state:
log.error('No state for transfer job id: %s' %
job.transfer_job.id)
return self.job_states.WAIT
if state['state'] in self.app.model.TransferJob.terminal_states:
return self.job_states.READY
log.debug("Checked on finish transfer job %s, not done yet." %
job.id)
return self.job_states.WAIT
else:
log.error('Unknown job type for SMRTPortalPlugin: %s' %
str(job.params['type']))
return self.job_states.INVALID
def get_chrom_info( self, dbkey, trans=None, custom_build_hack_get_len_from_fasta_conversion=True ):
# FIXME: flag to turn off custom_build_hack_get_len_from_fasta_conversion should not be required
chrom_info = None
db_dataset = None
# Collect chromInfo from custom builds
if trans:
db_dataset = trans.db_dataset_for( dbkey )
if db_dataset:
chrom_info = db_dataset.file_name
else:
# Do Custom Build handling
if trans.user and ( 'dbkeys' in trans.user.preferences ) and ( dbkey in loads( trans.user.preferences[ 'dbkeys' ] ) ):
custom_build_dict = loads( trans.user.preferences[ 'dbkeys' ] )[ dbkey ]
# HACK: the attempt to get chrom_info below will trigger the
# fasta-to-len converter if the dataset is not available or,
# which will in turn create a recursive loop when
# running the fasta-to-len tool. So, use a hack in the second
# condition below to avoid getting chrom_info when running the
# fasta-to-len converter.
if 'fasta' in custom_build_dict and custom_build_hack_get_len_from_fasta_conversion:
# Build is defined by fasta; get len file, which is obtained from converting fasta.
build_fasta_dataset = trans.sa_session.query( trans.app.model.HistoryDatasetAssociation ).get( custom_build_dict[ 'fasta' ] )
chrom_info = build_fasta_dataset.get_converted_dataset( trans, 'len' ).file_name
elif 'len' in custom_build_dict:
# Build is defined by len file, so use it.
chrom_info = trans.sa_session.query( trans.app.model.HistoryDatasetAssociation ).get( custom_build_dict[ 'len' ] ).file_name
# Check Data table
if not chrom_info:
dbkey_table = self._app.tool_data_tables.get( self._data_table_name, None )
if dbkey_table is not None:
chrom_info = dbkey_table.get_entry( 'value', dbkey, 'len_path', default=None )
# use configured server len path
if not chrom_info:
# Default to built-in build.
# Since we are using an unverified dbkey, we will sanitize the dbkey before use
chrom_info = os.path.join( self._static_chrom_info_path, "%s.len" % sanitize_lists_to_string( dbkey ) )
chrom_info = os.path.abspath( chrom_info )
return ( chrom_info, db_dataset )
def get_dbkeys( self, trans, chrom_info=False, **kwd ):
""" Returns all known dbkeys. If chrom_info is True, only dbkeys with
chromosome lengths are returned. """
dbkeys = []
# Add user's custom keys to dbkeys.
user_keys_dict = {}
user = trans.get_user()
if user:
if 'dbkeys' in user.preferences:
user_keys_dict = loads( user.preferences[ 'dbkeys' ] )
dbkeys.extend( [ (attributes[ 'name' ], key ) for key, attributes in user_keys_dict.items() ] )
# Add app keys to dbkeys.
# If chrom_info is True, only include keys with len files (which contain chromosome info).
filter_fn = lambda b: True
if chrom_info:
filter_fn = lambda b: b.len_file is not None
dbkeys.extend( [ ( genome.description, genome.key ) for key, genome in self.genomes.items() if filter_fn( genome ) ] )
return dbkeys
def __call__( self, trans, **kwargs ):
# Get basics.
# FIXME: pretty sure this is only here to pass along, can likely be eliminated
status = kwargs.get( 'status', None )
message = kwargs.get( 'message', None )
# Build a base filter and sort key that is the combination of the saved state and defaults.
# Saved state takes preference over defaults.
base_filter = {}
if self.default_filter:
# default_filter is a dictionary that provides a default set of filters based on the grid's columns.
base_filter = self.default_filter.copy()
base_sort_key = self.default_sort_key
if self.preserve_state:
pref_name = unicode( self.__class__.__name__ + self.cur_filter_pref_name )
if pref_name in trans.get_user().preferences:
saved_filter = loads( trans.get_user().preferences[pref_name] )
base_filter.update( saved_filter )
pref_name = unicode( self.__class__.__name__ + self.cur_sort_key_pref_name )
if pref_name in trans.get_user().preferences:
base_sort_key = loads( trans.get_user().preferences[pref_name] )
# Build initial query
query = self.build_initial_query( trans, **kwargs )
query = self.apply_query_filter( trans, query, **kwargs )
# Maintain sort state in generated urls
extra_url_args = {}
# Determine whether use_default_filter flag is set.
use_default_filter_str = kwargs.get( 'use_default_filter' )
use_default_filter = False
if use_default_filter_str:
use_default_filter = ( use_default_filter_str.lower() == 'true' )
# Process filtering arguments to (a) build a query that represents the filter and (b) build a
# dictionary that denotes the current filter.
cur_filter_dict = {}
for column in self.columns:
if column.key:
# Get the filter criterion for the column. Precedence is (a) if using default filter, only look there; otherwise,
# (b) look in kwargs; and (c) look in base filter.
column_filter = None
if use_default_filter:
if self.default_filter:
column_filter = self.default_filter.get( column.key )
elif "f-" + column.model_class.__name__ + ".%s" % column.key in kwargs:
# Queries that include table joins cannot guarantee unique column names. This problem is
# handled by setting the column_filter value to <TableName>.<ColumnName>.
column_filter = kwargs.get( "f-" + column.model_class.__name__ + ".%s" % column.key )
elif "f-" + column.key in kwargs:
column_filter = kwargs.get( "f-" + column.key )
elif column.key in base_filter:
column_filter = base_filter.get( column.key )
# Method (1) combines a mix of strings and lists of strings into a single string and (2) attempts to de-jsonify all strings.
def loads_recurse(item):
decoded_list = []
if isinstance( item, basestring):
try:
# Not clear what we're decoding, so recurse to ensure that we catch everything.
decoded_item = loads( item )
if isinstance( decoded_item, list):
decoded_list = loads_recurse( decoded_item )
else:
decoded_list = [ unicode( decoded_item ) ]
except ValueError:
decoded_list = [ unicode( item ) ]
elif isinstance( item, list):
for element in item:
a_list = loads_recurse( element )
decoded_list = decoded_list + a_list
return decoded_list
# If column filter found, apply it.
if column_filter is not None:
# TextColumns may have a mix of json and strings.
if isinstance( column, TextColumn ):
column_filter = loads_recurse( column_filter )
if len( column_filter ) == 1:
column_filter = column_filter[0]
# Interpret ',' as a separator for multiple terms.
if isinstance( column_filter, basestring ) and column_filter.find(',') != -1:
column_filter = column_filter.split(',')
# Check if filter is empty
if isinstance( column_filter, list ):
# Remove empty strings from filter list
column_filter = [x for x in column_filter if x != '']
if len(column_filter) == 0:
continue
elif isinstance(column_filter, basestring):
# If filter criterion is empty, do nothing.
if column_filter == '':
continue
# Update query.
query = column.filter( trans, trans.user, query, column_filter )
# Upate current filter dict.
# Column filters are rendered in various places, sanitize them all here.
cur_filter_dict[ column.key ] = sanitize_text(column_filter)
# Carry filter along to newly generated urls; make sure filter is a string so
# that we can encode to UTF-8 and thus handle user input to filters.
if isinstance( column_filter, list ):
# Filter is a list; process each item.
for filter in column_filter:
if not isinstance( filter, basestring ):
filter = unicode( filter ).encode("utf-8")
extra_url_args[ "f-" + column.key ] = dumps( column_filter )
else:
# Process singleton filter.
if not isinstance( column_filter, basestring ):
column_filter = unicode(column_filter)
extra_url_args[ "f-" + column.key ] = column_filter.encode("utf-8")
# Process sort arguments.
sort_key = None
if 'sort' in kwargs:
sort_key = kwargs['sort']
elif base_sort_key:
sort_key = base_sort_key
if sort_key:
ascending = not( sort_key.startswith( "-" ) )
# Queries that include table joins cannot guarantee unique column names. This problem is
# handled by setting the column_filter value to <TableName>.<ColumnName>.
table_name = None
if sort_key.find( '.' ) > -1:
a_list = sort_key.split( '.' )
if ascending:
table_name = a_list[0]
else:
table_name = a_list[0][1:]
column_name = a_list[1]
elif ascending:
column_name = sort_key
else:
column_name = sort_key[1:]
# Sort key is a column key.
for column in self.columns:
if column.key and column.key.find( '.' ) > -1:
column_key = column.key.split( '.' )[1]
else:
column_key = column.key
if ( table_name is None or table_name == column.model_class.__name__ ) and column_key == column_name:
query = column.sort( trans, query, ascending, column_name=column_name )
break
extra_url_args['sort'] = sort_key
# There might be a current row
current_item = self.get_current_item( trans, **kwargs )
# Process page number.
if self.use_paging:
if 'page' in kwargs:
if kwargs['page'] == 'all':
page_num = 0
else:
page_num = int( kwargs['page'] )
else:
page_num = 1
if page_num == 0:
# Show all rows in page.
total_num_rows = query.count()
page_num = 1
num_pages = 1
else:
# Show a limited number of rows. Before modifying query, get the total number of rows that query
# returns so that the total number of pages can be computed.
total_num_rows = query.count()
query = query.limit( self.num_rows_per_page ).offset( ( page_num - 1 ) * self.num_rows_per_page )
num_pages = int( math.ceil( float( total_num_rows ) / self.num_rows_per_page ) )
else:
# Defaults.
page_num = 1
num_pages = 1
# There are some places in grid templates where it's useful for a grid
# to have its current filter.
self.cur_filter_dict = cur_filter_dict
# Preserve grid state: save current filter and sort key.
if self.preserve_state:
pref_name = unicode( self.__class__.__name__ + self.cur_filter_pref_name )
trans.get_user().preferences[pref_name] = unicode( dumps( cur_filter_dict ) )
if sort_key:
pref_name = unicode( self.__class__.__name__ + self.cur_sort_key_pref_name )
trans.get_user().preferences[pref_name] = unicode( dumps( sort_key ) )
trans.sa_session.flush()
# Log grid view.
context = unicode( self.__class__.__name__ )
params = cur_filter_dict.copy()
params['sort'] = sort_key
params['async'] = ( 'async' in kwargs )
# TODO:??
# commenting this out; when this fn calls session.add( action ) and session.flush the query from this fn
# is effectively 'wiped' out. Nate believes it has something to do with our use of session( autocommit=True )
# in mapping.py. If you change that to False, the log_action doesn't affect the query
# Below, I'm rendering the template first (that uses query), then calling log_action, then returning the page
# trans.log_action( trans.get_user(), unicode( "grid.view" ), context, params )
# Render grid.
def url( *args, **kwargs ):
# Only include sort/filter arguments if not linking to another
# page. This is a bit of a hack.
if 'action' in kwargs:
new_kwargs = dict()
else:
new_kwargs = dict( extra_url_args )
# Extend new_kwargs with first argument if found
if len(args) > 0:
new_kwargs.update( args[0] )
new_kwargs.update( kwargs )
# We need to encode item ids
if 'id' in new_kwargs:
id = new_kwargs[ 'id' ]
if isinstance( id, list ):
new_kwargs[ 'id' ] = [ trans.security.encode_id( i ) for i in id ]
else:
new_kwargs[ 'id' ] = trans.security.encode_id( id )
# The url_for invocation *must* include a controller and action.
if 'controller' not in new_kwargs:
new_kwargs['controller'] = trans.controller
if 'action' not in new_kwargs:
new_kwargs['action'] = trans.action
return url_for( **new_kwargs)
self.use_panels = ( kwargs.get( 'use_panels', False ) in [ True, 'True', 'true' ] )
self.advanced_search = ( kwargs.get( 'advanced_search', False ) in [ True, 'True', 'true' ] )
async_request = ( ( self.use_async ) and ( kwargs.get( 'async', False ) in [ True, 'True', 'true'] ) )
# Currently, filling the template returns a str object; this requires decoding the string into a
# unicode object within mako templates. What probably should be done is to return the template as
# utf-8 unicode; however, this would require encoding the object as utf-8 before returning the grid
# results via a controller method, which is require substantial changes. Hence, for now, return grid
# as str.
page = trans.fill_template( iff( async_request, self.async_template, self.template ),
grid=self,
query=query,
cur_page_num=page_num,
num_pages=num_pages,
num_page_links=self.num_page_links,
default_filter_dict=self.default_filter,
cur_filter_dict=cur_filter_dict,
sort_key=sort_key,
current_item=current_item,
ids=kwargs.get( 'id', [] ),
url=url,
status=status,
message=message,
info_text=self.info_text,
use_panels=self.use_panels,
use_hide_message=self.use_hide_message,
advanced_search=self.advanced_search,
show_item_checkboxes=( self.show_item_checkboxes or
kwargs.get( 'show_item_checkboxes', '' ) in [ 'True', 'true' ] ),
# Pass back kwargs so that grid template can set and use args without
# grid explicitly having to pass them.
kwargs=kwargs )
trans.log_action( trans.get_user(), unicode( "grid.view" ), context, params )
return page
class DataManager(BaseUIController):
@web.expose
def index(self, trans, **kwd):
not_is_admin = not trans.user_is_admin()
if not_is_admin and not trans.app.config.enable_data_manager_user_view:
raise paste.httpexceptions.HTTPUnauthorized(
"This Galaxy instance is not configured to allow non-admins to view the data manager."
)
message = escape(kwd.get('message', ''))
status = escape(kwd.get('status', 'info'))
return trans.fill_template("data_manager/index.mako",
data_managers=trans.app.data_managers,
tool_data_tables=trans.app.tool_data_tables,
view_only=not_is_admin,
message=message,
status=status)
@web.expose
def manage_data_manager(self, trans, **kwd):
not_is_admin = not trans.user_is_admin()
if not_is_admin and not trans.app.config.enable_data_manager_user_view:
raise paste.httpexceptions.HTTPUnauthorized(
"This Galaxy instance is not configured to allow non-admins to view the data manager."
)
message = escape(kwd.get('message', ''))
status = escape(kwd.get('status', 'info'))
data_manager_id = kwd.get('id', None)
data_manager = trans.app.data_managers.get_manager(data_manager_id)
if data_manager is None:
return trans.response.send_redirect(
web.url_for(controller="data_manager",
action="index",
message="Invalid Data Manager (%s) was requested" %
data_manager_id,
status="error"))
jobs = list(
reversed([
assoc.job for assoc in trans.sa_session.query(
trans.app.model.DataManagerJobAssociation).filter_by(
data_manager_id=data_manager_id)
]))
return trans.fill_template("data_manager/manage_data_manager.mako",
data_manager=data_manager,
jobs=jobs,
view_only=not_is_admin,
message=message,
status=status)
@web.expose
def view_job(self, trans, **kwd):
not_is_admin = not trans.user_is_admin()
if not_is_admin and not trans.app.config.enable_data_manager_user_view:
raise paste.httpexceptions.HTTPUnauthorized(
"This Galaxy instance is not configured to allow non-admins to view the data manager."
)
message = escape(kwd.get('message', ''))
status = escape(kwd.get('status', 'info'))
job_id = kwd.get('id', None)
try:
job_id = trans.security.decode_id(job_id)
job = trans.sa_session.query(trans.app.model.Job).get(job_id)
except Exception, e:
job = None
log.error("Bad job id (%s) passed to view_job: %s" % (job_id, e))
if not job:
return trans.response.send_redirect(
web.url_for(controller="data_manager",
action="index",
message="Invalid job (%s) was requested" % job_id,
status="error"))
data_manager_id = job.data_manager_association.data_manager_id
data_manager = trans.app.data_managers.get_manager(data_manager_id)
hdas = [assoc.dataset for assoc in job.get_output_datasets()]
data_manager_output = []
error_messages = []
for hda in hdas:
try:
data_manager_json = loads(open(hda.get_file_name()).read())
except Exception, e:
data_manager_json = {}
error_messages.append(
escape(
"Unable to obtain data_table info for hda (%s): %s" %
(hda.id, e)))
values = []
for key, value in data_manager_json.get('data_tables',
{}).iteritems():
values.append((key, value))
data_manager_output.append(values)
def from_dict( Class, trans, d, secure=True ):
module = Class( trans )
state = loads( d["tool_state"] )
module.recover_state( state )
return module
def pretty_print_json(json_data, is_json_string=False):
if is_json_string:
json_data = json.loads(json_data)
return json.dumps(json_data, sort_keys=True, indent=4)
def cleanup_after_job(self):
""" Set history, datasets, and jobs' attributes and clean up archive directory. """
#
# Helper methods.
#
def file_in_dir(file_path, a_dir):
""" Returns true if file is in directory. """
abs_file_path = os.path.abspath(file_path)
return os.path.split(abs_file_path)[0] == a_dir
def read_file_contents(file_path):
""" Read contents of a file. """
fp = open(file_path, 'rb')
buffsize = 1048576
file_contents = ''
try:
while True:
file_contents += fp.read(buffsize)
if not file_contents or len(file_contents) % buffsize != 0:
break
except OverflowError:
pass
fp.close()
return file_contents
def get_tag_str(tag, value):
""" Builds a tag string for a tag, value pair. """
if not value:
return tag
else:
return tag + ":" + value
#
# Import history.
#
jiha = self.sa_session.query(model.JobImportHistoryArchive).filter_by(
job_id=self.job_id).first()
if jiha:
try:
archive_dir = jiha.archive_dir
user = jiha.job.user
#
# Create history.
#
history_attr_file_name = os.path.join(archive_dir,
'history_attrs.txt')
history_attr_str = read_file_contents(history_attr_file_name)
history_attrs = loads(history_attr_str)
# Create history.
new_history = model.History(
name='imported from archive: %s' %
history_attrs['name'].encode('utf-8'),
user=user)
new_history.importing = True
new_history.hid_counter = history_attrs['hid_counter']
new_history.genome_build = history_attrs['genome_build']
self.sa_session.add(new_history)
jiha.history = new_history
self.sa_session.flush()
# Add annotation, tags.
if user:
self.add_item_annotation(self.sa_session, user,
new_history,
history_attrs['annotation'])
"""
TODO: figure out to how add tags to item.
for tag, value in history_attrs[ 'tags' ].items():
trans.app.tag_handler.apply_item_tags( trans, trans.user, new_history, get_tag_str( tag, value ) )
"""
#
# Create datasets.
#
datasets_attrs_file_name = os.path.join(
archive_dir, 'datasets_attrs.txt')
datasets_attr_str = read_file_contents(
datasets_attrs_file_name)
datasets_attrs = loads(datasets_attr_str)
if os.path.exists(datasets_attrs_file_name + ".provenance"):
provenance_attr_str = read_file_contents(
datasets_attrs_file_name + ".provenance")
provenance_attrs = loads(provenance_attr_str)
datasets_attrs += provenance_attrs
# Get counts of how often each dataset file is used; a file can
# be linked to multiple dataset objects (HDAs).
datasets_usage_counts = {}
for dataset_attrs in datasets_attrs:
temp_dataset_file_name = \
os.path.abspath( os.path.join( archive_dir, dataset_attrs['file_name'] ) )
if (temp_dataset_file_name not in datasets_usage_counts):
datasets_usage_counts[temp_dataset_file_name] = 0
datasets_usage_counts[temp_dataset_file_name] += 1
# Create datasets.
for dataset_attrs in datasets_attrs:
metadata = dataset_attrs['metadata']
# Create dataset and HDA.
hda = model.HistoryDatasetAssociation(
name=dataset_attrs['name'].encode('utf-8'),
extension=dataset_attrs['extension'],
info=dataset_attrs['info'].encode('utf-8'),
blurb=dataset_attrs['blurb'],
peek=dataset_attrs['peek'],
designation=dataset_attrs['designation'],
visible=dataset_attrs['visible'],
dbkey=metadata['dbkey'],
metadata=metadata,
history=new_history,
create_dataset=True,
sa_session=self.sa_session)
if 'uuid' in dataset_attrs:
hda.dataset.uuid = dataset_attrs["uuid"]
if dataset_attrs.get('exported', True) is False:
hda.state = hda.states.DISCARDED
hda.deleted = True
hda.purged = True
else:
hda.state = hda.states.OK
self.sa_session.add(hda)
self.sa_session.flush()
new_history.add_dataset(hda, genome_build=None)
hda.hid = dataset_attrs[
'hid'] # Overwrite default hid set when HDA added to history.
# TODO: Is there a way to recover permissions? Is this needed?
# permissions = trans.app.security_agent.history_get_default_permissions( new_history )
# trans.app.security_agent.set_all_dataset_permissions( hda.dataset, permissions )
self.sa_session.flush()
if dataset_attrs.get('exported', True) is True:
# Do security check and move/copy dataset data.
temp_dataset_file_name = \
os.path.abspath( os.path.join( archive_dir, dataset_attrs['file_name'] ) )
if not file_in_dir(
temp_dataset_file_name,
os.path.join(archive_dir, "datasets")):
raise Exception("Invalid dataset path: %s" %
temp_dataset_file_name)
if datasets_usage_counts[temp_dataset_file_name] == 1:
self.app.object_store.update_from_file(
hda.dataset,
file_name=temp_dataset_file_name,
create=True)
# Import additional files if present. Histories exported previously might not have this attribute set.
dataset_extra_files_path = dataset_attrs.get(
'extra_files_path', None)
if dataset_extra_files_path:
try:
file_list = os.listdir(
os.path.join(archive_dir,
dataset_extra_files_path))
except OSError:
file_list = []
if file_list:
for extra_file in file_list:
self.app.object_store.update_from_file(
hda.dataset,
extra_dir='dataset_%s_files' %
hda.dataset.id,
alt_name=extra_file,
file_name=os.path.join(
archive_dir,
dataset_extra_files_path,
extra_file),
create=True)
else:
datasets_usage_counts[temp_dataset_file_name] -= 1
shutil.copyfile(temp_dataset_file_name,
hda.file_name)
hda.dataset.set_total_size(
) # update the filesize record in the database
# Set tags, annotations.
if user:
self.add_item_annotation(self.sa_session, user, hda,
dataset_attrs['annotation'])
# TODO: Set tags.
"""
for tag, value in dataset_attrs[ 'tags' ].items():
trans.app.tag_handler.apply_item_tags( trans, trans.user, hda, get_tag_str( tag, value ) )
self.sa_session.flush()
"""
# Although metadata is set above, need to set metadata to recover BAI for BAMs.
if hda.extension == 'bam':
self.app.datatypes_registry.set_external_metadata_tool.tool_action.execute_via_app(
self.app.datatypes_registry.
set_external_metadata_tool,
self.app,
jiha.job.session_id,
new_history.id,
jiha.job.user,
incoming={'input1': hda},
overwrite=False)
#
# Create jobs.
#
# Read jobs attributes.
jobs_attr_file_name = os.path.join(archive_dir,
'jobs_attrs.txt')
jobs_attr_str = read_file_contents(jobs_attr_file_name)
# Decode jobs attributes.
def as_hda(obj_dct):
""" Hook to 'decode' an HDA; method uses history and HID to get the HDA represented by
the encoded object. This only works because HDAs are created above. """
if obj_dct.get('__HistoryDatasetAssociation__', False):
return self.sa_session.query(
model.HistoryDatasetAssociation).filter_by(
history=new_history,
hid=obj_dct['hid']).first()
return obj_dct
jobs_attrs = loads(jobs_attr_str, object_hook=as_hda)
# Create each job.
for job_attrs in jobs_attrs:
imported_job = model.Job()
imported_job.user = user
# TODO: set session?
# imported_job.session = trans.get_galaxy_session().id
imported_job.history = new_history
imported_job.imported = True
imported_job.tool_id = job_attrs['tool_id']
imported_job.tool_version = job_attrs['tool_version']
imported_job.set_state(job_attrs['state'])
imported_job.info = job_attrs.get('info', None)
imported_job.exit_code = job_attrs.get('exit_code', None)
imported_job.traceback = job_attrs.get('traceback', None)
imported_job.stdout = job_attrs.get('stdout', None)
imported_job.stderr = job_attrs.get('stderr', None)
imported_job.command_line = job_attrs.get(
'command_line', None)
try:
imported_job.create_time = datetime.datetime.strptime(
job_attrs["create_time"], "%Y-%m-%dT%H:%M:%S.%f")
imported_job.update_time = datetime.datetime.strptime(
job_attrs["update_time"], "%Y-%m-%dT%H:%M:%S.%f")
except:
pass
self.sa_session.add(imported_job)
self.sa_session.flush()
class HistoryDatasetAssociationIDEncoder(json.JSONEncoder):
""" Custom JSONEncoder for a HistoryDatasetAssociation that encodes an HDA as its ID. """
def default(self, obj):
""" Encode an HDA, default encoding for everything else. """
if isinstance(obj,
model.HistoryDatasetAssociation):
return obj.id
return json.JSONEncoder.default(self, obj)
# Set parameters. May be useful to look at metadata.py for creating parameters.
# TODO: there may be a better way to set parameters, e.g.:
# for name, value in tool.params_to_strings( incoming, trans.app ).iteritems():
# job.add_parameter( name, value )
# to make this work, we'd need to flesh out the HDA objects. The code below is
# relatively similar.
for name, value in job_attrs['params'].items():
# Transform parameter values when necessary.
if isinstance(value, model.HistoryDatasetAssociation):
# HDA input: use hid to find input.
input_hda = self.sa_session.query( model.HistoryDatasetAssociation ) \
.filter_by( history=new_history, hid=value.hid ).first()
value = input_hda.id
# print "added parameter %s-->%s to job %i" % ( name, value, imported_job.id )
imported_job.add_parameter(
name,
dumps(value,
cls=HistoryDatasetAssociationIDEncoder))
# TODO: Connect jobs to input datasets.
# Connect jobs to output datasets.
for output_hid in job_attrs['output_datasets']:
# print "%s job has output dataset %i" % (imported_job.id, output_hid)
output_hda = self.sa_session.query(
model.HistoryDatasetAssociation).filter_by(
history=new_history, hid=output_hid).first()
if output_hda:
imported_job.add_output_dataset(
output_hda.name, output_hda)
# Connect jobs to input datasets.
if 'input_mapping' in job_attrs:
for input_name, input_hid in job_attrs[
'input_mapping'].items():
input_hda = self.sa_session.query( model.HistoryDatasetAssociation ) \
.filter_by( history=new_history, hid=input_hid ).first()
if input_hda:
imported_job.add_input_dataset(
input_name, input_hda)
self.sa_session.flush()
# Done importing.
new_history.importing = False
self.sa_session.flush()
# Cleanup.
if os.path.exists(archive_dir):
shutil.rmtree(archive_dir)
except Exception, e:
jiha.job.stderr += "Error cleaning up history import job: %s" % e
self.sa_session.flush()
def pretty_print_json(json_data, is_json_string=False):
if is_json_string:
json_data = json.loads(json_data)
return json.dumps(json_data, sort_keys=True, indent=4)
assert col is Request_table.c.notification
except Exception, e:
log.debug( "Creating column 'notification' in the 'request' table failed: %s" % ( str( e ) ) )
cmd = "SELECT id, user_id, notify FROM request"
result = migrate_engine.execute( cmd )
for r in result:
id = int(r[0])
notify_old = r[1]
notify_new = dict(email=[], sample_states=[], body='', subject='')
cmd = "update request set notification='%s' where id=%i" % (dumps(notify_new), id)
migrate_engine.execute( cmd )
cmd = "SELECT id, notification FROM request"
result = migrate_engine.execute( cmd )
for r in result:
rr = loads(str(r[1]))
# remove the 'notify' column for non-sqlite databases.
if migrate_engine.name != 'sqlite':
try:
Request_table.c.notify.drop()
except Exception, e:
log.debug( "Deleting column 'notify' from the 'request' table failed: %s" % ( str( e ) ) )
def downgrade(migrate_engine):
metadata.bind = migrate_engine
pass
