def get_chrom_info(self,
dbkey,
trans=None,
custom_build_hack_get_len_from_fasta_conversion=True):
# FIXME: flag to turn off custom_build_hack_get_len_from_fasta_conversion should not be required
chrom_info = None
db_dataset = None
# Collect chromInfo from custom builds
if trans:
db_dataset = trans.db_dataset_for(dbkey)
if db_dataset:
chrom_info = db_dataset.file_name
else:
# Do Custom Build handling
if trans.user and ('dbkeys' in trans.user.preferences) and (
dbkey in from_json_string(
trans.user.preferences['dbkeys'])):
custom_build_dict = from_json_string(
trans.user.preferences['dbkeys'])[dbkey]
# HACK: the attempt to get chrom_info below will trigger the
# fasta-to-len converter if the dataset is not available or,
# which will in turn create a recursive loop when
# running the fasta-to-len tool. So, use a hack in the second
# condition below to avoid getting chrom_info when running the
# fasta-to-len converter.
if 'fasta' in custom_build_dict and custom_build_hack_get_len_from_fasta_conversion:
# Build is defined by fasta; get len file, which is obtained from converting fasta.
build_fasta_dataset = trans.sa_session.query(
trans.app.model.HistoryDatasetAssociation).get(
custom_build_dict['fasta'])
chrom_info = build_fasta_dataset.get_converted_dataset(
trans, 'len').file_name
elif 'len' in custom_build_dict:
# Build is defined by len file, so use it.
chrom_info = trans.sa_session.query(
trans.app.model.HistoryDatasetAssociation).get(
custom_build_dict['len']).file_name
# Check Data table
if not chrom_info:
dbkey_table = self._app.tool_data_tables.get(
self._data_table_name, None)
if dbkey_table is not None:
chrom_info = dbkey_table.get_entry('value',
dbkey,
'len_path',
default=None)
# use configured server len path
if not chrom_info:
# Default to built-in build.
chrom_info = os.path.join(self._static_chrom_info_path,
"%s.len" % dbkey)
chrom_info = os.path.abspath(chrom_info)
return (chrom_info, db_dataset)
def chroms( self, trans, dbkey=None, num=None, chrom=None, low=None ):
"""
Returns a naturally sorted list of chroms/contigs for a given dbkey.
Use either chrom or low to specify the starting chrom in the return list.
"""
# If there is no dbkey owner, default to current user.
dbkey_owner, dbkey = decode_dbkey( dbkey )
if dbkey_owner:
dbkey_user = trans.sa_session.query( trans.app.model.User ).filter_by( username=dbkey_owner ).first()
else:
dbkey_user = trans.user
#
# Get/create genome object.
#
genome = None
twobit_file = None
# Look first in user's custom builds.
if dbkey_user and 'dbkeys' in dbkey_user.preferences:
user_keys = from_json_string( dbkey_user.preferences['dbkeys'] )
if dbkey in user_keys:
dbkey_attributes = user_keys[ dbkey ]
dbkey_name = dbkey_attributes[ 'name' ]
# If there's a fasta for genome, convert to 2bit for later use.
if 'fasta' in dbkey_attributes:
build_fasta = trans.sa_session.query( trans.app.model.HistoryDatasetAssociation ).get( dbkey_attributes[ 'fasta' ] )
len_file = build_fasta.get_converted_dataset( trans, 'len' ).file_name
build_fasta.get_converted_dataset( trans, 'twobit' )
# HACK: set twobit_file to True rather than a file name because
# get_converted_dataset returns null during conversion even though
# there will eventually be a twobit file available for genome.
twobit_file = True
# Backwards compatibility: look for len file directly.
elif 'len' in dbkey_attributes:
len_file = trans.sa_session.query( trans.app.model.HistoryDatasetAssociation ).get( user_keys[ dbkey ][ 'len' ] ).file_name
if len_file:
genome = Genome( dbkey, dbkey_name, len_file=len_file, twobit_file=twobit_file )
# Look in history and system builds.
if not genome:
# Look in history for chromosome len file.
len_ds = trans.db_dataset_for( dbkey )
if len_ds:
genome = Genome( dbkey, dbkey_name, len_file=len_ds.file_name )
# Look in system builds.
elif dbkey in self.genomes:
genome = self.genomes[ dbkey ]
# Set up return value or log exception if genome not found for key.
rval = None
if genome:
rval = genome.to_dict( num=num, chrom=chrom, low=low )
else:
log.exception( 'genome not found for key %s' % dbkey )
return rval
def from_workflow_step(Class, trans, step):
tool_id = step.tool_id
if trans.app.toolbox and tool_id not in trans.app.toolbox.tools_by_id:
# See if we have access to a different version of the tool.
# TODO: If workflows are ever enhanced to use tool version
# in addition to tool id, enhance the selection process here
# to retrieve the correct version of the tool.
tool = trans.app.toolbox.get_tool(tool_id)
if tool:
tool_id = tool.id
if (trans.app.toolbox and tool_id in trans.app.toolbox.tools_by_id):
if step.config:
# This step has its state saved in the config field due to the
# tool being previously unavailable.
return module_factory.from_dict(trans,
from_json_string(step.config),
secure=False)
module = Class(trans, tool_id)
module.state = galaxy.tools.DefaultToolState()
if step.tool_version and (step.tool_version !=
module.tool.version):
module.version_changes.append(
"%s: using version '%s' instead of version '%s' indicated in this workflow."
% (tool_id, module.tool.version, step.tool_version))
module.state.inputs = module.tool.params_from_strings(
step.tool_inputs, trans.app, ignore_errors=True)
module.errors = step.tool_errors
# module.post_job_actions = step.post_job_actions
module.workflow_outputs = step.workflow_outputs
pjadict = {}
for pja in step.post_job_actions:
pjadict[pja.action_type] = pja
module.post_job_actions = pjadict
return module
return None
def import_workflows( self, trans, **kwd ):
"""
POST /api/tool_shed_repositories/import_workflow
Import all of the exported workflows contained in the specified installed tool shed repository into Galaxy.
:param key: the API key of the Galaxy user with which the imported workflows will be associated.
:param id: the encoded id of the ToolShedRepository object
"""
api_key = kwd.get( 'key', None )
if api_key is None:
raise HTTPBadRequest( detail="Missing required parameter 'key' whose value is the API key for the Galaxy user importing the specified workflow." )
tool_shed_repository_id = kwd.get( 'id', '' )
if not tool_shed_repository_id:
raise HTTPBadRequest( detail="Missing required parameter 'id'." )
repository = suc.get_tool_shed_repository_by_id( trans, tool_shed_repository_id )
exported_workflows = json.from_json_string( self.exported_workflows( trans, tool_shed_repository_id ) )
imported_workflow_dicts = []
for exported_workflow_dict in exported_workflows:
workflow_name = exported_workflow_dict[ 'workflow_name' ]
workflow, status, message = workflow_util.import_workflow( trans, repository, workflow_name )
if status == 'error':
log.error( message, exc_info=True )
trans.response.status = 500
return message
else:
imported_workflow_dicts.append( workflow.to_dict( view='element' ) )
return imported_workflow_dicts
def set_tool_versions(self, trans, **kwd):
# Get the tool_versions from the tool shed for each tool in the installed change set.
repository = get_repository(trans, kwd['id'])
tool_shed_url = get_url_from_repository_tool_shed(
trans.app, repository)
url = '%s/repository/get_tool_versions?name=%s&owner=%s&changeset_revision=%s&webapp=galaxy&no_reset=true' % \
( tool_shed_url, repository.name, repository.owner, repository.changeset_revision )
response = urllib2.urlopen(url)
text = response.read()
response.close()
if text:
tool_version_dicts = from_json_string(text)
handle_tool_versions(trans.app, tool_version_dicts, repository)
message = "Tool versions have been set for all included tools."
status = 'done'
else:
message = "Version information for the tools included in the <b>%s</b> repository is missing. " % repository.name
message += "Reset all of this reppository's metadata in the tool shed, then set the installed tool versions "
message + + "from the installed repository's <b>Repository Actions</b> menu. "
status = 'error'
shed_tool_conf, tool_path, relative_install_dir = get_tool_panel_config_tool_path_install_dir(
trans.app, repository)
repo_files_dir = os.path.abspath(
os.path.join(relative_install_dir, repository.name))
return trans.fill_template(
'/admin/tool_shed_repository/manage_repository.mako',
repository=repository,
description=repository.description,
repo_files_dir=repo_files_dir,
message=message,
status=status)
def main():
#Parse Command Line
parser = optparse.OptionParser()
parser.add_option('-j',
'--jar_path',
dest='jar_path',
action='store',
type="string",
default=None,
help='snpEff.jar path')
(options, args) = parser.parse_args()
filename = args[0]
params = from_json_string(open(filename).read())
target_directory = params['output_data'][0]['extra_files_path']
os.mkdir(target_directory)
data_manager_dict = {}
#Create Defuse Reference Data
data_manager_dict = fetch_databases(data_manager_dict, target_directory,
options.jar_path)
#save info to json file
open(filename, 'wb').write(to_json_string(data_manager_dict))
def import_workflows(self, trans, **kwd):
"""
POST /api/tool_shed_repositories/import_workflows
Import all of the exported workflows contained in the specified installed tool shed repository into Galaxy.
:param key: the API key of the Galaxy user with which the imported workflows will be associated.
:param id: the encoded id of the ToolShedRepository object
"""
api_key = kwd.get('key', None)
if api_key is None:
raise HTTPBadRequest(
detail=
"Missing required parameter 'key' whose value is the API key for the Galaxy user importing the specified workflow."
)
tool_shed_repository_id = kwd.get('id', '')
if not tool_shed_repository_id:
raise HTTPBadRequest(detail="Missing required parameter 'id'.")
repository = suc.get_tool_shed_repository_by_id(
trans.app, tool_shed_repository_id)
exported_workflows = json.from_json_string(
self.exported_workflows(trans, tool_shed_repository_id))
imported_workflow_dicts = []
for exported_workflow_dict in exported_workflows:
workflow_name = exported_workflow_dict['workflow_name']
workflow, status, error_message = workflow_util.import_workflow(
trans, repository, workflow_name)
if status == 'error':
log.debug(error_message)
else:
imported_workflow_dicts.append(
workflow.to_dict(view='element'))
return imported_workflow_dicts
def get_genome_build_names(self, trans=None):
# FIXME: how to deal with key duplicates?
rval = []
# load user custom genome builds
if trans is not None:
if trans.history:
# This is a little bit Odd. We are adding every .len file in the current history to dbkey list,
# but this is previous behavior from trans.db_names, so we'll continue to do it.
# It does allow one-off, history specific dbkeys to be created by a user. But we are not filtering,
# so a len file will be listed twice (as the build name and again as dataset name),
# if custom dbkey creation/conversion occurred within the current history.
datasets = trans.sa_session.query( self._app.model.HistoryDatasetAssociation ) \
.filter_by( deleted=False, history_id=trans.history.id, extension="len" )
for dataset in datasets:
rval.append(
(dataset.dbkey,
"%s (%s) [History]" % (dataset.name, dataset.dbkey)))
user = trans.get_user()
if user and 'dbkeys' in user.preferences:
user_keys = from_json_string(user.preferences['dbkeys'])
for key, chrom_dict in user_keys.iteritems():
rval.append(
(key, "%s (%s) [Custom]" % (chrom_dict['name'], key)))
# Load old builds.txt static keys
rval.extend(self._static_dbkeys)
#load dbkeys from dbkey data table
dbkey_table = self._app.tool_data_tables.get(self._data_table_name,
None)
if dbkey_table is not None:
for field_dict in dbkey_table.get_named_fields_list():
rval.append((field_dict['value'], field_dict['name']))
return rval
def main():
#Parse Command Line
parser = optparse.OptionParser()
parser.add_option('-d',
'--dbkey_description',
dest='dbkey_description',
action='store',
type="string",
default=None,
help='dbkey_description')
(options, args) = parser.parse_args()
filename = args[0]
params = from_json_string(open(filename).read())
target_directory = params['output_data'][0]['extra_files_path']
os.mkdir(target_directory)
data_manager_dict = {}
database_id = params['param_dict']['database_id']
database_name = params['param_dict']['database_name']
#Fetch the FASTA
REFERENCE_SOURCE_TO_DOWNLOAD[
params['param_dict']['reference_source']['reference_source_selector']](
data_manager_dict, params, target_directory, database_id,
database_name)
#save info to json file
open(filename, 'wb').write(to_json_string(data_manager_dict))
def check_job( self, job ):
if self._missing_params( job.params, [ 'type' ] ):
return self.job_states.INVALID
if job.params[ 'type' ] == 'init_transfer':
if self._missing_params( job.params, [ 'smrt_host', 'smrt_job_id' ] ):
return self.job_states.INVALID
url = 'http://' + job.params[ 'smrt_host' ] + self.api_path + '/Jobs/' + job.params[ 'smrt_job_id' ] + '/Status'
r = urllib2.urlopen( url )
status = json.from_json_string( r.read() )
# TODO: error handling: unexpected json or bad response, bad url, etc.
if status[ 'Code' ] == 'Completed':
log.debug( "SMRT Portal job '%s' is Completed. Initiating transfer." % job.params[ 'smrt_job_id' ] )
return self.job_states.READY
return self.job_states.WAIT
if job.params[ 'type' ] == 'finish_transfer':
if self._missing_params( job.params, [ 'transfer_job_id' ] ):
return self.job_states.INVALID
# Get the TransferJob object and add it to the DeferredJob so we only look it up once.
if not hasattr( job, 'transfer_job' ):
job.transfer_job = self.sa_session.query( self.app.model.TransferJob ).get( int( job.params[ 'transfer_job_id' ] ) )
state = self.app.transfer_manager.get_state( job.transfer_job )
if not state:
log.error( 'No state for transfer job id: %s' % job.transfer_job.id )
return self.job_states.WAIT
if state[ 'state' ] in self.app.model.TransferJob.terminal_states:
return self.job_states.READY
log.debug( "Checked on finish transfer job %s, not done yet." % job.id )
return self.job_states.WAIT
else:
log.error( 'Unknown job type for SMRTPortalPlugin: %s' % str( job.params[ 'type' ] ) )
return self.job_states.INVALID
def get_tool_provided_job_metadata( self ):
if self.tool_provided_job_metadata is not None:
return self.tool_provided_job_metadata
# Look for JSONified job metadata
self.tool_provided_job_metadata = []
meta_file = os.path.join( self.working_directory, TOOL_PROVIDED_JOB_METADATA_FILE )
if os.path.exists( meta_file ):
for line in open( meta_file, 'r' ):
try:
line = from_json_string( line )
assert 'type' in line
except:
log.exception( '(%s) Got JSON data from tool, but data is improperly formatted or no "type" key in data' % self.job_id )
log.debug( 'Offending data was: %s' % line )
continue
# Set the dataset id if it's a dataset entry and isn't set.
# This isn't insecure. We loop the job's output datasets in
# the finish method, so if a tool writes out metadata for a
# dataset id that it doesn't own, it'll just be ignored.
if line['type'] == 'dataset' and 'dataset_id' not in line:
try:
line['dataset_id'] = self.get_output_file_id( line['dataset'] )
except KeyError:
log.warning( '(%s) Tool provided job dataset-specific metadata without specifying a dataset' % self.job_id )
continue
self.tool_provided_job_metadata.append( line )
return self.tool_provided_job_metadata
def __init__( self, job, queue ):
self.job_id = job.id
self.session_id = job.session_id
self.user_id = job.user_id
self.tool = queue.app.toolbox.tools_by_id.get( job.tool_id, None )
self.queue = queue
self.app = queue.app
self.sa_session = self.app.model.context
self.extra_filenames = []
self.command_line = None
# Tool versioning variables
self.version_string_cmd = None
self.version_string = ""
self.galaxy_lib_dir = None
# With job outputs in the working directory, we need the working
# directory to be set before prepare is run, or else premature deletion
# and job recovery fail.
# Create the working dir if necessary
try:
self.app.object_store.create(job, base_dir='job_work', dir_only=True, extra_dir=str(self.job_id))
self.working_directory = self.app.object_store.get_filename(job, base_dir='job_work', dir_only=True, extra_dir=str(self.job_id))
log.debug('(%s) Working directory for job is: %s' % (self.job_id, self.working_directory))
except ObjectInvalid:
raise Exception('Unable to create job working directory, job failure')
self.output_paths = None
self.output_hdas_and_paths = None
self.tool_provided_job_metadata = None
# Wrapper holding the info required to restore and clean up from files used for setting metadata externally
self.external_output_metadata = metadata.JobExternalOutputMetadataWrapper( job )
self.params = None
if job.params:
self.params = from_json_string( job.params )
self.__user_system_pwent = None
self.__galaxy_system_pwent = None
def main():
#Parse Command Line
parser = optparse.OptionParser()
parser.add_option( '-f', '--fasta_filename', dest='fasta_filename', action='store', type="string", default=None, help='fasta_filename' )
parser.add_option( '-d', '--fasta_dbkey', dest='fasta_dbkey', action='store', type="string", default=None, help='fasta_dbkey' )
parser.add_option( '-t', '--fasta_description', dest='fasta_description', action='store', type="string", default=None, help='fasta_description' )
parser.add_option( '-n', '--data_table_name', dest='data_table_name', action='store', type="string", default=None, help='data_table_name' )
parser.add_option( '-c', '--color_space', dest='color_space', action='store_true', default=False, help='color_space' )
(options, args) = parser.parse_args()
filename = args[0]
params = from_json_string( open( filename ).read() )
target_directory = params[ 'output_data' ][0]['extra_files_path']
os.mkdir( target_directory )
data_manager_dict = {}
dbkey = options.fasta_dbkey
if dbkey in [ None, '', '?' ]:
raise Exception( '"%s" is not a valid dbkey. You must specify a valid dbkey.' % ( dbkey ) )
sequence_id, sequence_name = get_id_name( params, dbkey=dbkey, fasta_description=options.fasta_description )
#build the index
build_bwa_index( data_manager_dict, options.fasta_filename, params, target_directory, dbkey, sequence_id, sequence_name, data_table_name=options.data_table_name or DEFAULT_DATA_TABLE_NAME, color_space=options.color_space )
#save info to json file
open( filename, 'wb' ).write( to_json_string( data_manager_dict ) )
def upgrade(migrate_engine):
metadata.bind = migrate_engine
print __doc__
metadata.reflect()
Visualization_table = Table( "visualization", metadata, autoload=True )
Visualization_revision_table = Table( "visualization_revision", metadata, autoload=True )
# Create dbkey columns.
x = Column( "dbkey", TEXT )
y = Column( "dbkey", TEXT )
x.create( Visualization_table )
y.create( Visualization_revision_table )
# Manually create indexes for compatability w/ mysql_length.
xi = Index( "ix_visualization_dbkey", Visualization_table.c.dbkey, mysql_length = 200)
xi.create()
yi = Index( "ix_visualization_revision_dbkey", Visualization_revision_table.c.dbkey, mysql_length = 200)
yi.create()
assert x is Visualization_table.c.dbkey
assert y is Visualization_revision_table.c.dbkey
all_viz = migrate_engine.execute( "SELECT visualization.id as viz_id, visualization_revision.id as viz_rev_id, visualization_revision.config FROM visualization_revision \
LEFT JOIN visualization ON visualization.id=visualization_revision.visualization_id" )
for viz in all_viz:
viz_id = viz['viz_id']
viz_rev_id = viz['viz_rev_id']
if viz[Visualization_revision_table.c.config]:
dbkey = from_json_string(viz[Visualization_revision_table.c.config]).get('dbkey', "").replace("'", "\\'")
migrate_engine.execute("UPDATE visualization_revision SET dbkey='%s' WHERE id=%s" % (dbkey, viz_rev_id))
migrate_engine.execute("UPDATE visualization SET dbkey='%s' WHERE id=%s" % (dbkey, viz_id))
def sweepster( self, trans, id=None, hda_ldda=None, dataset_id=None, regions=None ):
"""
Displays a sweepster visualization using the incoming parameters. If id is available,
get the visualization with the given id; otherwise, create a new visualization using
a given dataset and regions.
"""
regions = regions or '{}'
# Need to create history if necessary in order to create tool form.
trans.get_history( create=True )
if id:
# Loading a shared visualization.
viz = self.get_visualization( trans, id )
viz_config = self.get_visualization_config( trans, viz )
dataset = self.get_dataset( trans, viz_config[ 'dataset_id' ] )
else:
# Loading new visualization.
dataset = self.get_hda_or_ldda( trans, hda_ldda, dataset_id )
job = self.get_hda_job( dataset )
viz_config = {
'dataset_id': dataset_id,
'tool_id': job.tool_id,
'regions': from_json_string( regions )
}
# Add tool, dataset attributes to config based on id.
tool = trans.app.toolbox.get_tool( viz_config[ 'tool_id' ] )
viz_config[ 'tool' ] = tool.to_dict( trans, io_details=True )
viz_config[ 'dataset' ] = trans.security.encode_dict_ids( dataset.to_dict() )
return trans.fill_template_mako( "visualization/sweepster.mako", config=viz_config )
def get_dbkeys(self, trans, chrom_info=False):
""" Returns all known dbkeys. If chrom_info is True, only dbkeys with
chromosome lengths are returned. """
dbkeys = []
# Add user's custom keys to dbkeys.
user_keys_dict = {}
user = trans.get_user()
if 'dbkeys' in user.preferences:
user_keys_dict = from_json_string(user.preferences['dbkeys'])
dbkeys.extend([(attributes['name'], key)
for key, attributes in user_keys_dict.items()])
# Add app keys to dbkeys.
# If chrom_info is True, only include keys with len files (which contain chromosome info).
filter_fn = lambda b: True
if chrom_info:
filter_fn = lambda b: b.len_file is not None
dbkeys.extend([(genome.description, genome.key)
for key, genome in self.genomes.items()
if filter_fn(genome)])
return dbkeys
def main():
#Parse Command Line
parser = optparse.OptionParser()
parser.add_option( '-f', '--fasta_filename', dest='fasta_filename', action='store', type="string", default=None, help='fasta_filename' )
parser.add_option( '-d', '--fasta_dbkey', dest='fasta_dbkey', action='store', type="string", default=None, help='fasta_dbkey' )
parser.add_option( '-t', '--fasta_description', dest='fasta_description', action='store', type="string", default=None, help='fasta_description' )
parser.add_option( '-n', '--data_table_name', dest='data_table_name', action='store', type="string", default=None, help='data_table_name' )
(options, args) = parser.parse_args()
filename = args[0]
params = from_json_string( open( filename ).read() )
target_directory = params[ 'output_data' ][0]['extra_files_path']
os.mkdir( target_directory )
data_manager_dict = {}
dbkey = options.fasta_dbkey
if dbkey in [ None, '', '?' ]:
raise Exception( '"%s" is not a valid dbkey. You must specify a valid dbkey.' % ( dbkey ) )
sequence_id, sequence_name = get_id_name( params, dbkey=dbkey, fasta_description=options.fasta_description )
#build the index
build_bowtie2_index( data_manager_dict, options.fasta_filename, params, target_directory, dbkey, sequence_id, sequence_name, data_table_name=options.data_table_name or DEFAULT_DATA_TABLE_NAME )
#save info to json file
open( filename, 'wb' ).write( to_json_string( data_manager_dict ) )
def set_tool_versions( self, trans, **kwd ):
# Get the tool_versions from the tool shed for each tool in the installed change set.
repository = get_repository( trans, kwd[ 'id' ] )
tool_shed_url = get_url_from_repository_tool_shed( trans.app, repository )
url = '%s/repository/get_tool_versions?name=%s&owner=%s&changeset_revision=%s&webapp=galaxy&no_reset=true' % \
( tool_shed_url, repository.name, repository.owner, repository.changeset_revision )
response = urllib2.urlopen( url )
text = response.read()
response.close()
if text:
tool_version_dicts = from_json_string( text )
handle_tool_versions( trans.app, tool_version_dicts, repository )
message = "Tool versions have been set for all included tools."
status = 'done'
else:
message = "Version information for the tools included in the <b>%s</b> repository is missing. " % repository.name
message += "Reset all of this reppository's metadata in the tool shed, then set the installed tool versions "
message ++ "from the installed repository's <b>Repository Actions</b> menu. "
status = 'error'
shed_tool_conf, tool_path, relative_install_dir = get_tool_panel_config_tool_path_install_dir( trans.app, repository )
repo_files_dir = os.path.abspath( os.path.join( relative_install_dir, repository.name ) )
return trans.fill_template( '/admin/tool_shed_repository/manage_repository.mako',
repository=repository,
description=repository.description,
repo_files_dir=repo_files_dir,
message=message,
status=status )
def from_workflow_step( Class, trans, step ):
tool_id = step.tool_id
if trans.app.toolbox and tool_id not in trans.app.toolbox.tools_by_id:
# See if we have access to a different version of the tool.
# TODO: If workflows are ever enhanced to use tool version
# in addition to tool id, enhance the selection process here
# to retrieve the correct version of the tool.
tool = trans.app.toolbox.get_tool( tool_id )
if tool:
tool_id = tool.id
if ( trans.app.toolbox and tool_id in trans.app.toolbox.tools_by_id ):
if step.config:
# This step has its state saved in the config field due to the
# tool being previously unavailable.
return module_factory.from_dict(trans, from_json_string(step.config), secure=False)
module = Class( trans, tool_id )
module.state = galaxy.tools.DefaultToolState()
if step.tool_version and (step.tool_version != module.tool.version):
module.version_changes.append("%s: using version '%s' instead of version '%s' indicated in this workflow." % (tool_id, module.tool.version, step.tool_version))
module.state.inputs = module.tool.params_from_strings( step.tool_inputs, trans.app, ignore_errors=True )
module.errors = step.tool_errors
module.workflow_outputs = step.workflow_outputs
pjadict = {}
for pja in step.post_job_actions:
pjadict[pja.action_type] = pja
module.post_job_actions = pjadict
return module
return None
def sweepster( self, trans, id=None, hda_ldda=None, dataset_id=None, regions=None ):
"""
Displays a sweepster visualization using the incoming parameters. If id is available,
get the visualization with the given id; otherwise, create a new visualization using
a given dataset and regions.
"""
# Need to create history if necessary in order to create tool form.
trans.get_history( create=True )
if id:
# Loading a shared visualization.
viz = self.get_visualization( trans, id )
viz_config = self.get_visualization_config( trans, viz )
dataset = self.get_dataset( trans, viz_config[ 'dataset_id' ] )
else:
# Loading new visualization.
dataset = self.get_hda_or_ldda( trans, hda_ldda, dataset_id )
job = get_dataset_job( dataset )
viz_config = {
'dataset_id': dataset_id,
'tool_id': job.tool_id,
'regions': from_json_string( regions )
}
# Add tool, dataset attributes to config based on id.
tool = trans.app.toolbox.get_tool( viz_config[ 'tool_id' ] )
viz_config[ 'tool' ] = tool.to_dict( trans, for_display=True )
viz_config[ 'dataset' ] = dataset.get_api_value()
return trans.fill_template_mako( "visualization/sweepster.mako", config=viz_config )
def handle_result(self, result, param_dict, trans):
rval = from_json_string(result.content)
return trans.fill_template(
'/external_services/generic_jquery_grid.mako',
result=rval,
param_dict=param_dict,
action=self.parent)
def import_workflow( self, trans, payload, **kwd ):
"""
POST /api/tool_shed_repositories/import_workflow
Import the specified exported workflow contained in the specified installed tool shed repository into Galaxy.
:param key: the API key of the Galaxy user with which the imported workflow will be associated.
:param id: the encoded id of the ToolShedRepository object
The following parameters are included in the payload.
:param index: the index location of the workflow tuple in the list of exported workflows stored in the metadata for the specified repository
"""
api_key = kwd.get( 'key', None )
if api_key is None:
raise HTTPBadRequest( detail="Missing required parameter 'key' whose value is the API key for the Galaxy user importing the specified workflow." )
tool_shed_repository_id = kwd.get( 'id', '' )
if not tool_shed_repository_id:
raise HTTPBadRequest( detail="Missing required parameter 'id'." )
index = payload.get( 'index', None )
if index is None:
raise HTTPBadRequest( detail="Missing required parameter 'index'." )
repository = suc.get_tool_shed_repository_by_id( trans, tool_shed_repository_id )
exported_workflows = json.from_json_string( self.exported_workflows( trans, tool_shed_repository_id ) )
# Since we don't have an in-memory object with an id, we'll identify the exported workflow via it's location (i.e., index) in the list.
exported_workflow = exported_workflows[ int( index ) ]
workflow_name = exported_workflow[ 'workflow_name' ]
workflow, status, message = workflow_util.import_workflow( trans, repository, workflow_name )
if status == 'error':
log.error( message, exc_info=True )
trans.response.status = 500
return message
else:
return workflow.to_dict( view='element' )
def saved( self, trans, id=None, revision=None, type=None, config=None, title=None, **kwargs ):
"""
"""
DEFAULT_VISUALIZATION_NAME = 'Unnamed Visualization'
# post to saved in order to save a visualization
#TODO: re-route this one to clear up signature
if trans.request.method == 'POST':
if type is None or config is None:
return HTTPBadRequest( 'A visualization type and config are required to save a visualization' )
if isinstance( config, basestring ):
config = from_json_string( config )
title = title or DEFAULT_VISUALIZATION_NAME
#TODO: allow saving to (updating) a specific revision - should be part of UsesVisualization
#TODO: would be easier if this returned the visualization directly
returned = self.save_visualization( trans, config, type, id, title )
# redirect to GET to prevent annoying 'Do you want to post again?' dialog on page reload
render_url = web.url_for( controller='visualization', action='saved', id=returned.get( 'vis_id' ) )
return trans.response.send_redirect( render_url )
if id is None:
return HTTPBadRequest( 'A valid visualization id is required to load a visualization' )
# render the saved visualization by passing to render, sending latest revision config
#TODO: allow loading a specific revision - should be part of UsesVisualization
visualization = self.get_visualization( trans, id, check_ownership=True, check_accessible=False )
config = copy.copy( visualization.latest_revision.config )
# re-add title to kwargs for passing to render
if title:
kwargs[ 'title' ] = title
return self.render( trans, visualization.type, visualization, config=config, **kwargs )
def get_genome_build_names( self, trans=None ):
# FIXME: how to deal with key duplicates?
rval = []
# load user custom genome builds
if trans is not None:
if trans.history:
# This is a little bit Odd. We are adding every .len file in the current history to dbkey list,
# but this is previous behavior from trans.db_names, so we'll continue to do it.
# It does allow one-off, history specific dbkeys to be created by a user. But we are not filtering,
# so a len file will be listed twice (as the build name and again as dataset name),
# if custom dbkey creation/conversion occurred within the current history.
datasets = trans.sa_session.query( self._app.model.HistoryDatasetAssociation ) \
.filter_by( deleted=False, history_id=trans.history.id, extension="len" )
for dataset in datasets:
rval.append( ( dataset.dbkey, "%s (%s) [History]" % ( dataset.name, dataset.dbkey ) ) )
user = trans.user
if user and 'dbkeys' in user.preferences:
user_keys = from_json_string( user.preferences['dbkeys'] )
for key, chrom_dict in user_keys.iteritems():
rval.append( ( key, "%s (%s) [Custom]" % ( chrom_dict['name'], key ) ) )
# Load old builds.txt static keys
rval.extend( self._static_dbkeys )
#load dbkeys from dbkey data table
dbkey_table = self._app.tool_data_tables.get( self._data_table_name, None )
if dbkey_table is not None:
for field_dict in dbkey_table.get_named_fields_list():
rval.append( ( field_dict[ 'value' ], field_dict[ 'name' ] ) )
return rval
def main():
options, args = get_arg()
tool_dir = args[0]
path_to_alfa = os.path.join(tool_dir, 'ALFA.py')
if options.output_filename == None:
msg = 'No json output file specified'
sys.exit(msg)
output_filename = options.output_filename
# Interestingly the output file to return is not empty initially.
# it contains a dictionary, with notably the path to the dir where the alfa_indexes
# are expected to be found
params = from_json_string(open(output_filename).read())
target_directory = params['output_data'][0]['extra_files_path']
os.mkdir(target_directory)
tmp_dir = tempfile.mkdtemp(prefix='tmp', suffix='')
os.chdir(tmp_dir)
data_manager_dict = {}
if options.ensembl_info:
kingdom, species_name = options.ensembl_info
species_name = standardize_species_name(species_name)
url = get_ensembl_url_root(kingdom)
species_name, species_line = test_ensembl_species_exists(
kingdom, url, species_name)
gtf_archive_name = get_ensembl_gtf_archive(kingdom, url, species_name,
species_line)
data_table_entry = get_data_table_new_entry(gtf_archive_name)
gtf_file_name = '%s.gtf' % data_table_entry['prefix']
uncompress_gz(gtf_archive_name, gtf_file_name)
generate_alfa_indexes(path_to_alfa, gtf_file_name)
stranded_index_name = '%s.stranded.index' % data_table_entry['prefix']
unstranded_index_name = '%s.unstranded.index' % data_table_entry[
'prefix']
add_data_table_entry(data_manager_dict, data_table_entry)
print("____________________________________________________________")
print("*** General Info")
print("URL ROOT:\t%s" % url)
print("SPECIES:\t%s" % data_table_entry['species'])
print("VERSION:\t%s" % data_table_entry['version'])
print("RELEASE:\t%s" % data_table_entry['release'])
print("VALUE:\t%s" % data_table_entry['value'])
print("DBKEY:\t%s" % data_table_entry['dbkey'])
print("NAME:\t%s" % data_table_entry['name'])
print("PREFIX:\t%s" % data_table_entry['prefix'])
shutil.copyfile(stranded_index_name,
os.path.join(target_directory, stranded_index_name))
shutil.copyfile(unstranded_index_name,
os.path.join(target_directory, unstranded_index_name))
cleanup_before_exit(tmp_dir)
open(output_filename, 'wb').write(to_json_string(data_manager_dict))
def from_dict(Class, trans, d, secure=True):
module = Class(trans)
state = from_json_string(d["tool_state"])
module.state = dict(name=state.get("name", Class.default_name),
collection_type=state.get(
"collection_type",
Class.default_collection_type))
return module
def from_dict( Class, trans, d, secure=True ):
module = Class( trans )
state = from_json_string( d["tool_state"] )
module.state = dict(
name=state.get( "name", Class.default_name ),
collection_type=state.get( "collection_type", Class.default_collection_type )
)
return module
def db_builds( self ):
dbnames = []
if 'dbkeys' in self.user.preferences:
user_keys = from_json_string( self.user.preferences['dbkeys'] )
for key, chrom_dict in user_keys.iteritems():
dbnames.append((key, "%s (%s) [Custom]" % (chrom_dict['name'], key) ))
dbnames.extend( util.dbnames )
return dbnames
def import_workflow( trans, repository, workflow_name ):
"""Import a workflow contained in an installed tool shed repository into Galaxy (this method is called only from Galaxy)."""
status = 'done'
message = ''
changeset_revision = repository.changeset_revision
metadata = repository.metadata
workflows = metadata.get( 'workflows', [] )
tools_metadata = metadata.get( 'tools', [] )
workflow_dict = None
for workflow_data_tuple in workflows:
# The value of workflow_data_tuple is ( relative_path_to_workflow_file, exported_workflow_dict ).
relative_path_to_workflow_file, exported_workflow_dict = workflow_data_tuple
if exported_workflow_dict[ 'name' ] == workflow_name:
# If the exported workflow is available on disk, import it.
if os.path.exists( relative_path_to_workflow_file ):
workflow_file = open( relative_path_to_workflow_file, 'rb' )
workflow_data = workflow_file.read()
workflow_file.close()
workflow_dict = json.from_json_string( workflow_data )
else:
# Use the current exported_workflow_dict.
workflow_dict = exported_workflow_dict
break
if workflow_dict:
# Create workflow if possible.
workflow, missing_tool_tups = get_workflow_from_dict( trans=trans,
workflow_dict=workflow_dict,
tools_metadata=tools_metadata,
repository_id=repository.id,
changeset_revision=changeset_revision )
# Save the workflow in the Galaxy database. Pass workflow_dict along to create annotation at this point.
stored_workflow = save_workflow( trans, workflow, workflow_dict )
# Use the latest version of the saved workflow.
workflow = stored_workflow.latest_workflow
if workflow_name:
workflow.name = workflow_name
# Provide user feedback and show workflow list.
if workflow.has_errors:
message += "Imported, but some steps in this workflow have validation errors. "
status = "error"
if workflow.has_cycles:
message += "Imported, but this workflow contains cycles. "
status = "error"
else:
message += "Workflow <b>%s</b> imported successfully. " % workflow.name
if missing_tool_tups:
name_and_id_str = ''
for missing_tool_tup in missing_tool_tups:
tool_id, tool_name, other = missing_tool_tup
name_and_id_str += 'name: %s, id: %s' % ( str( tool_id ), str( tool_name ) )
message += "The following tools required by this workflow are missing from this Galaxy instance: %s. " % name_and_id_str
else:
workflow = None
message += 'The workflow named %s is not included in the metadata for revision %s of repository %s' % \
( str( workflow_name ), str( changeset_revision ), str( repository.name ) )
status = 'error'
return workflow, status, message
def __get_handler( self, job ):
try:
params = None
if job.params:
params = from_json_string( job.params )
return self.app.toolbox.tools_by_id.get( job.tool_id, None ).get_job_handler( params )
except:
log.exception( "(%s) Caught exception attempting to get tool-specific job handler for tool '%s', selecting at random from available handlers instead:" % ( job.id, job.tool_id ) )
return random.choice( self.app.config.job_handlers )
def import_workflow( trans, repository, workflow_name ):
"""Import a workflow contained in an installed tool shed repository into Galaxy (this method is called only from Galaxy)."""
status = 'done'
message = ''
changeset_revision = repository.changeset_revision
metadata = repository.metadata
workflows = metadata.get( 'workflows', [] )
tools_metadata = metadata.get( 'tools', [] )
workflow_dict = None
for workflow_data_tuple in workflows:
# The value of workflow_data_tuple is ( relative_path_to_workflow_file, exported_workflow_dict ).
relative_path_to_workflow_file, exported_workflow_dict = workflow_data_tuple
if exported_workflow_dict[ 'name' ] == workflow_name:
# If the exported workflow is available on disk, import it.
if os.path.exists( relative_path_to_workflow_file ):
workflow_file = open( relative_path_to_workflow_file, 'rb' )
workflow_data = workflow_file.read()
workflow_file.close()
workflow_dict = json.from_json_string( workflow_data )
else:
# Use the current exported_workflow_dict.
workflow_dict = exported_workflow_dict
break
if workflow_dict:
# Create workflow if possible.
workflow, missing_tool_tups = get_workflow_from_dict( trans=trans,
workflow_dict=workflow_dict,
tools_metadata=tools_metadata,
repository_id=repository.id,
changeset_revision=changeset_revision )
# Save the workflow in the Galaxy database. Pass workflow_dict along to create annotation at this point.
stored_workflow = save_workflow( trans, workflow, workflow_dict )
# Use the latest version of the saved workflow.
workflow = stored_workflow.latest_workflow
if workflow_name:
workflow.name = workflow_name
# Provide user feedback and show workflow list.
if workflow.has_errors:
message += "Imported, but some steps in this workflow have validation errors. "
status = "error"
if workflow.has_cycles:
message += "Imported, but this workflow contains cycles. "
status = "error"
else:
message += "Workflow <b>%s</b> imported successfully. " % workflow.name
if missing_tool_tups:
name_and_id_str = ''
for missing_tool_tup in missing_tool_tups:
tool_id, tool_name, other = missing_tool_tup
name_and_id_str += 'name: %s, id: %s' % ( str( tool_id ), str( tool_name ) )
message += "The following tools required by this workflow are missing from this Galaxy instance: %s. " % name_and_id_str
else:
workflow = None
message += 'The workflow named %s is not included in the metadata for revision %s of repository %s' % \
( str( workflow_name ), str( changeset_revision ), str( repository.name ) )
status = 'error'
return workflow, status, message
def json_from_url( url ):
error_message = ''
url_handle = urllib.urlopen( url )
url_contents = url_handle.read()
try:
parsed_json = from_json_string( url_contents )
except Exception, e:
error_message = str( url_contents )
return None, error_message
def main():
# Parse Command Line
parser = optparse.OptionParser()
parser.add_option('-f',
'--filename',
dest='filename',
action='store',
type='string',
default=None,
help='filename')
parser.add_option('-t',
'--tool_data_table_name',
dest='tool_data_table_name',
action='store',
type='string',
default=None,
help='tool_data_table_name')
(options, args) = parser.parse_args()
params = from_json_string(open(options.filename).read())
target_directory = params['output_data'][0]['extra_files_path']
os.mkdir(target_directory)
blastdb_name = params['param_dict']['blastdb_name'] # value
data_description = params['param_dict']['advanced'].get(
'data_description', None)
data_id = params['param_dict']['advanced'].get('data_id', None)
cmd_options = ['--decompress']
args = ['update_blastdb.pl'] + cmd_options + [blastdb_name]
proc = subprocess.Popen(args=args, shell=False, cwd=target_directory)
return_code = proc.wait()
if return_code != 1:
print >> sys.stderr, "Error obtaining blastdb (%s)" % return_code
sys.exit(1)
if not data_id:
data_id = "%s_%s" % (blastdb_name, get_dir_hash(target_directory))
if not data_description:
alias_date = None
try:
for line in open(
os.path.join(target_directory, "%s.nal" % (blastdb_name))):
if line.startswith('# Alias file created '):
alias_date = line.split('# Alias file created ',
1)[1].strip()
if line.startswith('TITLE'):
data_description = line.split(None, 1)[1].strip()
break
except Exception, e:
print >> sys.stderr, "Error Parsing Alias file for TITLE and date: %s" % (
e)
if alias_date and data_description:
data_description = "%s (%s)" % (data_description, alias_date)
def from_dict(Class,
trans,
repository_id,
changeset_revision,
step_dict,
tools_metadata=None,
secure=True):
module = Class(trans)
state = json.from_json_string(step_dict["tool_state"])
module.state = dict(name=state.get("name", "Input Dataset"))
return module
def main():
options, args = get_arg()
tool_dir = args[0]
path_to_alfa = os.path.join(tool_dir, 'ALFA.py')
if options.output_filename == None:
msg = 'No json output file specified'
sys.exit(msg)
output_filename = options.output_filename
# Interestingly the output file to return is not empty initially.
# it contains a dictionary, with notably the path to the dir where the alfa_indexes
# are expected to be found
params = from_json_string(open(output_filename).read())
target_directory = params['output_data'][0]['extra_files_path']
os.mkdir(target_directory)
tmp_dir = tempfile.mkdtemp(prefix='tmp', suffix='')
os.chdir(tmp_dir)
data_manager_dict = {}
if options.ensembl_info:
kingdom, species_name = options.ensembl_info
species_name = standardize_species_name(species_name)
url = get_ensembl_url_root(kingdom)
species_name, species_line = test_ensembl_species_exists(kingdom, url, species_name)
gtf_archive_name = get_ensembl_gtf_archive(kingdom, url, species_name, species_line)
data_table_entry = get_data_table_new_entry(gtf_archive_name)
gtf_file_name = '%s.gtf' % data_table_entry['prefix']
uncompress_gz(gtf_archive_name, gtf_file_name)
generate_alfa_indexes(path_to_alfa, gtf_file_name)
stranded_index_name = '%s.stranded.index' % data_table_entry['prefix']
unstranded_index_name = '%s.unstranded.index' % data_table_entry['prefix']
add_data_table_entry(data_manager_dict, data_table_entry)
print("____________________________________________________________")
print("*** General Info")
print("URL ROOT:\t%s" % url)
print("SPECIES:\t%s" % data_table_entry['species'])
print("VERSION:\t%s" % data_table_entry['version'])
print("RELEASE:\t%s" % data_table_entry['release'])
print("VALUE:\t%s" % data_table_entry['value'])
print("DBKEY:\t%s" % data_table_entry['dbkey'])
print("NAME:\t%s" % data_table_entry['name'])
print("PREFIX:\t%s" % data_table_entry['prefix'])
shutil.copyfile(stranded_index_name, os.path.join(target_directory, stranded_index_name))
shutil.copyfile(unstranded_index_name, os.path.join(target_directory, unstranded_index_name))
cleanup_before_exit(tmp_dir)
open(output_filename, 'wb').write(to_json_string(data_manager_dict))
def get_dbkeys_with_chrom_info( self, trans ):
""" Returns all valid dbkeys that have chromosome information. """
# All user keys have a len file.
user_keys = {}
user = trans.get_user()
if 'dbkeys' in user.preferences:
user_keys = from_json_string( user.preferences['dbkeys'] )
dbkeys = [ (v, k) for k, v in trans.db_builds if ( ( k in self.genomes and self.genomes[ k ].len_file ) or k in user_keys ) ]
return dbkeys
def __main__():
file_path = sys.argv.pop( 1 )
tmp_dir = sys.argv.pop( 1 )
galaxy.model.Dataset.file_path = file_path
galaxy.datatypes.metadata.MetadataTempFile.tmp_dir = tmp_dir
# Set up datatypes registry
config_root = sys.argv.pop( 1 )
datatypes_config = sys.argv.pop( 1 )
galaxy.model.set_datatypes_registry( galaxy.datatypes.registry.Registry( config_root, datatypes_config ) )
job_metadata = sys.argv.pop( 1 )
ext_override = dict()
if job_metadata != "None" and os.path.exists( job_metadata ):
for line in open( job_metadata, 'r' ):
try:
line = stringify_dictionary_keys( from_json_string( line ) )
assert line['type'] == 'dataset'
ext_override[line['dataset_id']] = line['ext']
except:
continue
for filenames in sys.argv[1:]:
fields = filenames.split( ',' )
filename_in = fields.pop( 0 )
filename_kwds = fields.pop( 0 )
filename_out = fields.pop( 0 )
filename_results_code = fields.pop( 0 )
dataset_filename_override = fields.pop( 0 )
#Need to be careful with the way that these parameters are populated from the filename splitting,
#because if a job is running when the server is updated, any existing external metadata command-lines
#will not have info about the newly added override_metadata file
if fields:
override_metadata = fields.pop( 0 )
else:
override_metadata = None
try:
dataset = cPickle.load( open( filename_in ) ) #load DatasetInstance
if dataset_filename_override:
dataset.dataset.external_filename = dataset_filename_override
if ext_override.get( dataset.dataset.id, None ):
dataset.extension = ext_override[ dataset.dataset.id ]
#Metadata FileParameter types may not be writable on a cluster node, and are therefore temporarily substituted with MetadataTempFiles
if override_metadata:
override_metadata = simplejson.load( open( override_metadata ) )
for metadata_name, metadata_file_override in override_metadata:
if galaxy.datatypes.metadata.MetadataTempFile.is_JSONified_value( metadata_file_override ):
metadata_file_override = galaxy.datatypes.metadata.MetadataTempFile.from_JSON( metadata_file_override )
setattr( dataset.metadata, metadata_name, metadata_file_override )
kwds = stringify_dictionary_keys( simplejson.load( open( filename_kwds ) ) )#load kwds; need to ensure our keywords are not unicode
dataset.datatype.set_meta( dataset, **kwds )
dataset.metadata.to_JSON_dict( filename_out ) # write out results of set_meta
simplejson.dump( ( True, 'Metadata has been set successfully' ), open( filename_results_code, 'wb+' ) ) #setting metadata has succeeded
except Exception, e:
simplejson.dump( ( False, str( e ) ), open( filename_results_code, 'wb+' ) ) #setting metadata has failed somehow
def save(self, trans, id, content, annotations):
id = trans.security.decode_id(id)
page = trans.sa_session.query(model.Page).get(id)
assert page.user == trans.user
# Sanitize content
content = sanitize_html(content, 'utf-8', 'text/html')
# Add a new revision to the page with the provided content.
page_revision = model.PageRevision()
page_revision.title = page.title
page_revision.page = page
page.latest_revision = page_revision
page_revision.content = content
# Save annotations.
annotations = from_json_string(annotations)
for annotation_dict in annotations:
item_id = trans.security.decode_id(annotation_dict['item_id'])
item_class = self.get_class(annotation_dict['item_class'])
item = trans.sa_session.query(item_class).filter_by(
id=item_id).first()
if not item:
raise RuntimeError("cannot find annotated item")
text = sanitize_html(annotation_dict['text'], 'utf-8', 'text/html')
# Add/update annotation.
if item_id and item_class and text:
# Get annotation association.
annotation_assoc_class = eval("model.%sAnnotationAssociation" %
item_class.__name__)
annotation_assoc = trans.sa_session.query(
annotation_assoc_class).filter_by(user=trans.get_user())
if item_class == model.History.__class__:
annotation_assoc = annotation_assoc.filter_by(history=item)
elif item_class == model.HistoryDatasetAssociation.__class__:
annotation_assoc = annotation_assoc.filter_by(hda=item)
elif item_class == model.StoredWorkflow.__class__:
annotation_assoc = annotation_assoc.filter_by(
stored_workflow=item)
elif item_class == model.WorkflowStep.__class__:
annotation_assoc = annotation_assoc.filter_by(
workflow_step=item)
annotation_assoc = annotation_assoc.first()
if not annotation_assoc:
# Create association.
annotation_assoc = annotation_assoc_class()
item.annotations.append(annotation_assoc)
annotation_assoc.user = trans.get_user()
# Set annotation user text.
annotation_assoc.annotation = text
trans.sa_session.flush()
def save( self, trans, vis_json=None, type=None, id=None, title=None, dbkey=None, annotation=None ):
"""
Save a visualization; if visualization does not have an ID, a new
visualization is created. Returns JSON of visualization.
"""
# Get visualization attributes from kwargs or from config.
vis_config = from_json_string( vis_json )
vis_type = type or vis_config[ 'type' ]
vis_id = id or vis_config.get( 'id', None )
vis_title = title or vis_config.get( 'title', None )
vis_dbkey = dbkey or vis_config.get( 'dbkey', None )
vis_annotation = annotation or vis_config.get( 'annotation', None )
return self.save_visualization( trans, vis_config, vis_type, vis_id, vis_title, vis_dbkey, vis_annotation )
def check_file_contents_for_email_alerts( trans ):
"""
See if any admin users have chosen to receive email alerts when a repository is updated. If so, the file contents of the update must be
checked for inappropriate content.
"""
admin_users = trans.app.config.get( "admin_users", "" ).split( "," )
for repository in trans.sa_session.query( trans.model.Repository ) \
.filter( trans.model.Repository.table.c.email_alerts != None ):
email_alerts = json.from_json_string( repository.email_alerts )
for user_email in email_alerts:
if user_email in admin_users:
return True
return False
def check_file_contents_for_email_alerts(trans):
"""
See if any admin users have chosen to receive email alerts when a repository is updated. If so, the file contents of the update must be
checked for inappropriate content.
"""
admin_users = trans.app.config.get("admin_users", "").split(",")
for repository in trans.sa_session.query( trans.model.Repository ) \
.filter( trans.model.Repository.table.c.email_alerts != None ):
email_alerts = json.from_json_string(repository.email_alerts)
for user_email in email_alerts:
if user_email in admin_users:
return True
return False
def main():
#Parse Command Line
parser = optparse.OptionParser()
parser.add_option('-j',
'--jar_path',
dest='jar_path',
action='store',
type="string",
default=None,
help='snpEff.jar path')
parser.add_option('-c',
'--config',
dest='config',
action='store',
type="string",
default=None,
help='snpEff.config path')
parser.add_option('-g',
'--genome_version',
dest='genome_version',
action='store',
type="string",
default=None,
help='genome_version')
parser.add_option('-o',
'--organism',
dest='organism',
action='store',
type="string",
default=None,
help='organism name')
(options, args) = parser.parse_args()
filename = args[0]
params = from_json_string(open(filename).read())
target_directory = params['output_data'][0]['extra_files_path']
os.mkdir(target_directory)
data_manager_dict = {}
#Create SnpEff Reference Data
for genome_version, organism in zip(
options.genome_version.split(','),
getOrganismNames(options.jar_path, options.genome_version,
options.organism).split(',')):
download_database(data_manager_dict, target_directory,
options.jar_path, options.config, genome_version,
organism)
#save info to json file
open(filename, 'wb').write(to_json_string(data_manager_dict))
def save( self, trans, vis_json=None, type=None, id=None, title=None, dbkey=None, annotation=None ):
"""
Save a visualization; if visualization does not have an ID, a new
visualization is created. Returns JSON of visualization.
"""
# Get visualization attributes from kwargs or from config.
vis_config = from_json_string( vis_json )
vis_type = type or vis_config[ 'type' ]
vis_id = id or vis_config.get( 'id', None )
vis_title = title or vis_config.get( 'title', None )
vis_dbkey = dbkey or vis_config.get( 'dbkey', None )
vis_annotation = annotation or vis_config.get( 'annotation', None )
return self.save_visualization( trans, vis_config, vis_type, vis_id, vis_title, vis_dbkey, vis_annotation )
def upgrade():
print __doc__
metadata.reflect()
Visualization_table = Table("visualization", metadata, autoload=True)
Visualization_revision_table = Table("visualization_revision",
metadata,
autoload=True)
# Create dbkey columns.
x = Column("dbkey", TEXT, index=True)
y = Column("dbkey", TEXT, index=True)
x.create(Visualization_table)
y.create(Visualization_revision_table)
assert x is Visualization_table.c.dbkey
assert y is Visualization_revision_table.c.dbkey
try:
i = Index("ix_visualization_dbkey", Visualization_table.c.dbkey)
i.create()
except:
pass
try:
i = Index("ix_visualization_revision_dbkey",
Visualization_revision_table.c.dbkey)
i.create()
except:
pass
all_viz = db_session.execute(
"SELECT visualization.id as viz_id, visualization_revision.id as viz_rev_id, visualization_revision.config FROM visualization_revision \
LEFT JOIN visualization ON visualization.id=visualization_revision.visualization_id"
)
for viz in all_viz:
viz_id = viz['viz_id']
viz_rev_id = viz['viz_rev_id']
if viz[Visualization_revision_table.c.config]:
dbkey = from_json_string(
viz[Visualization_revision_table.c.config]).get('dbkey',
"").replace(
"'", "\\'")
db_session.execute(
"UPDATE visualization_revision SET dbkey='%s' WHERE id=%s" %
(dbkey, viz_rev_id))
db_session.execute(
"UPDATE visualization SET dbkey='%s' WHERE id=%s" %
(dbkey, viz_id))
def get_readme_files_dict_for_display( trans, tool_shed_url, repo_info_dict ):
"""
Return a dictionary of README files contained in the single repository being installed so they can be displayed on the tool panel section
selection page.
"""
name = repo_info_dict.keys()[ 0 ]
repo_info_tuple = repo_info_dict[ name ]
description, repository_clone_url, changeset_revision, ctx_rev, repository_owner, repository_dependencies, installed_td = \
suc.get_repo_info_tuple_contents( repo_info_tuple )
# Handle README files.
url = suc.url_join( tool_shed_url,
'repository/get_readme_files?name=%s&owner=%s&changeset_revision=%s' % ( name, repository_owner, changeset_revision ) )
raw_text = common_util.tool_shed_get( trans.app, tool_shed_url, url )
readme_files_dict = json.from_json_string( raw_text )
return readme_files_dict
def main():
#Parse Command Line
parser = optparse.OptionParser()
parser.add_option( '-f', '--filename', dest='filename', action='store', type='string', default=None, help='filename' )
parser.add_option( '-t', '--tool_data_table_name', dest='tool_data_table_name', action='store', type='string', default=None, help='tool_data_table_name' )
(options, args) = parser.parse_args()
#Take the JSON input file for parsing
params = from_json_string( open( options.filename ).read() )
target_directory = params[ 'output_data' ][0]['extra_files_path']
os.mkdir( target_directory )
#Fetch parameters from input JSON file
blastdb_name = params['param_dict']['db_type'].get( 'blastdb_name' )
blastdb_type = params['param_dict']['db_type'].get( 'blastdb_type' )
data_description = params['param_dict']['advanced'].get( 'data_description', None )
data_id = params['param_dict']['advanced'].get( 'data_id', None )
date = None
#update_blastdb.pl doesn't download protein domains, so we use ftp
if blastdb_type == 'blastdb_d':
try:
archive_name = blastdb_name + '_LE.tar.gz'
tar_file = open( os.path.join( target_directory, archive_name ), "wb" )
#Connect via ftp and download
ftp = FTP('ftp.ncbi.nih.gov')
ftp.login()
ftp.cwd('pub/mmdb/cdd/little_endian')
date = datetime.strptime(ftp.sendcmd("MDTM " + archive_name)[4:], "%Y%m%d%H%M%S").strftime("%Y_%m_%d")
ftp.retrbinary('RETR %s' % archive_name, tar_file.write)
tar_file.close()
#Extract contents
tar_file = tarfile.open(os.path.join( target_directory, archive_name ), mode='r')
tar_file.extractall( target_directory )
tar_file.close()
#If the download fails, ftplib should generate an error in ftplib.all_errors
#Likewise, tarfile.ReadError should catch any errors when reading from the tar
#And other possible errors that can occur here...
except IOError, e:
print >> sys.stderr, "Cannot create file: %s: %s" % ( archive_name, e )
sys.exit( 1 )
except os.error, e:
print "Error while joining %s and %s: %s" % ( target_directory, archive_name, e )
sys.exit( 1 )
def main():
#Parse Command Line
parser = optparse.OptionParser()
parser.add_option('-k',
'--dbkey',
dest='dbkey',
action='store',
type="string",
default=None,
help='dbkey')
parser.add_option('-d',
'--description',
dest='description',
action='store',
type="string",
default=None,
help='description')
parser.add_option('-c',
'--defuse_config',
dest='defuse_config',
action='store',
type="string",
default=None,
help='defuse_config')
parser.add_option('-s',
'--defuse_script',
dest='defuse_script',
action='store',
type="string",
default=None,
help='defuse_script')
(options, args) = parser.parse_args()
filename = args[0]
params = from_json_string(open(filename).read())
target_directory = params['output_data'][0]['extra_files_path']
os.mkdir(target_directory)
data_manager_dict = {}
#Create Defuse Reference Data
run_defuse_script(data_manager_dict, params, target_directory,
options.dbkey, options.description,
options.defuse_config, options.defuse_script)
#save info to json file
open(filename, 'wb').write(to_json_string(data_manager_dict))
def save( self, trans, id, content, annotations ):
id = trans.security.decode_id( id )
page = trans.sa_session.query( model.Page ).get( id )
assert page.user == trans.user
# Sanitize content
content = sanitize_html( content, 'utf-8', 'text/html' )
# Add a new revision to the page with the provided content.
page_revision = model.PageRevision()
page_revision.title = page.title
page_revision.page = page
page.latest_revision = page_revision
page_revision.content = content
# Save annotations.
annotations = from_json_string( annotations )
for annotation_dict in annotations:
item_id = trans.security.decode_id( annotation_dict[ 'item_id' ] )
item_class = self.get_class( annotation_dict[ 'item_class' ] )
item = trans.sa_session.query( item_class ).filter_by( id=item_id ).first()
if not item:
raise RuntimeError( "cannot find annotated item" )
text = sanitize_html( annotation_dict[ 'text' ], 'utf-8', 'text/html' )
# Add/update annotation.
if item_id and item_class and text:
# Get annotation association.
annotation_assoc_class = eval( "model.%sAnnotationAssociation" % item_class.__name__ )
annotation_assoc = trans.sa_session.query( annotation_assoc_class ).filter_by( user=trans.get_user() )
if item_class == model.History.__class__:
annotation_assoc = annotation_assoc.filter_by( history=item )
elif item_class == model.HistoryDatasetAssociation.__class__:
annotation_assoc = annotation_assoc.filter_by( hda=item )
elif item_class == model.StoredWorkflow.__class__:
annotation_assoc = annotation_assoc.filter_by( stored_workflow=item )
elif item_class == model.WorkflowStep.__class__:
annotation_assoc = annotation_assoc.filter_by( workflow_step=item )
annotation_assoc = annotation_assoc.first()
if not annotation_assoc:
# Create association.
annotation_assoc = annotation_assoc_class()
item.annotations.append( annotation_assoc )
annotation_assoc.user = trans.get_user()
# Set annotation user text.
annotation_assoc.annotation = text
trans.sa_session.flush()
def load_input_parameters( filename, erase_file = False ):
datasource_params = {}
try:
json_params = from_json_string( open( filename, 'r' ).read() )
datasource_params = json_params.get( 'param_dict' )
except:
json_params = None
for line in open( filename, 'r' ):
try:
line = line.strip()
fields = line.split( '\t' )
datasource_params[ fields[0] ] = unquote(fields[1]).replace('\r','')
except:
continue
if erase_file:
open( filename, 'w' ).close() #open file for writing, then close, removes params from file
return json_params, datasource_params
def read_input_json(jsonfile):
"""Read the JSON supplied from the data manager tool
Returns a tuple (param_dict,extra_files_path)
'param_dict' is an arbitrary dictionary of parameters
input into the tool; 'extra_files_path' is the path
to a directory where output files must be put for the
receiving data manager to pick them up.
NB the directory pointed to by 'extra_files_path'
doesn't exist initially, it is the job of the script
to create it if necessary.
"""
params = from_json_string(open(jsonfile).read())
return (params['param_dict'], params['output_data'][0]['extra_files_path'])
def upgrade(migrate_engine):
metadata.bind = migrate_engine
print __doc__
metadata.reflect()
Visualization_table = Table("visualization", metadata, autoload=True)
Visualization_revision_table = Table("visualization_revision",
metadata,
autoload=True)
# Create dbkey columns.
x = Column("dbkey", TEXT)
y = Column("dbkey", TEXT)
x.create(Visualization_table)
y.create(Visualization_revision_table)
# Manually create indexes for compatability w/ mysql_length.
xi = Index("ix_visualization_dbkey",
Visualization_table.c.dbkey,
mysql_length=200)
xi.create()
yi = Index("ix_visualization_revision_dbkey",
Visualization_revision_table.c.dbkey,
mysql_length=200)
yi.create()
assert x is Visualization_table.c.dbkey
assert y is Visualization_revision_table.c.dbkey
all_viz = migrate_engine.execute(
"SELECT visualization.id as viz_id, visualization_revision.id as viz_rev_id, visualization_revision.config FROM visualization_revision \
LEFT JOIN visualization ON visualization.id=visualization_revision.visualization_id"
)
for viz in all_viz:
viz_id = viz['viz_id']
viz_rev_id = viz['viz_rev_id']
if viz[Visualization_revision_table.c.config]:
dbkey = from_json_string(
viz[Visualization_revision_table.c.config]).get('dbkey',
"").replace(
"'", "\\'")
migrate_engine.execute(
"UPDATE visualization_revision SET dbkey='%s' WHERE id=%s" %
(dbkey, viz_rev_id))
migrate_engine.execute(
"UPDATE visualization SET dbkey='%s' WHERE id=%s" %
(dbkey, viz_id))
def from_json_string_recurse(item):
decoded_list = []
if isinstance( item, basestring):
try:
# Not clear what we're decoding, so recurse to ensure that we catch everything.
decoded_item = from_json_string( item )
if isinstance( decoded_item, list):
decoded_list = from_json_string_recurse( decoded_item )
else:
decoded_list = [ unicode( decoded_item ) ]
except ValueError:
decoded_list = [ unicode ( item ) ]
elif isinstance( item, list):
for element in item:
a_list = from_json_string_recurse( element )
decoded_list = decoded_list + a_list
return decoded_list
def get_readme_files_dict_for_display( app, tool_shed_url, repo_info_dict ):
"""
Return a dictionary of README files contained in the single repository being installed so they can be displayed on the tool panel section
selection page.
"""
name = repo_info_dict.keys()[ 0 ]
repo_info_tuple = repo_info_dict[ name ]
description, repository_clone_url, changeset_revision, ctx_rev, repository_owner, repository_dependencies, installed_td = \
suc.get_repo_info_tuple_contents( repo_info_tuple )
# Handle changing HTTP protocols over time.
tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( app, tool_shed_url )
params = '?name=%s&owner=%s&changeset_revision=%s' % ( name, repository_owner, changeset_revision )
url = common_util.url_join( tool_shed_url,
'repository/get_readme_files%s' % params )
raw_text = common_util.tool_shed_get( app, tool_shed_url, url )
readme_files_dict = json.from_json_string( raw_text )
return readme_files_dict
def chroms( self, trans, dbkey=None, num=None, chrom=None, low=None ):
"""
Returns a naturally sorted list of chroms/contigs for a given dbkey.
Use either chrom or low to specify the starting chrom in the return list.
"""
# If there is no dbkey owner, default to current user.
dbkey_owner, dbkey = decode_dbkey( dbkey )
if dbkey_owner:
dbkey_user = trans.sa_session.query( trans.app.model.User ).filter_by( username=dbkey_owner ).first()
else:
dbkey_user = trans.user
#
# Get/create genome object.
#
genome = None
twobit_file = None
# Look first in user's custom builds.
if dbkey_user and 'dbkeys' in dbkey_user.preferences:
user_keys = from_json_string( dbkey_user.preferences['dbkeys'] )
if dbkey in user_keys:
dbkey_attributes = user_keys[ dbkey ]
if 'fasta' in dbkey_attributes:
build_fasta = trans.sa_session.query( trans.app.model.HistoryDatasetAssociation ).get( dbkey_attributes[ 'fasta' ] )
len_file = build_fasta.get_converted_dataset( trans, 'len' ).file_name
converted_dataset = build_fasta.get_converted_dataset( trans, 'twobit' )
if converted_dataset:
twobit_file = converted_dataset.file_name
# Backwards compatibility: look for len file directly.
elif 'len' in dbkey_attributes:
len_file = trans.sa_session.query( trans.app.model.HistoryDatasetAssociation ).get( user_keys[ dbkey ][ 'len' ] ).file_name
if len_file:
genome = Genome( dbkey, len_file=len_file, twobit_file=twobit_file )
# Look in system builds.
if not genome:
len_ds = trans.db_dataset_for( dbkey )
if not len_ds:
genome = self.genomes[ dbkey ]
else:
gneome = Genome( dbkey, len_file=len_ds.file_name )
return genome.to_dict( num=num, chrom=chrom, low=low )
def from_json_string_recurse(item):
decoded_list = []
if isinstance( item, basestring):
try:
# Not clear what we're decoding, so recurse to ensure that we catch everything.
decoded_item = from_json_string( item )
if isinstance( decoded_item, list):
decoded_list = from_json_string_recurse( decoded_item )
else:
decoded_list = [ unicode( decoded_item ) ]
except ValueError:
decoded_list = [ unicode ( item ) ]
elif isinstance( item, list):
for element in item:
a_list = from_json_string_recurse( element )
decoded_list = decoded_list + a_list
return decoded_list
def read_input_json(jsonfile):
"""Read the JSON supplied from the data manager tool
Returns a tuple (param_dict,extra_files_path)
'param_dict' is an arbitrary dictionary of parameters
input into the tool; 'extra_files_path' is the path
to a directory where output files must be put for the
receiving data manager to pick them up.
NB the directory pointed to by 'extra_files_path'
doesn't exist initially, it is the job of the script
to create it if necessary.
"""
params = from_json_string(open(jsonfile).read())
return (params['param_dict'],
params['output_data'][0]['extra_files_path'])