def configure_logging( config ): """ Allow some basic logging configuration to be read from the cherrpy config. """ # PasteScript will have already configured the logger if the appropriate # sections were found in the config file, so we do nothing if the # config has a loggers section, otherwise we do some simple setup # using the 'log_*' values from the config. if config.global_conf_parser.has_section( "loggers" ): return format = config.get( "log_format", "%(name)s %(levelname)s %(asctime)s %(message)s" ) level = logging._levelNames[ config.get( "log_level", "DEBUG" ) ] destination = config.get( "log_destination", "stdout" ) log.info( "Logging at '%s' level to '%s'" % ( level, destination ) ) # Get root logger root = logging.getLogger() # Set level root.setLevel( level ) # Turn down paste httpserver logging if level <= logging.DEBUG: logging.getLogger( "paste.httpserver.ThreadPool" ).setLevel( logging.WARN ) # Remove old handlers for h in root.handlers[:]: root.removeHandler(h) # Create handler if destination == "stdout": handler = logging.StreamHandler( sys.stdout ) else: handler = logging.FileHandler( destination ) # Create formatter formatter = logging.Formatter( format ) # Hook everything up handler.setFormatter( formatter ) root.addHandler( handler ) # If sentry is configured, also log to it if config.sentry_dsn: eggs.require( "raven" ) from raven.handlers.logging import SentryHandler sentry_handler = SentryHandler( config.sentry_dsn ) sentry_handler.setLevel( logging.WARN ) root.addHandler( sentry_handler )
def create_database(config_file): parser = ConfigParser.SafeConfigParser() parser.read(config_file) # Determine which database connection to use. database_connection = parser.get("app:main", "install_database_connection") if database_connection is None: database_connection = parser.get("app:main", "database_connection") if database_connection is None: database_connection = "sqlite:///%s" % parser.get("app:main", "database_file") if database_connection is None: print "Unable to determine correct database connection." exit(1) """Initialize the database file.""" dialect_to_egg = { "sqlite": "pysqlite>=2", "postgres": "psycopg2", "postgresql": "psycopg2", "mysql": "MySQL_python", } dialect = (database_connection.split(":", 1))[0] try: egg = dialect_to_egg[dialect] try: eggs.require(egg) print ("%s egg successfully loaded for %s dialect" % (egg, dialect)) except: # If the module is in the path elsewhere (i.e. non-egg), it'll still load. print ("%s egg not found, but an attempt will be made to use %s anyway" % (egg, dialect)) except KeyError: # Let this go, it could possibly work with db's we don't support. print ("database_connection contains an unknown SQLAlchemy database dialect: %s" % dialect) # Initialize the database connection. engine = create_engine(database_connection) MetaData(bind=engine) install_session = scoped_session(sessionmaker(bind=engine, autoflush=False, autocommit=True)) model = mapping.init(database_connection) return install_session, model
def get_track_window(self, dataset, data, start, end): """ Assumes we have a numpy file. """ # Maybe if we import here people will still be able to use Galaxy when numpy kills it eggs.require("numpy>=1.2.1") import numpy range = end - start # Determine appropriate resolution to plot ~1000 points resolution = ( 10 ** math.ceil( math.log10( range / 1000 ) ) ) # Restrict to valid range resolution = min( resolution, 10000 ) resolution = max( resolution, 1 ) # Memory map the array (don't load all the data) data = numpy.load( data ) # Grab just what we need t_start = math.floor( start / resolution ) t_end = math.ceil( end / resolution ) x = numpy.arange( t_start, t_end ) * resolution y = data[ t_start : t_end ] return zip(x.tolist(), y.tolist())
import json import logging import os import string import subprocess import time from galaxy import eggs from galaxy import model from galaxy.jobs import JobDestination from galaxy.jobs.handler import DEFAULT_JOB_PUT_FAILURE_MESSAGE from galaxy.jobs.runners import AsynchronousJobState, AsynchronousJobRunner from galaxy.util import asbool eggs.require( "drmaa" ) log = logging.getLogger( __name__ ) __all__ = [ 'DRMAAJobRunner' ] drmaa = None DRMAA_jobTemplate_attributes = [ 'args', 'remoteCommand', 'outputPath', 'errorPath', 'nativeSpecification', 'workingDirectory', 'jobName', 'email', 'project' ] class DRMAAJobRunner( AsynchronousJobRunner ): """ Job runner backed by a finite pool of worker threads. FIFO scheduling """
Populates blank uuid fields in datasets with randomly generated values Going forward, these ids will be generated for all new datasets. This script fixes datasets that were generated before the change. """ import sys, os, ConfigParser import galaxy.app from galaxy.util.bunch import Bunch import galaxy.datatypes.tabular from galaxy.model.orm.scripts import get_config from galaxy import eggs from galaxy.model import mapping import uuid eggs.require( "SQLAlchemy" ) from sqlalchemy import * assert sys.version_info[:2] >= ( 2, 4 ) def main(): ini_file = sys.argv.pop(1) config = get_config(ini_file) model = mapping.init( ini_file, config['db_url'], create_tables = False ) for row in model.context.query( model.Dataset ): if row.uuid is None: row.uuid = uuid.uuid4() print "Setting dataset:", row.id, " UUID to ", row.uuid
import logging from galaxy import eggs eggs.require('SQLAlchemy') from sqlalchemy import and_, or_ from tool_shed.util import hg_util from tool_shed.util import shed_util_common as suc log = logging.getLogger( __name__ ) class ToolVersionManager( object ): def __init__( self, app ): self.app = app def get_tool_version( self, tool_id ): context = self.app.install_model.context return context.query( self.app.install_model.ToolVersion ) \ .filter( self.app.install_model.ToolVersion.table.c.tool_id == tool_id ) \ .first() def get_tool_version_association( self, parent_tool_version, tool_version ): """ Return a ToolVersionAssociation if one exists that associates the two received tool_versions. This function is called only from Galaxy. """ context = self.app.install_model.context return context.query( self.app.install_model.ToolVersionAssociation ) \ .filter( and_( self.app.install_model.ToolVersionAssociation.table.c.parent_id == parent_tool_version.id,
will execute this script with the appropriate parameters. """ import os import sys # Assume we are run from the galaxy root directory, add lib to the python path cwd = os.getcwd() sys.path.append( cwd ) new_path = [ os.path.join( cwd, "scripts" ), os.path.join( cwd, "lib" ), os.path.join( cwd, 'test' ), os.path.join( cwd, 'scripts', 'api' ) ] new_path.extend( sys.path ) sys.path = new_path from galaxy import eggs eggs.require( "nose" ) eggs.require( "Paste" ) eggs.require( 'mercurial' ) # This should not be required, but it is under certain conditions thanks to this bug: # http://code.google.com/p/python-nose/issues/detail?id=284 eggs.require( "pysqlite" ) import httplib import install_and_test_tool_shed_repositories.base.test_db_util as test_db_util import install_and_test_tool_shed_repositories.functional.test_install_repositories as test_install_repositories import logging import nose import random import re import shutil import socket
import logging import os import Queue import shutil import subprocess import tempfile import threading import time from contextlib import contextmanager # TODO: eliminate the use of fabric here. from galaxy import eggs eggs.require( 'paramiko' ) eggs.require( 'ssh' ) eggs.require( 'Fabric' ) from fabric.operations import _AttributeString from fabric import state from galaxy.util import DATABASE_MAX_STRING_SIZE from galaxy.util import DATABASE_MAX_STRING_SIZE_PRETTY from galaxy.util import shrink_string_by_size from galaxy.util import unicodify from tool_shed.galaxy_install.tool_dependencies.recipe import asynchronous_reader from tool_shed.util import basic_util log = logging.getLogger( __name__ )
"""Binary classes""" import binascii import data import gzip import logging import os import shutil import struct import subprocess import tempfile import warnings import zipfile from galaxy import eggs eggs.require("bx-python") from bx.seq.twobit import TWOBIT_MAGIC_NUMBER, TWOBIT_MAGIC_NUMBER_SWAP, TWOBIT_MAGIC_SIZE from galaxy.util import sqlite from galaxy.datatypes.metadata import MetadataElement, MetadataParameter, ListParameter, DictParameter from galaxy.datatypes import metadata import dataproviders with warnings.catch_warnings(): warnings.simplefilter("ignore") eggs.require("pysam") from pysam import csamtools log = logging.getLogger(__name__)
Dataproviders that use either: - the file contents and/or metadata from a Galaxy DatasetInstance as their source. - or provide data in some way relevant to bioinformatic data (e.g. parsing genomic regions from their source) """ import base import line import column import external from galaxy.util import sqlite import re from galaxy import eggs eggs.require('bx-python') from bx import seq as bx_seq from bx import wiggle as bx_wig from bx import bbi as bx_bbi _TODO = """ use bx as much as possible gff3 hierarchies change SamtoolsDataProvider to use pysam """ import logging log = logging.getLogger(__name__)
def create_or_verify_database(url, engine_options={}, app=None): """ """ dialect = (url.split(':', 1))[0] try: egg = dialect_to_egg[dialect] try: eggs.require(egg) log.debug("%s egg successfully loaded for %s dialect" % (egg, dialect)) except: # If the module is in the path elsewhere (i.e. non-egg), it'll still load. log.warning( "%s egg not found, but an attempt will be made to use %s anyway" % (egg, dialect)) except KeyError: # Let this go, it could possibly work with db's we don't support log.error( "database_connection contains an unknown SQLAlchemy database dialect: %s" % dialect) # Create engine and metadata engine = create_engine(url, **engine_options) def migrate(): try: # Declare the database to be under a repository's version control db_schema = schema.ControlledSchema.create(engine, migrate_repository) except: # The database is already under version control db_schema = schema.ControlledSchema(engine, migrate_repository) # Apply all scripts to get to current version migrate_to_current_version(engine, db_schema) meta = MetaData(bind=engine) if app and getattr(app.config, 'database_auto_migrate', False): migrate() return # Try to load tool_shed_repository table try: Table("tool_shed_repository", meta, autoload=True) except NoSuchTableError: # No table means a completely uninitialized database. If we # have an app, we'll set its new_installation setting to True # so the tool migration process will be skipped. migrate() return try: Table("migrate_version", meta, autoload=True) except NoSuchTableError: # The database exists but is not yet under migrate version control, so init with version 1 log.info("Adding version control to existing database") try: Table("metadata_file", meta, autoload=True) schema.ControlledSchema.create(engine, migrate_repository, version=2) except NoSuchTableError: schema.ControlledSchema.create(engine, migrate_repository, version=1) # Verify that the code and the DB are in sync db_schema = schema.ControlledSchema(engine, migrate_repository) if migrate_repository.versions.latest != db_schema.version: exception_msg = "Your database has version '%d' but this code expects version '%d'. " % ( db_schema.version, migrate_repository.versions.latest) exception_msg += "Back up your database and then migrate the schema by running the following from your Galaxy installation directory:" exception_msg += "\n\nsh manage_db.sh upgrade install\n" else: log.info("At database version %d" % db_schema.version)
def verify_tools(app, url, galaxy_config_file, engine_options={}): # Check the value in the migrate_tools.version database table column to verify that the number is in # sync with the number of version scripts in ~/lib/galaxy/tools/migrate/versions. dialect = (url.split(':', 1))[0] try: egg = dialect_to_egg[dialect] try: eggs.require(egg) log.debug("%s egg successfully loaded for %s dialect" % (egg, dialect)) except: # If the module is in the path elsewhere (i.e. non-egg), it'll still load. log.warning( "%s egg not found, but an attempt will be made to use %s anyway" % (egg, dialect)) except KeyError: # Let this go, it could possibly work with db's we don't support log.error( "database_connection contains an unknown SQLAlchemy database dialect: %s" % dialect) # Create engine and metadata engine = create_engine(url, **engine_options) meta = MetaData(bind=engine) # The migrate_tools table was created in database version script 0092_add_migrate_tools_table.py. version_table = Table("migrate_tools", meta, autoload=True) # Verify that the code and the database are in sync. db_schema = schema.ControlledSchema(engine, migrate_repository) latest_tool_migration_script_number = migrate_repository.versions.latest if latest_tool_migration_script_number != db_schema.version: # The default behavior is that the tool shed is down. tool_shed_accessible = False if app.new_installation: # New installations will not be missing tools, so we don't need to worry about them. missing_tool_configs_dict = odict() else: tool_panel_configs = common_util.get_non_shed_tool_panel_configs( app) if tool_panel_configs: # The missing_tool_configs_dict contents are something like: # {'emboss_antigenic.xml': [('emboss', '5.0.0', 'package', '\nreadme blah blah blah\n')]} tool_shed_accessible, missing_tool_configs_dict = common_util.check_for_missing_tools( app, tool_panel_configs, latest_tool_migration_script_number) else: # It doesn't matter if the tool shed is accessible since there are no migrated tools defined in the local Galaxy instance, but # we have to set the value of tool_shed_accessible to True so that the value of migrate_tools.version can be correctly set in # the database. tool_shed_accessible = True missing_tool_configs_dict = odict() have_tool_dependencies = False for k, v in missing_tool_configs_dict.items(): if v: have_tool_dependencies = True break config_arg = '' if os.path.abspath(os.path.join( os.getcwd(), 'universe_wsgi.ini')) != galaxy_config_file: config_arg = ' -c %s' % galaxy_config_file.replace( os.path.abspath(os.getcwd()), '.') if not app.config.running_functional_tests: if tool_shed_accessible: # Automatically update the value of the migrate_tools.version database table column. cmd = 'sh manage_tools.sh%s upgrade' % config_arg proc = subprocess.Popen(args=cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) return_code = proc.wait() output = proc.stdout.read(32768) if return_code != 0: raise Exception( "Error attempting to update the value of migrate_tools.version: %s" % output) elif missing_tool_configs_dict: if len(tool_panel_configs) == 1: plural = '' tool_panel_config_file_names = tool_panel_configs[0] else: plural = 's' tool_panel_config_file_names = ', '.join( tool_panel_configs) msg = "\n>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>" msg += "\n\nThe list of files at the end of this message refers to tools that are configured to load into the tool panel for\n" msg += "this Galaxy instance, but have been removed from the Galaxy distribution. These tools and their dependencies can be\n" msg += "automatically installed from the Galaxy tool shed at http://toolshed.g2.bx.psu.edu.\n\n" msg += "To skip this process, attempt to start your Galaxy server again (e.g., sh run.sh or whatever you use). If you do this,\n" msg += "be aware that these tools will no longer be available in your Galaxy tool panel, and entries for each of them should\n" msg += "be removed from your file%s named %s.\n\n" % ( plural, tool_panel_config_file_names) msg += "CRITICAL NOTE IF YOU PLAN TO INSTALL\n" msg += "The location in which the tool repositories will be installed is the value of the 'tool_path' attribute in the <tool>\n" msg += 'tag of the file named ./migrated_tool_conf.xml (i.e., <toolbox tool_path="../shed_tools">). The default location\n' msg += "setting is '../shed_tools', which may be problematic for some cluster environments, so make sure to change it before\n" msg += "you execute the installation process if appropriate. The configured location must be outside of the Galaxy installation\n" msg += "directory or it must be in a sub-directory protected by a properly configured .hgignore file if the directory is within\n" msg += "the Galaxy installation directory hierarchy. This is because tool shed repositories will be installed using mercurial's\n" msg += "clone feature, which creates .hg directories and associated mercurial repository files. Not having .hgignore properly\n" msg += "configured could result in undesired behavior when modifying or updating your local Galaxy instance or the tool shed\n" msg += "repositories if they are in directories that pose conflicts. See mercurial's .hgignore documentation at the following\n" msg += "URL for details.\n\nhttp://mercurial.selenic.com/wiki/.hgignore\n\n" if have_tool_dependencies: msg += "The following tool dependencies can also optionally be installed (see the option flag in the command below). If you\n" msg += "choose to install them (recommended), they will be installed within the location specified by the 'tool_dependency_dir'\n" msg += "setting in your main Galaxy configuration file (e.g., uninverse_wsgi.ini).\n" processed_tool_dependencies = [] for missing_tool_config, tool_dependencies in missing_tool_configs_dict.items( ): for tool_dependencies_tup in missing_tool_configs_dict[ missing_tool_config]['tool_dependencies']: if tool_dependencies_tup not in processed_tool_dependencies: msg += "------------------------------------\n" msg += "Tool Dependency\n" msg += "------------------------------------\n" msg += "Name: %s, Version: %s, Type: %s\n" % ( tool_dependencies_tup[0], tool_dependencies_tup[1], tool_dependencies_tup[2]) if len(tool_dependencies_tup) >= 4: msg += "Requirements and installation information:\n" msg += "%s\n" % tool_dependencies_tup[3] else: msg += "\n" msg += "------------------------------------\n" processed_tool_dependencies.append( tool_dependencies_tup) msg += "\n" msg += "%s" % output.replace('done', '') msg += "vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv\n" msg += "sh ./scripts/migrate_tools/%04d_tools.sh\n" % latest_tool_migration_script_number msg += "^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\n" if have_tool_dependencies: msg += "The tool dependencies listed above will be installed along with the repositories if you add the 'install_dependencies'\n" msg += "option to the above command like this:\n\n" msg += "vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv\n" msg += "sh ./scripts/migrate_tools/%04d_tools.sh install_dependencies\n" % latest_tool_migration_script_number msg += "^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\n" msg += "Tool dependencies can be installed after the repositories have been installed as well.\n\n" msg += "After the installation process finishes, you can start your Galaxy server. As part of this installation process,\n" msg += "entries for each of the following tool config files will be added to the file named ./migrated_tool_conf.xml, so these\n" msg += "tools will continue to be loaded into your tool panel. Because of this, existing entries for these tools have been\n" msg += "removed from your file%s named %s.\n\n" % ( plural, tool_panel_config_file_names) for missing_tool_config, tool_dependencies in missing_tool_configs_dict.items( ): msg += "%s\n" % missing_tool_config msg += "<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<\n" raise Exception(msg) else: log.debug( "The main Galaxy tool shed is not currently available, so skipped tool migration %s until next server startup" % db_schema.version) else: log.info("At migrate_tools version %d" % db_schema.version)
import os import ConfigParser from optparse import OptionParser new_path = [ os.path.join( os.getcwd(), "lib" ) ] new_path.extend( sys.path[1:] ) # remove scripts/ from the path sys.path = new_path from galaxy.util import pretty_print_time_interval import galaxy.webapps.tool_shed.model.mapping from galaxy.webapps.tool_shed import config from galaxy.webapps.tool_shed import model from galaxy.tools.loader_directory import load_tool_elements_from_path from galaxy import eggs eggs.require( "SQLAlchemy" ) eggs.require( "Whoosh" ) from whoosh.filedb.filestore import FileStorage from whoosh.fields import Schema, STORED, TEXT repo_schema = Schema( id=STORED, name=TEXT( stored=True ), description=TEXT( stored=True ), long_description=TEXT( stored=True ), homepage_url=TEXT( stored=True ), remote_repository_url=TEXT( stored=True ), repo_owner_username=TEXT( stored=True ), times_downloaded=STORED, approved=STORED, last_updated=STORED,
def create_or_verify_database(url, engine_options={}): """ Check that the database is use-able, possibly creating it if empty (this is the only time we automatically create tables, otherwise we force the user to do it using the management script so they can create backups). 1) Empty database --> initialize with latest version and return 2) Database older than migration support --> fail and require manual update 3) Database at state where migrate support introduced --> add version control information but make no changes (might still require manual update) 4) Database versioned but out of date --> fail with informative message, user must run "sh manage_db.sh upgrade" """ dialect = (url.split(':', 1))[0] try: egg = dialect_to_egg[dialect] try: eggs.require(egg) log.debug("%s egg successfully loaded for %s dialect" % (egg, dialect)) except: # If the module is in the path elsewhere (i.e. non-egg), it'll still load. log.warning( "%s egg not found, but an attempt will be made to use %s anyway" % (egg, dialect)) except KeyError: # Let this go, it could possibly work with db's we don't support log.error( "database_connection contains an unknown SQLAlchemy database dialect: %s" % dialect) # Create engine and metadata engine = create_engine(url, **engine_options) meta = MetaData(bind=engine) # Try to load dataset table try: galaxy_user_table = Table("galaxy_user", meta, autoload=True) except NoSuchTableError: # No 'galaxy_user' table means a completely uninitialized database, which # is fine, init the database in a versioned state log.info("No database, initializing") # Database might or might not be versioned try: # Declare the database to be under a repository's version control db_schema = schema.ControlledSchema.create(engine, migrate_repository) except: # The database is already under version control db_schema = schema.ControlledSchema(engine, migrate_repository) # Apply all scripts to get to current version migrate_to_current_version(engine, db_schema) return try: version_table = Table("migrate_version", meta, autoload=True) except NoSuchTableError: # The database exists but is not yet under migrate version control, so init with version 1 log.info("Adding version control to existing database") try: metadata_file_table = Table("metadata_file", meta, autoload=True) schema.ControlledSchema.create(engine, migrate_repository, version=2) except NoSuchTableError: schema.ControlledSchema.create(engine, migrate_repository, version=1) # Verify that the code and the DB are in sync db_schema = schema.ControlledSchema(engine, migrate_repository) if migrate_repository.versions.latest != db_schema.version: exception_msg = "Your database has version '%d' but this code expects version '%d'. " % ( db_schema.version, migrate_repository.versions.latest) exception_msg += "Back up your database and then migrate the schema by running the following from your Galaxy installation directory:" exception_msg += "\n\nsh manage_db.sh upgrade tool_shed\n" raise Exception(exception_msg) else: log.info("At database version %d" % db_schema.version)
from __future__ import absolute_import import logging try: from urllib2 import urlopen except ImportError: from urllib.request import urlopen try: from urllib2 import Request except ImportError: from urllib.request import Request try: from galaxy import eggs eggs.require("poster") except ImportError: pass try: import poster except ImportError: poster = None POSTER_UNAVAILABLE_MESSAGE = "Pulsar configured to use poster module - but it is unavailable. Please install poster." log = logging.getLogger(__name__) if poster is not None: poster.streaminghttp.register_openers()
import tempfile import threading import urlparse from galaxy.util import json from datetime import datetime from email.MIMEText import MIMEText from os.path import relpath from hashlib import md5 from itertools import izip from galaxy import eggs eggs.require( 'docutils' ) import docutils.core import docutils.writers.html4css1 from xml.etree import ElementTree, ElementInclude eggs.require( "wchartype" ) import wchartype from .inflection import Inflector, English inflector = Inflector(English) log = logging.getLogger(__name__) _lock = threading.RLock() CHUNK_SIZE = 65536 # 64k
import logging import operator import os from galaxy import util from galaxy.util.bunch import Bunch from galaxy.util.hash_util import new_secure_hash from galaxy.model.item_attrs import Dictifiable import tool_shed.repository_types.util as rt_util from galaxy import eggs eggs.require("mercurial") from mercurial import hg from mercurial import ui log = logging.getLogger(__name__) class APIKeys(object): pass class User(object, Dictifiable): dict_collection_visible_keys = ("id", "email") dict_element_visible_keys = ("id", "email", "username") def __init__(self, email=None, password=None): self.email = email self.password = password self.external = False self.deleted = False
import ConfigParser as configparser import galaxy.app from galaxy.config import process_is_uwsgi import galaxy.model import galaxy.model.mapping import galaxy.datatypes.registry import galaxy.web.framework import galaxy.web.framework.webapp from galaxy import util from galaxy.util import asbool from galaxy.util.properties import load_app_properties from galaxy import eggs eggs.require('Paste') from paste import httpexceptions import logging log = logging.getLogger( __name__ ) try: from uwsgidecorators import postfork except: # TODO: Make this function more like flask's @before_first_request w/ # registered methods etc. def pf_dec(func): return func postfork = pf_dec
def create_or_verify_database( url, galaxy_config_file, engine_options={}, app=None ): """ Check that the database is use-able, possibly creating it if empty (this is the only time we automatically create tables, otherwise we force the user to do it using the management script so they can create backups). 1) Empty database --> initialize with latest version and return 2) Database older than migration support --> fail and require manual update 3) Database at state where migrate support introduced --> add version control information but make no changes (might still require manual update) 4) Database versioned but out of date --> fail with informative message, user must run "sh manage_db.sh upgrade" """ dialect = ( url.split( ':', 1 ) )[0] try: egg = dialect_to_egg[dialect] try: eggs.require( egg ) log.debug( "%s egg successfully loaded for %s dialect" % ( egg, dialect ) ) except: # If the module is in the path elsewhere (i.e. non-egg), it'll still load. log.warning( "%s egg not found, but an attempt will be made to use %s anyway" % ( egg, dialect ) ) except KeyError: # Let this go, it could possibly work with db's we don't support log.error( "database_connection contains an unknown SQLAlchemy database dialect: %s" % dialect ) # Create engine and metadata engine = create_engine( url, **engine_options ) meta = MetaData( bind=engine ) # Try to load dataset table try: dataset_table = Table( "dataset", meta, autoload=True ) except NoSuchTableError: # No 'dataset' table means a completely uninitialized database. If we have an app, we'll # set it's new_installation setting to True so the tool migration process will be skipped. if app: app.new_installation = True log.info( "No database, initializing" ) # Database might or might not be versioned try: # Declare the database to be under a repository's version control db_schema = schema.ControlledSchema.create( engine, migrate_repository ) except: # The database is already under version control db_schema = schema.ControlledSchema( engine, migrate_repository ) # Apply all scripts to get to current version migrate_to_current_version( engine, db_schema ) return try: hda_table = Table( "history_dataset_association", meta, autoload=True ) except NoSuchTableError: raise Exception( "Your database is older than hg revision 1464:c7acaa1bb88f and will need to be updated manually" ) # There is a 'history_dataset_association' table, so we (hopefully) have # version 1 of the database, but without the migrate_version table. This # happens if the user has a build from right before migration was added. # Verify that this is true, if it is any older they'll have to update # manually if 'copied_from_history_dataset_association_id' not in hda_table.c: # The 'copied_from_history_dataset_association_id' column was added in # rev 1464:c7acaa1bb88f. This is the oldest revision we currently do # automated versioning for, so stop here raise Exception( "Your database is older than hg revision 1464:c7acaa1bb88f and will need to be updated manually" ) # At revision 1464:c7acaa1bb88f or greater (database version 1), make sure # that the db has version information. This is the trickiest case -- we # have a database but no version control, and are assuming it is a certain # version. If the user has postion version 1 changes this could cause # problems try: version_table = Table( "migrate_version", meta, autoload=True ) except NoSuchTableError: # The database exists but is not yet under migrate version control, so init with version 1 log.info( "Adding version control to existing database" ) try: metadata_file_table = Table( "metadata_file", meta, autoload=True ) schema.ControlledSchema.create( engine, migrate_repository, version=2 ) except NoSuchTableError: schema.ControlledSchema.create( engine, migrate_repository, version=1 ) # Verify that the code and the DB are in sync db_schema = schema.ControlledSchema( engine, migrate_repository ) if migrate_repository.versions.latest != db_schema.version: config_arg = '' if os.path.abspath( os.path.join( os.getcwd(), 'universe_wsgi.ini' ) ) != galaxy_config_file: config_arg = ' -c %s' % galaxy_config_file.replace( os.path.abspath( os.getcwd() ), '.' ) raise Exception( "Your database has version '%d' but this code expects version '%d'. Please backup your database and then migrate the schema by running 'sh manage_db.sh%s upgrade'." % ( db_schema.version, migrate_repository.versions.latest, config_arg ) ) else: log.info( "At database version %d" % db_schema.version )
def wrap_in_middleware( app, global_conf, **local_conf ): """ Based on the configuration wrap `app` in a set of common and useful middleware. """ # Merge the global and local configurations conf = global_conf.copy() conf.update(local_conf) debug = asbool( conf.get( 'debug', False ) ) # First put into place httpexceptions, which must be most closely # wrapped around the application (it can interact poorly with # other middleware): app = httpexceptions.make_middleware( app, conf ) log.debug( "Enabling 'httpexceptions' middleware" ) # If we're using remote_user authentication, add middleware that # protects Galaxy from improperly configured authentication in the # upstream server if asbool(conf.get( 'use_remote_user', False )): from galaxy.web.framework.middleware.remoteuser import RemoteUser app = RemoteUser( app, maildomain=conf.get( 'remote_user_maildomain', None ), display_servers=util.listify( conf.get( 'display_servers', '' ) ), admin_users=conf.get( 'admin_users', '' ).split( ',' ), remote_user_header=conf.get( 'remote_user_header', 'HTTP_REMOTE_USER' ), remote_user_secret_header=conf.get('remote_user_secret', None) ) # The recursive middleware allows for including requests in other # requests or forwarding of requests, all on the server side. if asbool(conf.get('use_recursive', True)): from paste import recursive app = recursive.RecursiveMiddleware( app, conf ) log.debug( "Enabling 'recursive' middleware" ) # If sentry logging is enabled, log here before propogating up to # the error middleware sentry_dsn = conf.get( 'sentry_dsn', None ) if sentry_dsn: from galaxy.web.framework.middleware.sentry import Sentry app = Sentry( app, sentry_dsn ) # Various debug middleware that can only be turned on if the debug # flag is set, either because they are insecure or greatly hurt # performance if debug: # Middleware to check for WSGI compliance if asbool( conf.get( 'use_lint', False ) ): from paste import lint app = lint.make_middleware( app, conf ) log.debug( "Enabling 'lint' middleware" ) # Middleware to run the python profiler on each request if asbool( conf.get( 'use_profile', False ) ): from paste.debug import profile app = profile.ProfileMiddleware( app, conf ) log.debug( "Enabling 'profile' middleware" ) if debug and asbool( conf.get( 'use_interactive', False ) ) and not process_is_uwsgi: # Interactive exception debugging, scary dangerous if publicly # accessible, if not enabled we'll use the regular error printing # middleware. eggs.require( "WebError" ) from weberror import evalexception app = evalexception.EvalException( app, conf, templating_formatters=build_template_error_formatters() ) log.debug( "Enabling 'eval exceptions' middleware" ) else: if debug and asbool( conf.get( 'use_interactive', False ) ) and process_is_uwsgi: log.error("Interactive debugging middleware is enabled in your configuration " "but this is a uwsgi process. Refusing to wrap in interactive error middleware.") # Not in interactive debug mode, just use the regular error middleware import galaxy.web.framework.middleware.error app = galaxy.web.framework.middleware.error.ErrorMiddleware( app, conf ) log.debug( "Enabling 'error' middleware" ) # Transaction logging (apache access.log style) if asbool( conf.get( 'use_translogger', True ) ): from galaxy.web.framework.middleware.translogger import TransLogger app = TransLogger( app ) log.debug( "Enabling 'trans logger' middleware" ) # X-Forwarded-Host handling from galaxy.web.framework.middleware.xforwardedhost import XForwardedHostMiddleware app = XForwardedHostMiddleware( app ) log.debug( "Enabling 'x-forwarded-host' middleware" ) # Request ID middleware from galaxy.web.framework.middleware.request_id import RequestIDMiddleware app = RequestIDMiddleware( app ) log.debug( "Enabling 'Request ID' middleware" ) return app
"""Binary classes""" import binascii import gzip import logging import os import shutil import struct import subprocess import tempfile import warnings import zipfile from galaxy import eggs eggs.require( "bx-python" ) from bx.seq.twobit import TWOBIT_MAGIC_NUMBER, TWOBIT_MAGIC_NUMBER_SWAP, TWOBIT_MAGIC_SIZE from galaxy.datatypes.metadata import MetadataElement, MetadataParameter, ListParameter, DictParameter from galaxy.datatypes import metadata from galaxy.util import nice_size, sqlite from . import data, dataproviders with warnings.catch_warnings(): warnings.simplefilter( "ignore" ) eggs.require( "pysam" ) from pysam import csamtools log = logging.getLogger(__name__) # Currently these supported binary data types must be manually set on upload
import sys import os import pprint import unittest __GALAXY_ROOT__ = os.getcwd() + '/../../../' sys.path.append( __GALAXY_ROOT__ + 'lib' ) from galaxy import eggs eggs.require( 'SQLAlchemy >= 0.4' ) import sqlalchemy from galaxy import model from galaxy import exceptions from galaxy.util.bunch import Bunch import mock from test_ModelManager import BaseTestCase from galaxy.managers.histories import HistoryManager from galaxy.managers.datasets import DatasetManager from galaxy.managers.hdas import HDAManager # ============================================================================= default_password = '******' user2_data = dict( email='[email protected]', username='******', password=default_password ) user3_data = dict( email='[email protected]', username='******', password=default_password ) # ============================================================================= class HDAManagerTestCase( BaseTestCase ):
""" Galaxy Metadata """ from galaxy import eggs eggs.require("simplejson") import copy import cPickle import logging import os import shutil import simplejson import sys import tempfile import weakref from os.path import abspath import galaxy.model from galaxy.util import listify, stringify_dictionary_keys, string_as_bool from galaxy.util.odict import odict from galaxy.web import form_builder from sqlalchemy.orm import object_session log = logging.getLogger(__name__) STATEMENTS = "__galaxy_statements__" #this is the name of the property in a Datatype class where new metadata spec element Statements are stored class Statement( object ): """
""" Galaxy control queue and worker. This is used to handle 'app' control like reloading the toolbox, etc., across multiple processes. """ import logging import threading import sys import galaxy.queues from galaxy import eggs, util eggs.require('anyjson') if sys.version_info < (2, 7, 0): # Kombu requires importlib and ordereddict to function under Python 2.6. eggs.require('importlib') eggs.require('ordereddict') eggs.require('kombu') from kombu import Connection from kombu.mixins import ConsumerMixin from kombu.pools import producers log = logging.getLogger(__name__) class GalaxyQueueWorker(ConsumerMixin, threading.Thread): """ This is a flexible worker for galaxy's queues. Each process, web or handler, will have one of these used for dispatching so called 'control' tasks. """
def verify_tools( app, url, galaxy_config_file, engine_options={} ): # Check the value in the migrate_tools.version database table column to verify that the number is in # sync with the number of version scripts in ~/lib/galaxy/tools/migrate/versions. dialect = ( url.split( ':', 1 ) )[0] try: egg = dialect_to_egg[ dialect ] try: eggs.require( egg ) log.debug( "%s egg successfully loaded for %s dialect" % ( egg, dialect ) ) except: # If the module is in the path elsewhere (i.e. non-egg), it'll still load. log.warning( "%s egg not found, but an attempt will be made to use %s anyway" % ( egg, dialect ) ) except KeyError: # Let this go, it could possibly work with db's we don't support log.error( "database_connection contains an unknown SQLAlchemy database dialect: %s" % dialect ) # Create engine and metadata engine = create_engine( url, **engine_options ) meta = MetaData( bind=engine ) # The migrate_tools table was created in database version script 0092_add_migrate_tools_table.py. version_table = Table( "migrate_tools", meta, autoload=True ) # Verify that the code and the database are in sync. db_schema = schema.ControlledSchema( engine, migrate_repository ) latest_tool_migration_script_number = migrate_repository.versions.latest if latest_tool_migration_script_number != db_schema.version: # The default behavior is that the tool shed is down. tool_shed_accessible = False if app.new_installation: # New installations will not be missing tools, so we don't need to worry about them. missing_tool_configs_dict = odict() else: tool_panel_configs = common_util.get_non_shed_tool_panel_configs( app ) if tool_panel_configs: # The missing_tool_configs_dict contents are something like: # {'emboss_antigenic.xml': [('emboss', '5.0.0', 'package', '\nreadme blah blah blah\n')]} tool_shed_accessible, missing_tool_configs_dict = common_util.check_for_missing_tools( app, tool_panel_configs, latest_tool_migration_script_number ) else: # It doesn't matter if the tool shed is accessible since there are no migrated tools defined in the local Galaxy instance, but # we have to set the value of tool_shed_accessible to True so that the value of migrate_tools.version can be correctly set in # the database. tool_shed_accessible = True missing_tool_configs_dict = odict() have_tool_dependencies = False for k, v in missing_tool_configs_dict.items(): if v: have_tool_dependencies = True break config_arg = '' if os.path.abspath( os.path.join( os.getcwd(), 'galaxy.ini' ) ) != galaxy_config_file: config_arg = ' -c %s' % galaxy_config_file.replace( os.path.abspath( os.getcwd() ), '.' ) if not app.config.running_functional_tests: if tool_shed_accessible: # Automatically update the value of the migrate_tools.version database table column. cmd = 'sh manage_tools.sh%s upgrade' % config_arg proc = subprocess.Popen( args=cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT ) return_code = proc.wait() output = proc.stdout.read( 32768 ) if return_code != 0: raise Exception( "Error attempting to update the value of migrate_tools.version: %s" % output ) elif missing_tool_configs_dict: if len( tool_panel_configs ) == 1: plural = '' tool_panel_config_file_names = tool_panel_configs[ 0 ] else: plural = 's' tool_panel_config_file_names = ', '.join( tool_panel_configs ) msg = "\n>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>" msg += "\n\nThe list of files at the end of this message refers to tools that are configured to load into the tool panel for\n" msg += "this Galaxy instance, but have been removed from the Galaxy distribution. These tools and their dependencies can be\n" msg += "automatically installed from the Galaxy tool shed at http://toolshed.g2.bx.psu.edu.\n\n" msg += "To skip this process, attempt to start your Galaxy server again (e.g., sh run.sh or whatever you use). If you do this,\n" msg += "be aware that these tools will no longer be available in your Galaxy tool panel, and entries for each of them should\n" msg += "be removed from your file%s named %s.\n\n" % ( plural, tool_panel_config_file_names ) msg += "CRITICAL NOTE IF YOU PLAN TO INSTALL\n" msg += "The location in which the tool repositories will be installed is the value of the 'tool_path' attribute in the <tool>\n" msg += 'tag of the file named ./migrated_tool_conf.xml (i.e., <toolbox tool_path="../shed_tools">). The default location\n' msg += "setting is '../shed_tools', which may be problematic for some cluster environments, so make sure to change it before\n" msg += "you execute the installation process if appropriate. The configured location must be outside of the Galaxy installation\n" msg += "directory or it must be in a sub-directory protected by a properly configured .hgignore file if the directory is within\n" msg += "the Galaxy installation directory hierarchy. This is because tool shed repositories will be installed using mercurial's\n" msg += "clone feature, which creates .hg directories and associated mercurial repository files. Not having .hgignore properly\n" msg += "configured could result in undesired behavior when modifying or updating your local Galaxy instance or the tool shed\n" msg += "repositories if they are in directories that pose conflicts. See mercurial's .hgignore documentation at the following\n" msg += "URL for details.\n\nhttp://mercurial.selenic.com/wiki/.hgignore\n\n" if have_tool_dependencies: msg += "The following tool dependencies can also optionally be installed (see the option flag in the command below). If you\n" msg += "choose to install them (recommended), they will be installed within the location specified by the 'tool_dependency_dir'\n" msg += "setting in your main Galaxy configuration file (e.g., uninverse_wsgi.ini).\n" processed_tool_dependencies = [] for missing_tool_config, tool_dependencies in missing_tool_configs_dict.items(): for tool_dependencies_tup in missing_tool_configs_dict[ missing_tool_config ][ 'tool_dependencies' ]: if tool_dependencies_tup not in processed_tool_dependencies: msg += "------------------------------------\n" msg += "Tool Dependency\n" msg += "------------------------------------\n" msg += "Name: %s, Version: %s, Type: %s\n" % ( tool_dependencies_tup[ 0 ], tool_dependencies_tup[ 1 ], tool_dependencies_tup[ 2 ] ) if len( tool_dependencies_tup ) >= 4: msg += "Requirements and installation information:\n" msg += "%s\n" % tool_dependencies_tup[ 3 ] else: msg += "\n" msg += "------------------------------------\n" processed_tool_dependencies.append( tool_dependencies_tup ) msg += "\n" msg += "%s" % output.replace( 'done', '' ) msg += "vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv\n" msg += "sh ./scripts/migrate_tools/%04d_tools.sh\n" % latest_tool_migration_script_number msg += "^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\n" if have_tool_dependencies: msg += "The tool dependencies listed above will be installed along with the repositories if you add the 'install_dependencies'\n" msg += "option to the above command like this:\n\n" msg += "vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv\n" msg += "sh ./scripts/migrate_tools/%04d_tools.sh install_dependencies\n" % latest_tool_migration_script_number msg += "^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\n" msg += "Tool dependencies can be installed after the repositories have been installed as well.\n\n" msg += "After the installation process finishes, you can start your Galaxy server. As part of this installation process,\n" msg += "entries for each of the following tool config files will be added to the file named ./migrated_tool_conf.xml, so these\n" msg += "tools will continue to be loaded into your tool panel. Because of this, existing entries for these tools have been\n" msg += "removed from your file%s named %s.\n\n" % ( plural, tool_panel_config_file_names ) for missing_tool_config, tool_dependencies in missing_tool_configs_dict.items(): msg += "%s\n" % missing_tool_config msg += "<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<\n" raise Exception( msg ) else: log.debug( "The main Galaxy tool shed is not currently available, so skipped tool migration %s until next server startup" % db_schema.version ) else: log.info( "At migrate_tools version %d" % db_schema.version )
import inspect from traceback import format_exc from functools import wraps from galaxy import eggs eggs.require("Paste") import paste.httpexceptions from galaxy.web.framework import url_for from galaxy import util from galaxy.exceptions import error_codes from galaxy.exceptions import MessageException from galaxy.util.json import loads from galaxy.util.json import safe_dumps as dumps import logging log = logging.getLogger(__name__) JSON_CONTENT_TYPE = "application/json" def error(message): raise MessageException(message, type='error') # ----------------------------------------------------------------------------- web controller decorators def _save_orig_fn(wrapped, orig): if not hasattr(orig, '_orig'): wrapped._orig = orig return wrapped
#!/usr/bin/env python """ Convert from pileup file to interval index file. usage: %prog <options> in_file out_file """ from __future__ import division import optparse from galaxy import eggs eggs.require( "bx-python" ) from bx.interval_index_file import Indexes def main(): # Read options, args. parser = optparse.OptionParser() (options, args) = parser.parse_args() input_fname, output_fname = args # Do conversion. index = Indexes() offset = 0 for line in open( input_fname, "r" ): chrom, start = line.split()[ 0:2 ] # Pileup format is 1-based. start = int( start ) - 1 index.add( chrom, start, start + 1, offset ) offset += len( line )
import sys import os.path import logging from galaxy import eggs eggs.require("SQLAlchemy") eggs.require("six") # Required by sqlalchemy-migrate eggs.require("sqlparse") # Required by sqlalchemy-migrate eggs.require("decorator") # Required by sqlalchemy-migrate eggs.require("Tempita ") # Required by sqlalchemy-migrate eggs.require("sqlalchemy-migrate") from migrate.versioning import repository, schema from sqlalchemy import create_engine from galaxy.model.orm import dialect_to_egg from sqlalchemy import MetaData from sqlalchemy.exc import NoSuchTableError from sqlalchemy import Table log = logging.getLogger(__name__) # path relative to galaxy migrate_repository_directory = os.path.abspath( os.path.dirname(__file__)).replace(os.getcwd() + os.path.sep, '', 1) migrate_repository = repository.Repository(migrate_repository_directory) def create_or_verify_database(url, galaxy_config_file, engine_options={},
#!/usr/bin/env python import ConfigParser import logging import os import re import sys import argparse new_path = [ os.path.join( os.getcwd(), "lib" ) ] new_path.extend( sys.path[1:] ) sys.path = new_path from galaxy import eggs eggs.require( "SQLAlchemy >= 0.4" ) eggs.require( 'mercurial' ) import galaxy.config from galaxy.web import security from galaxy.model import mapping log = logging.getLogger( __name__ ) VALID_PUBLICNAME_RE = re.compile( "^[a-z0-9\-]+$" ) VALID_EMAIL_RE = re.compile( "[^@]+@[^@]+\.[^@]+" ) class BootstrapGalaxyApplication( object ): """
def create_or_verify_database(url, galaxy_config_file, engine_options={}, app=None): """ Check that the database is use-able, possibly creating it if empty (this is the only time we automatically create tables, otherwise we force the user to do it using the management script so they can create backups). 1) Empty database --> initialize with latest version and return 2) Database older than migration support --> fail and require manual update 3) Database at state where migrate support introduced --> add version control information but make no changes (might still require manual update) 4) Database versioned but out of date --> fail with informative message, user must run "sh manage_db.sh upgrade" """ dialect = (url.split(':', 1))[0] try: egg = dialect_to_egg[dialect] try: eggs.require(egg) log.debug("%s egg successfully loaded for %s dialect" % (egg, dialect)) except: # If the module is in the path elsewhere (i.e. non-egg), it'll still load. log.warning( "%s egg not found, but an attempt will be made to use %s anyway" % (egg, dialect)) except KeyError: # Let this go, it could possibly work with db's we don't support log.error( "database_connection contains an unknown SQLAlchemy database dialect: %s" % dialect) # Create engine and metadata engine = create_engine(url, **engine_options) def migrate(): try: # Declare the database to be under a repository's version control db_schema = schema.ControlledSchema.create(engine, migrate_repository) except: # The database is already under version control db_schema = schema.ControlledSchema(engine, migrate_repository) # Apply all scripts to get to current version migrate_to_current_version(engine, db_schema) meta = MetaData(bind=engine) if app and getattr(app.config, 'database_auto_migrate', False): migrate() return # Try to load dataset table try: Table("dataset", meta, autoload=True) except NoSuchTableError: # No 'dataset' table means a completely uninitialized database. If we have an app, we'll # set its new_installation setting to True so the tool migration process will be skipped. if app: app.new_installation = True log.info("No database, initializing") migrate() return try: hda_table = Table("history_dataset_association", meta, autoload=True) except NoSuchTableError: raise Exception( "Your database is older than hg revision 1464:c7acaa1bb88f and will need to be updated manually" ) # There is a 'history_dataset_association' table, so we (hopefully) have # version 1 of the database, but without the migrate_version table. This # happens if the user has a build from right before migration was added. # Verify that this is true, if it is any older they'll have to update # manually if 'copied_from_history_dataset_association_id' not in hda_table.c: # The 'copied_from_history_dataset_association_id' column was added in # rev 1464:c7acaa1bb88f. This is the oldest revision we currently do # automated versioning for, so stop here raise Exception( "Your database is older than hg revision 1464:c7acaa1bb88f and will need to be updated manually" ) # At revision 1464:c7acaa1bb88f or greater (database version 1), make sure # that the db has version information. This is the trickiest case -- we # have a database but no version control, and are assuming it is a certain # version. If the user has postion version 1 changes this could cause # problems try: Table("migrate_version", meta, autoload=True) except NoSuchTableError: # The database exists but is not yet under migrate version control, so init with version 1 log.info("Adding version control to existing database") try: Table("metadata_file", meta, autoload=True) schema.ControlledSchema.create(engine, migrate_repository, version=2) except NoSuchTableError: schema.ControlledSchema.create(engine, migrate_repository, version=1) # Verify that the code and the DB are in sync db_schema = schema.ControlledSchema(engine, migrate_repository) if migrate_repository.versions.latest != db_schema.version: config_arg = '' if os.path.abspath(os.path.join(os.getcwd(), 'config', 'galaxy.ini')) != galaxy_config_file: config_arg = ' -c %s' % galaxy_config_file.replace( os.path.abspath(os.getcwd()), '.') raise Exception( "Your database has version '%d' but this code expects version '%d'. Please backup your database and then migrate the schema by running 'sh manage_db.sh%s upgrade'." % (db_schema.version, migrate_repository.versions.latest, config_arg)) else: log.info("At database version %d" % db_schema.version)
"""Module for searching the toolshed repositories""" import datetime from galaxy import exceptions from galaxy import eggs from galaxy.webapps.tool_shed import model import logging log = logging.getLogger(__name__) # Whoosh is compatible with Python 2.5+ # Try to import Whoosh and set flag to indicate whether # the tool search is ready. try: eggs.require("Whoosh") import whoosh.index from whoosh import scoring from whoosh.fields import Schema, STORED, ID, KEYWORD, TEXT, STORED from whoosh.scoring import BM25F from whoosh.qparser import MultifieldParser from whoosh.index import Index search_ready = True schema = Schema(id=STORED, name=TEXT(field_boost=1.7, stored=True), description=TEXT(field_boost=1.5, stored=True), long_description=TEXT(stored=True), homepage_url=TEXT(stored=True), remote_repository_url=TEXT(stored=True), repo_owner_username=TEXT(stored=True), times_downloaded=STORED, approved=STORED, last_updated=STORED,
""" Code to support database helper scripts (create_db.py, manage_db.py, etc...). """ import logging import os.path from ConfigParser import SafeConfigParser from galaxy import eggs eggs.require("decorator") eggs.require("Tempita") eggs.require("SQLAlchemy") eggs.require("sqlalchemy_migrate") from galaxy.model.orm import dialect_to_egg import pkg_resources log = logging.getLogger(__name__) DEFAULT_CONFIG_FILE = 'universe_wsgi.ini' DEFAULT_CONFIG_PREFIX = '' DEFAULT_DATABASE = 'galaxy' DATABASE = { "galaxy": { 'repo': 'lib/galaxy/model/migrate', 'default_sqlite_file': './database/universe.sqlite', }, "tool_shed": { 'repo': 'lib/galaxy/webapps/tool_shed/model/migrate',
from __future__ import absolute_import try: from galaxy import eggs eggs.require("requests") except ImportError: pass try: import requests except ImportError: requests = None requests_multipart_post_available = False try: import requests_toolbelt requests_multipart_post_available = True except ImportError: requests_toolbelt = None REQUESTS_UNAVAILABLE_MESSAGE = "Pulsar configured to use requests module - but it is unavailable. Please install requests." REQUESTS_TOOLBELT_UNAVAILABLE_MESSAGE = "Pulsar configured to use requests_toolbelt module - but it is unavailable. Please install requests_toolbelt." import logging log = logging.getLogger(__name__) def post_file(url, path): if requests_toolbelt is None: raise ImportError(REQUESTS_TOOLBELT_UNAVAILABLE_MESSAGE) __ensure_requests()
import tempfile import re from ConfigParser import SafeConfigParser # Assume we are run from the galaxy root directory, add lib to the python path cwd = os.getcwd() new_path = [os.path.join(cwd, "lib"), os.path.join(cwd, "test")] new_path.extend(sys.path[1:]) sys.path = new_path from base.tool_shed_util import parse_tool_panel_config from galaxy import eggs from galaxy.util.properties import load_app_properties eggs.require("nose") eggs.require("NoseHTML") eggs.require("NoseTestDiff") eggs.require("twill==0.9") eggs.require("Paste") eggs.require("PasteDeploy") eggs.require("Cheetah") # this should not be required, but it is under certain conditions, thanks to this bug: # http://code.google.com/p/python-nose/issues/detail?id=284 eggs.require("pysqlite") import atexit import logging import os.path import twill
import smtplib import stat import string import sys import tempfile import threading from email.MIMEText import MIMEText from os.path import relpath from hashlib import md5 from itertools import izip from galaxy import eggs eggs.require( 'docutils' ) import docutils.core import docutils.writers.html4css1 eggs.require( 'elementtree' ) from elementtree import ElementTree, ElementInclude eggs.require( "wchartype" ) import wchartype from .inflection import Inflector, English inflector = Inflector(English) log = logging.getLogger(__name__) _lock = threading.RLock()
""" Galaxy web framework helpers """ import time from cgi import escape from datetime import datetime, timedelta from galaxy import eggs from galaxy.util import hash_util from galaxy.util.json import dumps eggs.require( "MarkupSafe" ) #required by WebHelpers eggs.require( "WebHelpers" ) from webhelpers import date from webhelpers.html.tags import stylesheet_link, javascript_link eggs.require( "Routes" ) from routes import url_for server_starttime = int(time.time()) def time_ago( x ): """ Convert a datetime to a string. """ delta = timedelta(weeks=1) # If the date is more than one week ago, then display the actual date instead of in words if (datetime.utcnow() - x) > delta: # Greater than a week difference return x.strftime("%b %d, %Y") else: date_array = date.distance_of_time_in_words( x, datetime.utcnow() ).replace(",", "").split(" ")
from __future__ import absolute_import import logging from galaxy import eggs eggs.require('SQLAlchemy') from sqlalchemy import false eggs.require( "MarkupSafe" ) from markupsafe import escape from galaxy import model, util from galaxy.web.base.controller import BaseUIController, UsesFormDefinitionsMixin, web from galaxy.web.form_builder import build_select_field, TextField from galaxy.web.framework.helpers import iff, grids from .requests_common import invalid_id_redirect log = logging.getLogger( __name__ ) class RequestTypeGrid( grids.Grid ): # Custom column types class NameColumn( grids.TextColumn ): def get_value(self, trans, grid, request_type): return escape(request_type.name) class DescriptionColumn( grids.TextColumn ): def get_value(self, trans, grid, request_type): return escape(request_type.desc) class RequestFormColumn( grids.TextColumn ): def get_value(self, trans, grid, request_type):
""" import copy import cPickle import json import os import shutil import sys import tempfile import weakref from os.path import abspath from galaxy import eggs eggs.require("SQLAlchemy >= 0.4") from sqlalchemy.orm import object_session import galaxy.model from galaxy.util import listify from galaxy.util.object_wrapper import sanitize_lists_to_string from galaxy.util import stringify_dictionary_keys from galaxy.util import string_as_bool from galaxy.util import in_directory from galaxy.util.odict import odict from galaxy.web import form_builder import logging log = logging.getLogger(__name__) STATEMENTS = "__galaxy_statements__" #this is the name of the property in a Datatype class where new metadata spec element Statements are stored
import os import sys import shutil import logging import inspect import datetime from ConfigParser import ConfigParser from optparse import OptionParser galaxy_root = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) sys.path.insert(0, os.path.join(galaxy_root, "lib")) from galaxy import eggs eggs.require("psycopg2") eggs.require("SQLAlchemy") import psycopg2 from sqlalchemy.engine.url import make_url import galaxy.config from galaxy.exceptions import ObjectNotFound from galaxy.objectstore import build_object_store_from_config from galaxy.util.bunch import Bunch log = logging.getLogger() class MetadataFile(Bunch): pass
from __future__ import absolute_import from galaxy import eggs eggs.require("Mako") import mako.exceptions def build_template_error_formatters(): """ Build a list of template error formatters for WebError. When an error occurs, WebError pass the exception to each function in this list until one returns a value, which will be displayed on the error page. """ formatters = [] # Formatter for mako def mako_html_data(exc_value): if isinstance(exc_value, (mako.exceptions.CompileException, mako.exceptions.SyntaxException)): return mako.exceptions.html_error_template().render(full=False, css=False) if isinstance(exc_value, AttributeError) and exc_value.args[0].startswith( "'Undefined' object has no attribute"): return mako.exceptions.html_error_template().render(full=False, css=False) formatters.append(mako_html_data) return formatters
import logging import os import shutil import sys from string import Template from galaxy.util import unicodify, nice_size from galaxy import eggs eggs.require( 'MarkupSafe' ) import markupsafe log = logging.getLogger( __name__ ) CHUNK_SIZE = 2**20 # 1Mb INSTALLATION_LOG = 'INSTALLATION.log' # Set no activity timeout to 20 minutes. NO_OUTPUT_TIMEOUT = 3600.0 MAXDIFFSIZE = 8000 MAX_DISPLAY_SIZE = 32768 DOCKER_IMAGE_TEMPLATE = ''' # Galaxy Docker image FROM bgruening/galaxy-stable MAINTAINER Bjoern A. Gruning, [email protected] WORKDIR /galaxy-central
#!/usr/bin/env python """ Small library with cheminformatic functions based on openbabel and pgchem. Copyright 2012, Bjoern Gruening and Xavier Lucas """ import os, sys try: from galaxy import eggs eggs.require('psycopg2') except: print( 'psycopg2 is not available. It is currently used in the pgchem wrappers, that are not shipped with default CTB' ) try: import pybel import openbabel except: print( 'OpenBabel could not be found. A few functions are not available without OpenBabel.' ) from multiprocessing import Pool import glob, tempfile, re import subprocess def CountLines(path): out = subprocess.Popen(['wc', '-l', path],
# -*- coding: utf-8 -*- import os import imp import unittest test_utils = imp.load_source( 'test_utils', os.path.join( os.path.dirname( __file__), '../unittest_utils/utility.py' ) ) from galaxy import eggs eggs.require( 'SQLAlchemy >= 0.4' ) import sqlalchemy from galaxy import model from galaxy import exceptions from base import BaseTestCase from galaxy.managers.histories import HistoryManager from galaxy.managers.datasets import DatasetManager from galaxy.managers import hdas # ============================================================================= default_password = '******' user2_data = dict( email='[email protected]', username='******', password=default_password ) user3_data = dict( email='[email protected]', username='******', password=default_password ) # ============================================================================= class HDATestCase( BaseTestCase ):
import tempfile import threading from galaxy.util import json from email.MIMEText import MIMEText from os.path import relpath from hashlib import md5 from itertools import izip from urlparse import urlparse from galaxy import eggs eggs.require( 'docutils' ) import docutils.core import docutils.writers.html4css1 eggs.require( 'elementtree' ) from elementtree import ElementTree, ElementInclude eggs.require( "wchartype" ) import wchartype from .inflection import Inflector, English inflector = Inflector(English) log = logging.getLogger(__name__) _lock = threading.RLock()
from galaxy import eggs eggs.require( "SVGFig" ) import svgfig MARGIN = 5 LINE_SPACING = 15 class WorkflowCanvas( object ): def __init__( self ): self.canvas = svgfig.canvas( style="stroke:black; fill:none; stroke-width:1px; stroke-linejoin:round; text-anchor:left" ) self.text = svgfig.SVG( "g" ) self.connectors = svgfig.SVG( "g" ) self.boxes = svgfig.SVG( "g" ) svgfig.Text.defaults[ "font-size" ] = "10px" self.in_pos = {} self.out_pos = {} self.widths = {} self.max_x = 0 self.max_y = 0 self.max_width = 0 self.data = [] def finish( self ): max_x, max_y, max_width = self.max_x, self.max_y, self.max_width
import json import logging import os import string import subprocess import sys import time from galaxy import eggs from galaxy import model from galaxy.jobs import JobDestination from galaxy.jobs.handler import DEFAULT_JOB_PUT_FAILURE_MESSAGE from galaxy.jobs.runners import AsynchronousJobState, AsynchronousJobRunner eggs.require("drmaa") log = logging.getLogger(__name__) __all__ = ['DRMAAJobRunner'] drmaa = None DRMAA_jobTemplate_attributes = [ 'args', 'remoteCommand', 'outputPath', 'errorPath', 'nativeSpecification', 'jobName', 'email', 'project' ] class DRMAAJobRunner(AsynchronousJobRunner): """
""" """ import datetime import inspect import os import hashlib import random import socket import string import time from Cookie import CookieError from galaxy import eggs eggs.require( "Cheetah" ) from Cheetah.Template import Template eggs.require( "Mako" ) import mako.runtime import mako.lookup # pytz is used by Babel. eggs.require( "pytz" ) eggs.require( "Babel" ) from babel.support import Translations from babel import Locale eggs.require( "SQLAlchemy >= 0.4" ) from sqlalchemy import and_ from sqlalchemy.orm.exc import NoResultFound from sqlalchemy.orm import joinedload from galaxy.exceptions import MessageException from galaxy import util
""" Actions to be run at job completion (or output hda creation, as in the case of immediate_actions listed below. Currently only used in workflows. """ import datetime import logging import socket from galaxy.util import send_mail from galaxy.util.json import dumps from galaxy import eggs eggs.require("MarkupSafe") from markupsafe import escape log = logging.getLogger(__name__) def get_form_template(action_type, title, content, help, on_output=True): if on_output: form = """ if (pja.action_type == "%s"){ p_str = "<div class='pjaForm toolForm'><span class='action_tag' style='display:none'>"+ pja.action_type + pja.output_name + "</span><div class='toolFormTitle'> %s <br/> on " + pja.output_name + "\ <div style='float: right;' class='buttons'><img src='/static/images/history-buttons/delete_icon.png'></div></div><div class='toolFormBody'>"; %s p_str += "</div><div class='toolParamHelp'>%s</div></div>"; }""" % (action_type, title, content, help) else: form = """ if (pja.action_type == "%s"){ p_str = "<div class='pjaForm toolForm'><span class='action_tag' style='display:none'>"+ pja.action_type + "</span><div class='toolFormTitle'> %s \ <div style='float: right;' class='buttons'><img src='/static/images/history-buttons/delete_icon.png'></div></div><div class='toolFormBody'>";
import logging import os import subprocess import sys from galaxy import eggs eggs.require( "decorator" ) eggs.require( "Tempita" ) eggs.require( "six" ) # Required by sqlalchemy-migrate eggs.require( "sqlparse" ) # Required by sqlalchemy-migrate eggs.require( "SQLAlchemy" ) eggs.require( "sqlalchemy_migrate" ) from migrate.versioning import repository from migrate.versioning import schema from sqlalchemy import create_engine from sqlalchemy import MetaData from sqlalchemy import Table from galaxy.util.odict import odict from galaxy.model.orm import dialect_to_egg from tool_shed.util import common_util log = logging.getLogger( __name__ ) # Path relative to galaxy migrate_repository_directory = os.path.dirname( __file__ ).replace( os.getcwd() + os.path.sep, '', 1 ) migrate_repository = repository.Repository( migrate_repository_directory ) def verify_tools( app, url, galaxy_config_file, engine_options={} ): # Check the value in the migrate_tools.version database table column to verify that the number is in # sync with the number of version scripts in ~/lib/galaxy/tools/migrate/versions. dialect = ( url.split( ':', 1 ) )[0] try:
#!/usr/bin/env python # Guruprasad Ananda # Refactored 2011 to use numpy instead of rpy, Kanwei Li """ This tool provides the SQL "group by" functionality. """ import sys, commands, tempfile, random try: import numpy except: from galaxy import eggs eggs.require( "numpy" ) import numpy from itertools import groupby def stop_err(msg): sys.stderr.write(msg) sys.exit() def mode(data): counts = {} for x in data: counts[x] = counts.get(x,0) + 1 maxcount = max(counts.values()) modelist = [] for x in counts: if counts[x] == maxcount: modelist.append( str(x) ) return ','.join(modelist)
""" Module for building and searching the index of tools installed within this Galaxy. """ from galaxy import eggs from galaxy.web.framework.helpers import to_unicode eggs.require( "Whoosh" ) from whoosh.filedb.filestore import RamStorage from whoosh.fields import Schema, STORED, TEXT from whoosh.scoring import BM25F from whoosh.qparser import MultifieldParser schema = Schema( id=STORED, name=TEXT, description=TEXT, section=TEXT, help=TEXT ) import logging log = logging.getLogger( __name__ ) class ToolBoxSearch( object ): """ Support searching tools in a toolbox. This implementation uses the Whoosh search library. """ def __init__( self, toolbox, index_help=True ): """ Create a searcher for `toolbox`. """
import logging import os from datetime import datetime, timedelta from decimal import Decimal from galaxy import eggs eggs.require('SQLAlchemy') from sqlalchemy import and_, desc, false, null, true from sqlalchemy.orm import eagerload from galaxy.web.base.controller import BaseUIController, web from galaxy import model, util log = logging.getLogger( __name__ ) class System( BaseUIController ): @web.expose def index( self, trans, **kwd ): params = util.Params( kwd ) message = '' if params.userless_histories_days: userless_histories_days = params.userless_histories_days else: userless_histories_days = '60' if params.deleted_histories_days: deleted_histories_days = params.deleted_histories_days else: deleted_histories_days = '60' if params.deleted_datasets_days: deleted_datasets_days = params.deleted_datasets_days
import logging from tool_shed.repository_types.metadata import TipOnly import tool_shed.repository_types.util as rt_util from tool_shed.util import basic_util from galaxy import eggs eggs.require( 'mercurial' ) from mercurial import hg from mercurial import ui log = logging.getLogger( __name__ ) class ToolDependencyDefinition( TipOnly ): def __init__( self ): self.type = rt_util.TOOL_DEPENDENCY_DEFINITION self.label = 'Tool dependency definition' self.valid_file_names = [ 'tool_dependencies.xml' ] def is_valid_for_type( self, app, repository, revisions_to_check=None ): """ Inspect the received repository's contents to determine if they abide by the rules defined for the contents of this type. If the received revisions_to_check is a list of changeset revisions, then inspection will be restricted to the revisions in the list. """ repo = hg.repository( ui.ui(), repository.repo_path( app ) ) if revisions_to_check: changeset_revisions = revisions_to_check else: changeset_revisions = repo.changelog for changeset in changeset_revisions:
#!/usr/bin/env python """ Convert from VCF file to interval index file. """ from __future__ import division import optparse from galaxy import eggs eggs.require("bx-python") from bx.interval_index_file import Indexes import galaxy_utils.sequence.vcf def main(): # Read options, args. parser = optparse.OptionParser() (options, args) = parser.parse_args() in_file, out_file = args # Do conversion. index = Indexes() reader = galaxy_utils.sequence.vcf.Reader(open(in_file)) offset = reader.metadata_len for vcf_line in reader: # VCF format provides a chrom and 1-based position for each variant. # IntervalIndex expects 0-based coordinates. index.add(vcf_line.chrom, vcf_line.pos - 1, vcf_line.pos, offset) offset += len(vcf_line.raw_line)
from galaxy.eggs import require from galaxy.web.framework.helpers import to_unicode # Whoosh is compatible with Python 2.5+ Try to import Whoosh and set flag to indicate whether tool search is enabled. try: require( "Whoosh" ) from whoosh.filedb.filestore import RamStorage from whoosh.fields import Schema, STORED, ID, KEYWORD, TEXT from whoosh.index import Index from whoosh.scoring import BM25F from whoosh.qparser import MultifieldParser tool_search_enabled = True schema = Schema( id = STORED, title = TEXT, description = TEXT, help = TEXT ) except ImportError, e: tool_search_enabled = False schema = None class ToolBoxSearch( object ): """ Support searching tools in a toolbox. This implementation uses the "whoosh" search library. """ def __init__( self, toolbox ): """ Create a searcher for `toolbox`. """ self.toolbox = toolbox self.enabled = tool_search_enabled if tool_search_enabled: self.build_index()
This parallelizes the task over available cores using multiprocessing. Code mostly taken form CloudBioLinux. """ import contextlib import functools import glob import multiprocessing import os import subprocess from multiprocessing.pool import IMapIterator try: from galaxy import eggs eggs.require('boto') except ImportError: pass try: import boto from boto.s3.connection import S3Connection except ImportError: boto = None def map_wrap(f): @functools.wraps(f) def wrapper(*args, **kwargs): return apply(f, *args, **kwargs)
# # You can also use this script as a library, for instance see https://gist.github.com/1979583 # TODO: This script overlaps a lot with manage_db.py and create_db.py, # these should maybe be refactored to remove duplication. import sys import os.path db_shell_path = __file__ new_path = [ os.path.join( os.path.dirname( db_shell_path ), os.path.pardir, "lib" ) ] new_path.extend( sys.path[1:] ) # remove scripts/ from the path sys.path = new_path from galaxy.model.orm.scripts import get_config from galaxy import eggs eggs.require( "decorator" ) eggs.require( "Tempita" ) eggs.require( "SQLAlchemy" ) db_url = get_config( sys.argv )['db_url'] # Setup DB scripting environment from sqlalchemy import * from sqlalchemy.orm import * from sqlalchemy.exc import * from galaxy.model.mapping import init sa_session = init( '/tmp/', db_url ).context from galaxy.model import *
import os, logging from galaxy.web.base.controller import * from galaxy.webapps.community.controllers.common import * from galaxy import eggs eggs.require('mercurial') from mercurial.hgweb.hgwebdir_mod import hgwebdir from mercurial.hgweb.request import wsgiapplication log = logging.getLogger(__name__) class HgController(BaseUIController): @web.expose def handle_request(self, trans, **kwd): # The os command that results in this method being called will look something like # hg clone http://[email protected]:9009/repos/test/convert_characters1 cmd = kwd.get('cmd', None) wsgi_app = wsgiapplication(make_web_app) # Hack: Add a parameter to requests for which we do not want all repository metadata reset. reset_metadata = not (kwd.get('no_reset', False)) if cmd == 'listkeys' and reset_metadata: # This possibly results from an "hg push" from the command line. When doing this, the following 7 commands, in order, # will be retrieved from environ: between -> capabilities -> heads -> branchmap -> unbundle -> unbundle -> listkeys path_info = kwd.get('path_info', None) if path_info: owner, name = path_info.split('/') repository = get_repository_by_name_and_owner( trans, name, owner) if repository:
import sys import tempfile from ConfigParser import SafeConfigParser # Assume we are run from the galaxy root directory, add lib to the python path cwd = os.getcwd() new_path = [ os.path.join( cwd, "lib" ), os.path.join( cwd, "test" ) ] new_path.extend( sys.path[1:] ) sys.path = new_path from base.tool_shed_util import parse_tool_panel_config from galaxy import eggs from galaxy.util.properties import load_app_properties eggs.require( "nose" ) eggs.require( "NoseHTML" ) eggs.require( "NoseTestDiff" ) eggs.require( "Paste" ) eggs.require( "PasteDeploy" ) eggs.require( "Cheetah" ) # this should not be required, but it is under certain conditions, thanks to this bug: # http://code.google.com/p/python-nose/issues/detail?id=284 eggs.require( "pysqlite" ) import logging import os.path import time import threading import random
from __future__ import absolute_import import logging try: from galaxy import eggs eggs.require("requests") except ImportError: pass try: import requests except ImportError: requests = None try: import requests_toolbelt requests_multipart_post_available = True except ImportError: requests_multipart_post_available = False requests_toolbelt = None REQUESTS_UNAVAILABLE_MESSAGE = "Pulsar configured to use requests module - but it is unavailable. Please install requests." REQUESTS_TOOLBELT_UNAVAILABLE_MESSAGE = "Pulsar configured to use requests_toolbelt module - but it is unavailable. Please install requests_toolbelt." log = logging.getLogger(__name__) def post_file(url, path): if requests_toolbelt is None: raise ImportError(REQUESTS_TOOLBELT_UNAVAILABLE_MESSAGE)