def configure_logging( config ):
"""
Allow some basic logging configuration to be read from the cherrpy
config.
"""
# PasteScript will have already configured the logger if the appropriate
# sections were found in the config file, so we do nothing if the
# config has a loggers section, otherwise we do some simple setup
# using the 'log_*' values from the config.
if config.global_conf_parser.has_section( "loggers" ):
return
format = config.get( "log_format", "%(name)s %(levelname)s %(asctime)s %(message)s" )
level = logging._levelNames[ config.get( "log_level", "DEBUG" ) ]
destination = config.get( "log_destination", "stdout" )
log.info( "Logging at '%s' level to '%s'" % ( level, destination ) )
# Get root logger
root = logging.getLogger()
# Set level
root.setLevel( level )
# Turn down paste httpserver logging
if level <= logging.DEBUG:
logging.getLogger( "paste.httpserver.ThreadPool" ).setLevel( logging.WARN )
# Remove old handlers
for h in root.handlers[:]:
root.removeHandler(h)
# Create handler
if destination == "stdout":
handler = logging.StreamHandler( sys.stdout )
else:
handler = logging.FileHandler( destination )
# Create formatter
formatter = logging.Formatter( format )
# Hook everything up
handler.setFormatter( formatter )
root.addHandler( handler )
# If sentry is configured, also log to it
if config.sentry_dsn:
eggs.require( "raven" )
from raven.handlers.logging import SentryHandler
sentry_handler = SentryHandler( config.sentry_dsn )
sentry_handler.setLevel( logging.WARN )
root.addHandler( sentry_handler )
def create_database(config_file):
parser = ConfigParser.SafeConfigParser()
parser.read(config_file)
# Determine which database connection to use.
database_connection = parser.get("app:main", "install_database_connection")
if database_connection is None:
database_connection = parser.get("app:main", "database_connection")
if database_connection is None:
database_connection = "sqlite:///%s" % parser.get("app:main", "database_file")
if database_connection is None:
print "Unable to determine correct database connection."
exit(1)
"""Initialize the database file."""
dialect_to_egg = {
"sqlite": "pysqlite>=2",
"postgres": "psycopg2",
"postgresql": "psycopg2",
"mysql": "MySQL_python",
}
dialect = (database_connection.split(":", 1))[0]
try:
egg = dialect_to_egg[dialect]
try:
eggs.require(egg)
print ("%s egg successfully loaded for %s dialect" % (egg, dialect))
except:
# If the module is in the path elsewhere (i.e. non-egg), it'll still load.
print ("%s egg not found, but an attempt will be made to use %s anyway" % (egg, dialect))
except KeyError:
# Let this go, it could possibly work with db's we don't support.
print ("database_connection contains an unknown SQLAlchemy database dialect: %s" % dialect)
# Initialize the database connection.
engine = create_engine(database_connection)
MetaData(bind=engine)
install_session = scoped_session(sessionmaker(bind=engine, autoflush=False, autocommit=True))
model = mapping.init(database_connection)
return install_session, model
def get_track_window(self, dataset, data, start, end):
"""
Assumes we have a numpy file.
"""
# Maybe if we import here people will still be able to use Galaxy when numpy kills it
eggs.require("numpy>=1.2.1")
import numpy
range = end - start
# Determine appropriate resolution to plot ~1000 points
resolution = ( 10 ** math.ceil( math.log10( range / 1000 ) ) )
# Restrict to valid range
resolution = min( resolution, 10000 )
resolution = max( resolution, 1 )
# Memory map the array (don't load all the data)
data = numpy.load( data )
# Grab just what we need
t_start = math.floor( start / resolution )
t_end = math.ceil( end / resolution )
x = numpy.arange( t_start, t_end ) * resolution
y = data[ t_start : t_end ]
return zip(x.tolist(), y.tolist())
import json
import logging
import os
import string
import subprocess
import time
from galaxy import eggs
from galaxy import model
from galaxy.jobs import JobDestination
from galaxy.jobs.handler import DEFAULT_JOB_PUT_FAILURE_MESSAGE
from galaxy.jobs.runners import AsynchronousJobState, AsynchronousJobRunner
from galaxy.util import asbool
eggs.require( "drmaa" )
log = logging.getLogger( __name__ )
__all__ = [ 'DRMAAJobRunner' ]
drmaa = None
DRMAA_jobTemplate_attributes = [ 'args', 'remoteCommand', 'outputPath', 'errorPath', 'nativeSpecification',
'workingDirectory', 'jobName', 'email', 'project' ]
class DRMAAJobRunner( AsynchronousJobRunner ):
"""
Job runner backed by a finite pool of worker threads. FIFO scheduling
"""
Populates blank uuid fields in datasets with randomly generated values
Going forward, these ids will be generated for all new datasets. This
script fixes datasets that were generated before the change.
"""
import sys, os, ConfigParser
import galaxy.app
from galaxy.util.bunch import Bunch
import galaxy.datatypes.tabular
from galaxy.model.orm.scripts import get_config
from galaxy import eggs
from galaxy.model import mapping
import uuid
eggs.require( "SQLAlchemy" )
from sqlalchemy import *
assert sys.version_info[:2] >= ( 2, 4 )
def main():
ini_file = sys.argv.pop(1)
config = get_config(ini_file)
model = mapping.init( ini_file, config['db_url'], create_tables = False )
for row in model.context.query( model.Dataset ):
if row.uuid is None:
row.uuid = uuid.uuid4()
print "Setting dataset:", row.id, " UUID to ", row.uuid
import logging
from galaxy import eggs
eggs.require('SQLAlchemy')
from sqlalchemy import and_, or_
from tool_shed.util import hg_util
from tool_shed.util import shed_util_common as suc
log = logging.getLogger( __name__ )
class ToolVersionManager( object ):
def __init__( self, app ):
self.app = app
def get_tool_version( self, tool_id ):
context = self.app.install_model.context
return context.query( self.app.install_model.ToolVersion ) \
.filter( self.app.install_model.ToolVersion.table.c.tool_id == tool_id ) \
.first()
def get_tool_version_association( self, parent_tool_version, tool_version ):
"""
Return a ToolVersionAssociation if one exists that associates the two
received tool_versions. This function is called only from Galaxy.
"""
context = self.app.install_model.context
return context.query( self.app.install_model.ToolVersionAssociation ) \
.filter( and_( self.app.install_model.ToolVersionAssociation.table.c.parent_id == parent_tool_version.id,
will execute this script with the appropriate parameters.
"""
import os
import sys
# Assume we are run from the galaxy root directory, add lib to the python path
cwd = os.getcwd()
sys.path.append( cwd )
new_path = [ os.path.join( cwd, "scripts" ),
os.path.join( cwd, "lib" ),
os.path.join( cwd, 'test' ),
os.path.join( cwd, 'scripts', 'api' ) ]
new_path.extend( sys.path )
sys.path = new_path
from galaxy import eggs
eggs.require( "nose" )
eggs.require( "Paste" )
eggs.require( 'mercurial' )
# This should not be required, but it is under certain conditions thanks to this bug:
# http://code.google.com/p/python-nose/issues/detail?id=284
eggs.require( "pysqlite" )
import httplib
import install_and_test_tool_shed_repositories.base.test_db_util as test_db_util
import install_and_test_tool_shed_repositories.functional.test_install_repositories as test_install_repositories
import logging
import nose
import random
import re
import shutil
import socket
import logging import os import Queue import shutil import subprocess import tempfile import threading import time from contextlib import contextmanager # TODO: eliminate the use of fabric here. from galaxy import eggs eggs.require( 'paramiko' ) eggs.require( 'ssh' ) eggs.require( 'Fabric' ) from fabric.operations import _AttributeString from fabric import state from galaxy.util import DATABASE_MAX_STRING_SIZE from galaxy.util import DATABASE_MAX_STRING_SIZE_PRETTY from galaxy.util import shrink_string_by_size from galaxy.util import unicodify from tool_shed.galaxy_install.tool_dependencies.recipe import asynchronous_reader from tool_shed.util import basic_util log = logging.getLogger( __name__ )
"""Binary classes"""
import binascii
import data
import gzip
import logging
import os
import shutil
import struct
import subprocess
import tempfile
import warnings
import zipfile
from galaxy import eggs
eggs.require("bx-python")
from bx.seq.twobit import TWOBIT_MAGIC_NUMBER, TWOBIT_MAGIC_NUMBER_SWAP, TWOBIT_MAGIC_SIZE
from galaxy.util import sqlite
from galaxy.datatypes.metadata import MetadataElement, MetadataParameter, ListParameter, DictParameter
from galaxy.datatypes import metadata
import dataproviders
with warnings.catch_warnings():
warnings.simplefilter("ignore")
eggs.require("pysam")
from pysam import csamtools
log = logging.getLogger(__name__)
Dataproviders that use either:
- the file contents and/or metadata from a Galaxy DatasetInstance as
their source.
- or provide data in some way relevant to bioinformatic data
(e.g. parsing genomic regions from their source)
"""
import base
import line
import column
import external
from galaxy.util import sqlite
import re
from galaxy import eggs
eggs.require('bx-python')
from bx import seq as bx_seq
from bx import wiggle as bx_wig
from bx import bbi as bx_bbi
_TODO = """
use bx as much as possible
gff3 hierarchies
change SamtoolsDataProvider to use pysam
"""
import logging
log = logging.getLogger(__name__)
def create_or_verify_database(url, engine_options={}, app=None):
"""
"""
dialect = (url.split(':', 1))[0]
try:
egg = dialect_to_egg[dialect]
try:
eggs.require(egg)
log.debug("%s egg successfully loaded for %s dialect" %
(egg, dialect))
except:
# If the module is in the path elsewhere (i.e. non-egg), it'll still load.
log.warning(
"%s egg not found, but an attempt will be made to use %s anyway"
% (egg, dialect))
except KeyError:
# Let this go, it could possibly work with db's we don't support
log.error(
"database_connection contains an unknown SQLAlchemy database dialect: %s"
% dialect)
# Create engine and metadata
engine = create_engine(url, **engine_options)
def migrate():
try:
# Declare the database to be under a repository's version control
db_schema = schema.ControlledSchema.create(engine,
migrate_repository)
except:
# The database is already under version control
db_schema = schema.ControlledSchema(engine, migrate_repository)
# Apply all scripts to get to current version
migrate_to_current_version(engine, db_schema)
meta = MetaData(bind=engine)
if app and getattr(app.config, 'database_auto_migrate', False):
migrate()
return
# Try to load tool_shed_repository table
try:
Table("tool_shed_repository", meta, autoload=True)
except NoSuchTableError:
# No table means a completely uninitialized database. If we
# have an app, we'll set its new_installation setting to True
# so the tool migration process will be skipped.
migrate()
return
try:
Table("migrate_version", meta, autoload=True)
except NoSuchTableError:
# The database exists but is not yet under migrate version control, so init with version 1
log.info("Adding version control to existing database")
try:
Table("metadata_file", meta, autoload=True)
schema.ControlledSchema.create(engine,
migrate_repository,
version=2)
except NoSuchTableError:
schema.ControlledSchema.create(engine,
migrate_repository,
version=1)
# Verify that the code and the DB are in sync
db_schema = schema.ControlledSchema(engine, migrate_repository)
if migrate_repository.versions.latest != db_schema.version:
exception_msg = "Your database has version '%d' but this code expects version '%d'. " % (
db_schema.version, migrate_repository.versions.latest)
exception_msg += "Back up your database and then migrate the schema by running the following from your Galaxy installation directory:"
exception_msg += "\n\nsh manage_db.sh upgrade install\n"
else:
log.info("At database version %d" % db_schema.version)
def verify_tools(app, url, galaxy_config_file, engine_options={}):
# Check the value in the migrate_tools.version database table column to verify that the number is in
# sync with the number of version scripts in ~/lib/galaxy/tools/migrate/versions.
dialect = (url.split(':', 1))[0]
try:
egg = dialect_to_egg[dialect]
try:
eggs.require(egg)
log.debug("%s egg successfully loaded for %s dialect" %
(egg, dialect))
except:
# If the module is in the path elsewhere (i.e. non-egg), it'll still load.
log.warning(
"%s egg not found, but an attempt will be made to use %s anyway"
% (egg, dialect))
except KeyError:
# Let this go, it could possibly work with db's we don't support
log.error(
"database_connection contains an unknown SQLAlchemy database dialect: %s"
% dialect)
# Create engine and metadata
engine = create_engine(url, **engine_options)
meta = MetaData(bind=engine)
# The migrate_tools table was created in database version script 0092_add_migrate_tools_table.py.
version_table = Table("migrate_tools", meta, autoload=True)
# Verify that the code and the database are in sync.
db_schema = schema.ControlledSchema(engine, migrate_repository)
latest_tool_migration_script_number = migrate_repository.versions.latest
if latest_tool_migration_script_number != db_schema.version:
# The default behavior is that the tool shed is down.
tool_shed_accessible = False
if app.new_installation:
# New installations will not be missing tools, so we don't need to worry about them.
missing_tool_configs_dict = odict()
else:
tool_panel_configs = common_util.get_non_shed_tool_panel_configs(
app)
if tool_panel_configs:
# The missing_tool_configs_dict contents are something like:
# {'emboss_antigenic.xml': [('emboss', '5.0.0', 'package', '\nreadme blah blah blah\n')]}
tool_shed_accessible, missing_tool_configs_dict = common_util.check_for_missing_tools(
app, tool_panel_configs,
latest_tool_migration_script_number)
else:
# It doesn't matter if the tool shed is accessible since there are no migrated tools defined in the local Galaxy instance, but
# we have to set the value of tool_shed_accessible to True so that the value of migrate_tools.version can be correctly set in
# the database.
tool_shed_accessible = True
missing_tool_configs_dict = odict()
have_tool_dependencies = False
for k, v in missing_tool_configs_dict.items():
if v:
have_tool_dependencies = True
break
config_arg = ''
if os.path.abspath(os.path.join(
os.getcwd(), 'universe_wsgi.ini')) != galaxy_config_file:
config_arg = ' -c %s' % galaxy_config_file.replace(
os.path.abspath(os.getcwd()), '.')
if not app.config.running_functional_tests:
if tool_shed_accessible:
# Automatically update the value of the migrate_tools.version database table column.
cmd = 'sh manage_tools.sh%s upgrade' % config_arg
proc = subprocess.Popen(args=cmd,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
return_code = proc.wait()
output = proc.stdout.read(32768)
if return_code != 0:
raise Exception(
"Error attempting to update the value of migrate_tools.version: %s"
% output)
elif missing_tool_configs_dict:
if len(tool_panel_configs) == 1:
plural = ''
tool_panel_config_file_names = tool_panel_configs[0]
else:
plural = 's'
tool_panel_config_file_names = ', '.join(
tool_panel_configs)
msg = "\n>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>"
msg += "\n\nThe list of files at the end of this message refers to tools that are configured to load into the tool panel for\n"
msg += "this Galaxy instance, but have been removed from the Galaxy distribution. These tools and their dependencies can be\n"
msg += "automatically installed from the Galaxy tool shed at http://toolshed.g2.bx.psu.edu.\n\n"
msg += "To skip this process, attempt to start your Galaxy server again (e.g., sh run.sh or whatever you use). If you do this,\n"
msg += "be aware that these tools will no longer be available in your Galaxy tool panel, and entries for each of them should\n"
msg += "be removed from your file%s named %s.\n\n" % (
plural, tool_panel_config_file_names)
msg += "CRITICAL NOTE IF YOU PLAN TO INSTALL\n"
msg += "The location in which the tool repositories will be installed is the value of the 'tool_path' attribute in the <tool>\n"
msg += 'tag of the file named ./migrated_tool_conf.xml (i.e., <toolbox tool_path="../shed_tools">). The default location\n'
msg += "setting is '../shed_tools', which may be problematic for some cluster environments, so make sure to change it before\n"
msg += "you execute the installation process if appropriate. The configured location must be outside of the Galaxy installation\n"
msg += "directory or it must be in a sub-directory protected by a properly configured .hgignore file if the directory is within\n"
msg += "the Galaxy installation directory hierarchy. This is because tool shed repositories will be installed using mercurial's\n"
msg += "clone feature, which creates .hg directories and associated mercurial repository files. Not having .hgignore properly\n"
msg += "configured could result in undesired behavior when modifying or updating your local Galaxy instance or the tool shed\n"
msg += "repositories if they are in directories that pose conflicts. See mercurial's .hgignore documentation at the following\n"
msg += "URL for details.\n\nhttp://mercurial.selenic.com/wiki/.hgignore\n\n"
if have_tool_dependencies:
msg += "The following tool dependencies can also optionally be installed (see the option flag in the command below). If you\n"
msg += "choose to install them (recommended), they will be installed within the location specified by the 'tool_dependency_dir'\n"
msg += "setting in your main Galaxy configuration file (e.g., uninverse_wsgi.ini).\n"
processed_tool_dependencies = []
for missing_tool_config, tool_dependencies in missing_tool_configs_dict.items(
):
for tool_dependencies_tup in missing_tool_configs_dict[
missing_tool_config]['tool_dependencies']:
if tool_dependencies_tup not in processed_tool_dependencies:
msg += "------------------------------------\n"
msg += "Tool Dependency\n"
msg += "------------------------------------\n"
msg += "Name: %s, Version: %s, Type: %s\n" % (
tool_dependencies_tup[0],
tool_dependencies_tup[1],
tool_dependencies_tup[2])
if len(tool_dependencies_tup) >= 4:
msg += "Requirements and installation information:\n"
msg += "%s\n" % tool_dependencies_tup[3]
else:
msg += "\n"
msg += "------------------------------------\n"
processed_tool_dependencies.append(
tool_dependencies_tup)
msg += "\n"
msg += "%s" % output.replace('done', '')
msg += "vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv\n"
msg += "sh ./scripts/migrate_tools/%04d_tools.sh\n" % latest_tool_migration_script_number
msg += "^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\n"
if have_tool_dependencies:
msg += "The tool dependencies listed above will be installed along with the repositories if you add the 'install_dependencies'\n"
msg += "option to the above command like this:\n\n"
msg += "vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv\n"
msg += "sh ./scripts/migrate_tools/%04d_tools.sh install_dependencies\n" % latest_tool_migration_script_number
msg += "^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\n"
msg += "Tool dependencies can be installed after the repositories have been installed as well.\n\n"
msg += "After the installation process finishes, you can start your Galaxy server. As part of this installation process,\n"
msg += "entries for each of the following tool config files will be added to the file named ./migrated_tool_conf.xml, so these\n"
msg += "tools will continue to be loaded into your tool panel. Because of this, existing entries for these tools have been\n"
msg += "removed from your file%s named %s.\n\n" % (
plural, tool_panel_config_file_names)
for missing_tool_config, tool_dependencies in missing_tool_configs_dict.items(
):
msg += "%s\n" % missing_tool_config
msg += "<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<\n"
raise Exception(msg)
else:
log.debug(
"The main Galaxy tool shed is not currently available, so skipped tool migration %s until next server startup"
% db_schema.version)
else:
log.info("At migrate_tools version %d" % db_schema.version)
import os
import ConfigParser
from optparse import OptionParser
new_path = [ os.path.join( os.getcwd(), "lib" ) ]
new_path.extend( sys.path[1:] ) # remove scripts/ from the path
sys.path = new_path
from galaxy.util import pretty_print_time_interval
import galaxy.webapps.tool_shed.model.mapping
from galaxy.webapps.tool_shed import config
from galaxy.webapps.tool_shed import model
from galaxy.tools.loader_directory import load_tool_elements_from_path
from galaxy import eggs
eggs.require( "SQLAlchemy" )
eggs.require( "Whoosh" )
from whoosh.filedb.filestore import FileStorage
from whoosh.fields import Schema, STORED, TEXT
repo_schema = Schema(
id=STORED,
name=TEXT( stored=True ),
description=TEXT( stored=True ),
long_description=TEXT( stored=True ),
homepage_url=TEXT( stored=True ),
remote_repository_url=TEXT( stored=True ),
repo_owner_username=TEXT( stored=True ),
times_downloaded=STORED,
approved=STORED,
last_updated=STORED,
def create_or_verify_database(url, engine_options={}):
"""
Check that the database is use-able, possibly creating it if empty (this is
the only time we automatically create tables, otherwise we force the
user to do it using the management script so they can create backups).
1) Empty database --> initialize with latest version and return
2) Database older than migration support --> fail and require manual update
3) Database at state where migrate support introduced --> add version control information but make no changes (might still require manual update)
4) Database versioned but out of date --> fail with informative message, user must run "sh manage_db.sh upgrade"
"""
dialect = (url.split(':', 1))[0]
try:
egg = dialect_to_egg[dialect]
try:
eggs.require(egg)
log.debug("%s egg successfully loaded for %s dialect" %
(egg, dialect))
except:
# If the module is in the path elsewhere (i.e. non-egg), it'll still load.
log.warning(
"%s egg not found, but an attempt will be made to use %s anyway"
% (egg, dialect))
except KeyError:
# Let this go, it could possibly work with db's we don't support
log.error(
"database_connection contains an unknown SQLAlchemy database dialect: %s"
% dialect)
# Create engine and metadata
engine = create_engine(url, **engine_options)
meta = MetaData(bind=engine)
# Try to load dataset table
try:
galaxy_user_table = Table("galaxy_user", meta, autoload=True)
except NoSuchTableError:
# No 'galaxy_user' table means a completely uninitialized database, which
# is fine, init the database in a versioned state
log.info("No database, initializing")
# Database might or might not be versioned
try:
# Declare the database to be under a repository's version control
db_schema = schema.ControlledSchema.create(engine,
migrate_repository)
except:
# The database is already under version control
db_schema = schema.ControlledSchema(engine, migrate_repository)
# Apply all scripts to get to current version
migrate_to_current_version(engine, db_schema)
return
try:
version_table = Table("migrate_version", meta, autoload=True)
except NoSuchTableError:
# The database exists but is not yet under migrate version control, so init with version 1
log.info("Adding version control to existing database")
try:
metadata_file_table = Table("metadata_file", meta, autoload=True)
schema.ControlledSchema.create(engine,
migrate_repository,
version=2)
except NoSuchTableError:
schema.ControlledSchema.create(engine,
migrate_repository,
version=1)
# Verify that the code and the DB are in sync
db_schema = schema.ControlledSchema(engine, migrate_repository)
if migrate_repository.versions.latest != db_schema.version:
exception_msg = "Your database has version '%d' but this code expects version '%d'. " % (
db_schema.version, migrate_repository.versions.latest)
exception_msg += "Back up your database and then migrate the schema by running the following from your Galaxy installation directory:"
exception_msg += "\n\nsh manage_db.sh upgrade tool_shed\n"
raise Exception(exception_msg)
else:
log.info("At database version %d" % db_schema.version)
from __future__ import absolute_import
import logging
try:
from urllib2 import urlopen
except ImportError:
from urllib.request import urlopen
try:
from urllib2 import Request
except ImportError:
from urllib.request import Request
try:
from galaxy import eggs
eggs.require("poster")
except ImportError:
pass
try:
import poster
except ImportError:
poster = None
POSTER_UNAVAILABLE_MESSAGE = "Pulsar configured to use poster module - but it is unavailable. Please install poster."
log = logging.getLogger(__name__)
if poster is not None:
poster.streaminghttp.register_openers()
import tempfile import threading import urlparse from galaxy.util import json from datetime import datetime from email.MIMEText import MIMEText from os.path import relpath from hashlib import md5 from itertools import izip from galaxy import eggs eggs.require( 'docutils' ) import docutils.core import docutils.writers.html4css1 from xml.etree import ElementTree, ElementInclude eggs.require( "wchartype" ) import wchartype from .inflection import Inflector, English inflector = Inflector(English) log = logging.getLogger(__name__) _lock = threading.RLock() CHUNK_SIZE = 65536 # 64k
import logging
import operator
import os
from galaxy import util
from galaxy.util.bunch import Bunch
from galaxy.util.hash_util import new_secure_hash
from galaxy.model.item_attrs import Dictifiable
import tool_shed.repository_types.util as rt_util
from galaxy import eggs
eggs.require("mercurial")
from mercurial import hg
from mercurial import ui
log = logging.getLogger(__name__)
class APIKeys(object):
pass
class User(object, Dictifiable):
dict_collection_visible_keys = ("id", "email")
dict_element_visible_keys = ("id", "email", "username")
def __init__(self, email=None, password=None):
self.email = email
self.password = password
self.external = False
self.deleted = False
import ConfigParser as configparser
import galaxy.app
from galaxy.config import process_is_uwsgi
import galaxy.model
import galaxy.model.mapping
import galaxy.datatypes.registry
import galaxy.web.framework
import galaxy.web.framework.webapp
from galaxy import util
from galaxy.util import asbool
from galaxy.util.properties import load_app_properties
from galaxy import eggs
eggs.require('Paste')
from paste import httpexceptions
import logging
log = logging.getLogger( __name__ )
try:
from uwsgidecorators import postfork
except:
# TODO: Make this function more like flask's @before_first_request w/
# registered methods etc.
def pf_dec(func):
return func
postfork = pf_dec
def create_or_verify_database( url, galaxy_config_file, engine_options={}, app=None ):
"""
Check that the database is use-able, possibly creating it if empty (this is
the only time we automatically create tables, otherwise we force the
user to do it using the management script so they can create backups).
1) Empty database --> initialize with latest version and return
2) Database older than migration support --> fail and require manual update
3) Database at state where migrate support introduced --> add version control information but make no changes (might still require manual update)
4) Database versioned but out of date --> fail with informative message, user must run "sh manage_db.sh upgrade"
"""
dialect = ( url.split( ':', 1 ) )[0]
try:
egg = dialect_to_egg[dialect]
try:
eggs.require( egg )
log.debug( "%s egg successfully loaded for %s dialect" % ( egg, dialect ) )
except:
# If the module is in the path elsewhere (i.e. non-egg), it'll still load.
log.warning( "%s egg not found, but an attempt will be made to use %s anyway" % ( egg, dialect ) )
except KeyError:
# Let this go, it could possibly work with db's we don't support
log.error( "database_connection contains an unknown SQLAlchemy database dialect: %s" % dialect )
# Create engine and metadata
engine = create_engine( url, **engine_options )
meta = MetaData( bind=engine )
# Try to load dataset table
try:
dataset_table = Table( "dataset", meta, autoload=True )
except NoSuchTableError:
# No 'dataset' table means a completely uninitialized database. If we have an app, we'll
# set it's new_installation setting to True so the tool migration process will be skipped.
if app:
app.new_installation = True
log.info( "No database, initializing" )
# Database might or might not be versioned
try:
# Declare the database to be under a repository's version control
db_schema = schema.ControlledSchema.create( engine, migrate_repository )
except:
# The database is already under version control
db_schema = schema.ControlledSchema( engine, migrate_repository )
# Apply all scripts to get to current version
migrate_to_current_version( engine, db_schema )
return
try:
hda_table = Table( "history_dataset_association", meta, autoload=True )
except NoSuchTableError:
raise Exception( "Your database is older than hg revision 1464:c7acaa1bb88f and will need to be updated manually" )
# There is a 'history_dataset_association' table, so we (hopefully) have
# version 1 of the database, but without the migrate_version table. This
# happens if the user has a build from right before migration was added.
# Verify that this is true, if it is any older they'll have to update
# manually
if 'copied_from_history_dataset_association_id' not in hda_table.c:
# The 'copied_from_history_dataset_association_id' column was added in
# rev 1464:c7acaa1bb88f. This is the oldest revision we currently do
# automated versioning for, so stop here
raise Exception( "Your database is older than hg revision 1464:c7acaa1bb88f and will need to be updated manually" )
# At revision 1464:c7acaa1bb88f or greater (database version 1), make sure
# that the db has version information. This is the trickiest case -- we
# have a database but no version control, and are assuming it is a certain
# version. If the user has postion version 1 changes this could cause
# problems
try:
version_table = Table( "migrate_version", meta, autoload=True )
except NoSuchTableError:
# The database exists but is not yet under migrate version control, so init with version 1
log.info( "Adding version control to existing database" )
try:
metadata_file_table = Table( "metadata_file", meta, autoload=True )
schema.ControlledSchema.create( engine, migrate_repository, version=2 )
except NoSuchTableError:
schema.ControlledSchema.create( engine, migrate_repository, version=1 )
# Verify that the code and the DB are in sync
db_schema = schema.ControlledSchema( engine, migrate_repository )
if migrate_repository.versions.latest != db_schema.version:
config_arg = ''
if os.path.abspath( os.path.join( os.getcwd(), 'universe_wsgi.ini' ) ) != galaxy_config_file:
config_arg = ' -c %s' % galaxy_config_file.replace( os.path.abspath( os.getcwd() ), '.' )
raise Exception( "Your database has version '%d' but this code expects version '%d'. Please backup your database and then migrate the schema by running 'sh manage_db.sh%s upgrade'."
% ( db_schema.version, migrate_repository.versions.latest, config_arg ) )
else:
log.info( "At database version %d" % db_schema.version )
def wrap_in_middleware( app, global_conf, **local_conf ):
"""
Based on the configuration wrap `app` in a set of common and useful
middleware.
"""
# Merge the global and local configurations
conf = global_conf.copy()
conf.update(local_conf)
debug = asbool( conf.get( 'debug', False ) )
# First put into place httpexceptions, which must be most closely
# wrapped around the application (it can interact poorly with
# other middleware):
app = httpexceptions.make_middleware( app, conf )
log.debug( "Enabling 'httpexceptions' middleware" )
# If we're using remote_user authentication, add middleware that
# protects Galaxy from improperly configured authentication in the
# upstream server
if asbool(conf.get( 'use_remote_user', False )):
from galaxy.web.framework.middleware.remoteuser import RemoteUser
app = RemoteUser( app, maildomain=conf.get( 'remote_user_maildomain', None ),
display_servers=util.listify( conf.get( 'display_servers', '' ) ),
admin_users=conf.get( 'admin_users', '' ).split( ',' ),
remote_user_header=conf.get( 'remote_user_header', 'HTTP_REMOTE_USER' ),
remote_user_secret_header=conf.get('remote_user_secret', None) )
# The recursive middleware allows for including requests in other
# requests or forwarding of requests, all on the server side.
if asbool(conf.get('use_recursive', True)):
from paste import recursive
app = recursive.RecursiveMiddleware( app, conf )
log.debug( "Enabling 'recursive' middleware" )
# If sentry logging is enabled, log here before propogating up to
# the error middleware
sentry_dsn = conf.get( 'sentry_dsn', None )
if sentry_dsn:
from galaxy.web.framework.middleware.sentry import Sentry
app = Sentry( app, sentry_dsn )
# Various debug middleware that can only be turned on if the debug
# flag is set, either because they are insecure or greatly hurt
# performance
if debug:
# Middleware to check for WSGI compliance
if asbool( conf.get( 'use_lint', False ) ):
from paste import lint
app = lint.make_middleware( app, conf )
log.debug( "Enabling 'lint' middleware" )
# Middleware to run the python profiler on each request
if asbool( conf.get( 'use_profile', False ) ):
from paste.debug import profile
app = profile.ProfileMiddleware( app, conf )
log.debug( "Enabling 'profile' middleware" )
if debug and asbool( conf.get( 'use_interactive', False ) ) and not process_is_uwsgi:
# Interactive exception debugging, scary dangerous if publicly
# accessible, if not enabled we'll use the regular error printing
# middleware.
eggs.require( "WebError" )
from weberror import evalexception
app = evalexception.EvalException( app, conf,
templating_formatters=build_template_error_formatters() )
log.debug( "Enabling 'eval exceptions' middleware" )
else:
if debug and asbool( conf.get( 'use_interactive', False ) ) and process_is_uwsgi:
log.error("Interactive debugging middleware is enabled in your configuration "
"but this is a uwsgi process. Refusing to wrap in interactive error middleware.")
# Not in interactive debug mode, just use the regular error middleware
import galaxy.web.framework.middleware.error
app = galaxy.web.framework.middleware.error.ErrorMiddleware( app, conf )
log.debug( "Enabling 'error' middleware" )
# Transaction logging (apache access.log style)
if asbool( conf.get( 'use_translogger', True ) ):
from galaxy.web.framework.middleware.translogger import TransLogger
app = TransLogger( app )
log.debug( "Enabling 'trans logger' middleware" )
# X-Forwarded-Host handling
from galaxy.web.framework.middleware.xforwardedhost import XForwardedHostMiddleware
app = XForwardedHostMiddleware( app )
log.debug( "Enabling 'x-forwarded-host' middleware" )
# Request ID middleware
from galaxy.web.framework.middleware.request_id import RequestIDMiddleware
app = RequestIDMiddleware( app )
log.debug( "Enabling 'Request ID' middleware" )
return app
"""Binary classes"""
import binascii
import gzip
import logging
import os
import shutil
import struct
import subprocess
import tempfile
import warnings
import zipfile
from galaxy import eggs
eggs.require( "bx-python" )
from bx.seq.twobit import TWOBIT_MAGIC_NUMBER, TWOBIT_MAGIC_NUMBER_SWAP, TWOBIT_MAGIC_SIZE
from galaxy.datatypes.metadata import MetadataElement, MetadataParameter, ListParameter, DictParameter
from galaxy.datatypes import metadata
from galaxy.util import nice_size, sqlite
from . import data, dataproviders
with warnings.catch_warnings():
warnings.simplefilter( "ignore" )
eggs.require( "pysam" )
from pysam import csamtools
log = logging.getLogger(__name__)
# Currently these supported binary data types must be manually set on upload
import sys import os import pprint import unittest __GALAXY_ROOT__ = os.getcwd() + '/../../../' sys.path.append( __GALAXY_ROOT__ + 'lib' ) from galaxy import eggs eggs.require( 'SQLAlchemy >= 0.4' ) import sqlalchemy from galaxy import model from galaxy import exceptions from galaxy.util.bunch import Bunch import mock from test_ModelManager import BaseTestCase from galaxy.managers.histories import HistoryManager from galaxy.managers.datasets import DatasetManager from galaxy.managers.hdas import HDAManager # ============================================================================= default_password = '******' user2_data = dict( email='[email protected]', username='******', password=default_password ) user3_data = dict( email='[email protected]', username='******', password=default_password ) # ============================================================================= class HDAManagerTestCase( BaseTestCase ):
"""
Galaxy Metadata
"""
from galaxy import eggs
eggs.require("simplejson")
import copy
import cPickle
import logging
import os
import shutil
import simplejson
import sys
import tempfile
import weakref
from os.path import abspath
import galaxy.model
from galaxy.util import listify, stringify_dictionary_keys, string_as_bool
from galaxy.util.odict import odict
from galaxy.web import form_builder
from sqlalchemy.orm import object_session
log = logging.getLogger(__name__)
STATEMENTS = "__galaxy_statements__" #this is the name of the property in a Datatype class where new metadata spec element Statements are stored
class Statement( object ):
"""
"""
Galaxy control queue and worker. This is used to handle 'app' control like
reloading the toolbox, etc., across multiple processes.
"""
import logging
import threading
import sys
import galaxy.queues
from galaxy import eggs, util
eggs.require('anyjson')
if sys.version_info < (2, 7, 0):
# Kombu requires importlib and ordereddict to function under Python 2.6.
eggs.require('importlib')
eggs.require('ordereddict')
eggs.require('kombu')
from kombu import Connection
from kombu.mixins import ConsumerMixin
from kombu.pools import producers
log = logging.getLogger(__name__)
class GalaxyQueueWorker(ConsumerMixin, threading.Thread):
"""
This is a flexible worker for galaxy's queues. Each process, web or
handler, will have one of these used for dispatching so called 'control'
tasks.
"""
def verify_tools( app, url, galaxy_config_file, engine_options={} ):
# Check the value in the migrate_tools.version database table column to verify that the number is in
# sync with the number of version scripts in ~/lib/galaxy/tools/migrate/versions.
dialect = ( url.split( ':', 1 ) )[0]
try:
egg = dialect_to_egg[ dialect ]
try:
eggs.require( egg )
log.debug( "%s egg successfully loaded for %s dialect" % ( egg, dialect ) )
except:
# If the module is in the path elsewhere (i.e. non-egg), it'll still load.
log.warning( "%s egg not found, but an attempt will be made to use %s anyway" % ( egg, dialect ) )
except KeyError:
# Let this go, it could possibly work with db's we don't support
log.error( "database_connection contains an unknown SQLAlchemy database dialect: %s" % dialect )
# Create engine and metadata
engine = create_engine( url, **engine_options )
meta = MetaData( bind=engine )
# The migrate_tools table was created in database version script 0092_add_migrate_tools_table.py.
version_table = Table( "migrate_tools", meta, autoload=True )
# Verify that the code and the database are in sync.
db_schema = schema.ControlledSchema( engine, migrate_repository )
latest_tool_migration_script_number = migrate_repository.versions.latest
if latest_tool_migration_script_number != db_schema.version:
# The default behavior is that the tool shed is down.
tool_shed_accessible = False
if app.new_installation:
# New installations will not be missing tools, so we don't need to worry about them.
missing_tool_configs_dict = odict()
else:
tool_panel_configs = common_util.get_non_shed_tool_panel_configs( app )
if tool_panel_configs:
# The missing_tool_configs_dict contents are something like:
# {'emboss_antigenic.xml': [('emboss', '5.0.0', 'package', '\nreadme blah blah blah\n')]}
tool_shed_accessible, missing_tool_configs_dict = common_util.check_for_missing_tools( app,
tool_panel_configs,
latest_tool_migration_script_number )
else:
# It doesn't matter if the tool shed is accessible since there are no migrated tools defined in the local Galaxy instance, but
# we have to set the value of tool_shed_accessible to True so that the value of migrate_tools.version can be correctly set in
# the database.
tool_shed_accessible = True
missing_tool_configs_dict = odict()
have_tool_dependencies = False
for k, v in missing_tool_configs_dict.items():
if v:
have_tool_dependencies = True
break
config_arg = ''
if os.path.abspath( os.path.join( os.getcwd(), 'galaxy.ini' ) ) != galaxy_config_file:
config_arg = ' -c %s' % galaxy_config_file.replace( os.path.abspath( os.getcwd() ), '.' )
if not app.config.running_functional_tests:
if tool_shed_accessible:
# Automatically update the value of the migrate_tools.version database table column.
cmd = 'sh manage_tools.sh%s upgrade' % config_arg
proc = subprocess.Popen( args=cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT )
return_code = proc.wait()
output = proc.stdout.read( 32768 )
if return_code != 0:
raise Exception( "Error attempting to update the value of migrate_tools.version: %s" % output )
elif missing_tool_configs_dict:
if len( tool_panel_configs ) == 1:
plural = ''
tool_panel_config_file_names = tool_panel_configs[ 0 ]
else:
plural = 's'
tool_panel_config_file_names = ', '.join( tool_panel_configs )
msg = "\n>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>"
msg += "\n\nThe list of files at the end of this message refers to tools that are configured to load into the tool panel for\n"
msg += "this Galaxy instance, but have been removed from the Galaxy distribution. These tools and their dependencies can be\n"
msg += "automatically installed from the Galaxy tool shed at http://toolshed.g2.bx.psu.edu.\n\n"
msg += "To skip this process, attempt to start your Galaxy server again (e.g., sh run.sh or whatever you use). If you do this,\n"
msg += "be aware that these tools will no longer be available in your Galaxy tool panel, and entries for each of them should\n"
msg += "be removed from your file%s named %s.\n\n" % ( plural, tool_panel_config_file_names )
msg += "CRITICAL NOTE IF YOU PLAN TO INSTALL\n"
msg += "The location in which the tool repositories will be installed is the value of the 'tool_path' attribute in the <tool>\n"
msg += 'tag of the file named ./migrated_tool_conf.xml (i.e., <toolbox tool_path="../shed_tools">). The default location\n'
msg += "setting is '../shed_tools', which may be problematic for some cluster environments, so make sure to change it before\n"
msg += "you execute the installation process if appropriate. The configured location must be outside of the Galaxy installation\n"
msg += "directory or it must be in a sub-directory protected by a properly configured .hgignore file if the directory is within\n"
msg += "the Galaxy installation directory hierarchy. This is because tool shed repositories will be installed using mercurial's\n"
msg += "clone feature, which creates .hg directories and associated mercurial repository files. Not having .hgignore properly\n"
msg += "configured could result in undesired behavior when modifying or updating your local Galaxy instance or the tool shed\n"
msg += "repositories if they are in directories that pose conflicts. See mercurial's .hgignore documentation at the following\n"
msg += "URL for details.\n\nhttp://mercurial.selenic.com/wiki/.hgignore\n\n"
if have_tool_dependencies:
msg += "The following tool dependencies can also optionally be installed (see the option flag in the command below). If you\n"
msg += "choose to install them (recommended), they will be installed within the location specified by the 'tool_dependency_dir'\n"
msg += "setting in your main Galaxy configuration file (e.g., uninverse_wsgi.ini).\n"
processed_tool_dependencies = []
for missing_tool_config, tool_dependencies in missing_tool_configs_dict.items():
for tool_dependencies_tup in missing_tool_configs_dict[ missing_tool_config ][ 'tool_dependencies' ]:
if tool_dependencies_tup not in processed_tool_dependencies:
msg += "------------------------------------\n"
msg += "Tool Dependency\n"
msg += "------------------------------------\n"
msg += "Name: %s, Version: %s, Type: %s\n" % ( tool_dependencies_tup[ 0 ],
tool_dependencies_tup[ 1 ],
tool_dependencies_tup[ 2 ] )
if len( tool_dependencies_tup ) >= 4:
msg += "Requirements and installation information:\n"
msg += "%s\n" % tool_dependencies_tup[ 3 ]
else:
msg += "\n"
msg += "------------------------------------\n"
processed_tool_dependencies.append( tool_dependencies_tup )
msg += "\n"
msg += "%s" % output.replace( 'done', '' )
msg += "vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv\n"
msg += "sh ./scripts/migrate_tools/%04d_tools.sh\n" % latest_tool_migration_script_number
msg += "^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\n"
if have_tool_dependencies:
msg += "The tool dependencies listed above will be installed along with the repositories if you add the 'install_dependencies'\n"
msg += "option to the above command like this:\n\n"
msg += "vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv\n"
msg += "sh ./scripts/migrate_tools/%04d_tools.sh install_dependencies\n" % latest_tool_migration_script_number
msg += "^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n\n"
msg += "Tool dependencies can be installed after the repositories have been installed as well.\n\n"
msg += "After the installation process finishes, you can start your Galaxy server. As part of this installation process,\n"
msg += "entries for each of the following tool config files will be added to the file named ./migrated_tool_conf.xml, so these\n"
msg += "tools will continue to be loaded into your tool panel. Because of this, existing entries for these tools have been\n"
msg += "removed from your file%s named %s.\n\n" % ( plural, tool_panel_config_file_names )
for missing_tool_config, tool_dependencies in missing_tool_configs_dict.items():
msg += "%s\n" % missing_tool_config
msg += "<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<\n"
raise Exception( msg )
else:
log.debug( "The main Galaxy tool shed is not currently available, so skipped tool migration %s until next server startup" % db_schema.version )
else:
log.info( "At migrate_tools version %d" % db_schema.version )
import inspect
from traceback import format_exc
from functools import wraps
from galaxy import eggs
eggs.require("Paste")
import paste.httpexceptions
from galaxy.web.framework import url_for
from galaxy import util
from galaxy.exceptions import error_codes
from galaxy.exceptions import MessageException
from galaxy.util.json import loads
from galaxy.util.json import safe_dumps as dumps
import logging
log = logging.getLogger(__name__)
JSON_CONTENT_TYPE = "application/json"
def error(message):
raise MessageException(message, type='error')
# ----------------------------------------------------------------------------- web controller decorators
def _save_orig_fn(wrapped, orig):
if not hasattr(orig, '_orig'):
wrapped._orig = orig
return wrapped
#!/usr/bin/env python
"""
Convert from pileup file to interval index file.
usage: %prog <options> in_file out_file
"""
from __future__ import division
import optparse
from galaxy import eggs
eggs.require( "bx-python" )
from bx.interval_index_file import Indexes
def main():
# Read options, args.
parser = optparse.OptionParser()
(options, args) = parser.parse_args()
input_fname, output_fname = args
# Do conversion.
index = Indexes()
offset = 0
for line in open( input_fname, "r" ):
chrom, start = line.split()[ 0:2 ]
# Pileup format is 1-based.
start = int( start ) - 1
index.add( chrom, start, start + 1, offset )
offset += len( line )
import sys
import os.path
import logging
from galaxy import eggs
eggs.require("SQLAlchemy")
eggs.require("six") # Required by sqlalchemy-migrate
eggs.require("sqlparse") # Required by sqlalchemy-migrate
eggs.require("decorator") # Required by sqlalchemy-migrate
eggs.require("Tempita ") # Required by sqlalchemy-migrate
eggs.require("sqlalchemy-migrate")
from migrate.versioning import repository, schema
from sqlalchemy import create_engine
from galaxy.model.orm import dialect_to_egg
from sqlalchemy import MetaData
from sqlalchemy.exc import NoSuchTableError
from sqlalchemy import Table
log = logging.getLogger(__name__)
# path relative to galaxy
migrate_repository_directory = os.path.abspath(
os.path.dirname(__file__)).replace(os.getcwd() + os.path.sep, '', 1)
migrate_repository = repository.Repository(migrate_repository_directory)
def create_or_verify_database(url,
galaxy_config_file,
engine_options={},
#!/usr/bin/env python
import ConfigParser
import logging
import os
import re
import sys
import argparse
new_path = [ os.path.join( os.getcwd(), "lib" ) ]
new_path.extend( sys.path[1:] )
sys.path = new_path
from galaxy import eggs
eggs.require( "SQLAlchemy >= 0.4" )
eggs.require( 'mercurial' )
import galaxy.config
from galaxy.web import security
from galaxy.model import mapping
log = logging.getLogger( __name__ )
VALID_PUBLICNAME_RE = re.compile( "^[a-z0-9\-]+$" )
VALID_EMAIL_RE = re.compile( "[^@]+@[^@]+\.[^@]+" )
class BootstrapGalaxyApplication( object ):
"""
def create_or_verify_database(url,
galaxy_config_file,
engine_options={},
app=None):
"""
Check that the database is use-able, possibly creating it if empty (this is
the only time we automatically create tables, otherwise we force the
user to do it using the management script so they can create backups).
1) Empty database --> initialize with latest version and return
2) Database older than migration support --> fail and require manual update
3) Database at state where migrate support introduced --> add version control information but make no changes (might still require manual update)
4) Database versioned but out of date --> fail with informative message, user must run "sh manage_db.sh upgrade"
"""
dialect = (url.split(':', 1))[0]
try:
egg = dialect_to_egg[dialect]
try:
eggs.require(egg)
log.debug("%s egg successfully loaded for %s dialect" %
(egg, dialect))
except:
# If the module is in the path elsewhere (i.e. non-egg), it'll still load.
log.warning(
"%s egg not found, but an attempt will be made to use %s anyway"
% (egg, dialect))
except KeyError:
# Let this go, it could possibly work with db's we don't support
log.error(
"database_connection contains an unknown SQLAlchemy database dialect: %s"
% dialect)
# Create engine and metadata
engine = create_engine(url, **engine_options)
def migrate():
try:
# Declare the database to be under a repository's version control
db_schema = schema.ControlledSchema.create(engine,
migrate_repository)
except:
# The database is already under version control
db_schema = schema.ControlledSchema(engine, migrate_repository)
# Apply all scripts to get to current version
migrate_to_current_version(engine, db_schema)
meta = MetaData(bind=engine)
if app and getattr(app.config, 'database_auto_migrate', False):
migrate()
return
# Try to load dataset table
try:
Table("dataset", meta, autoload=True)
except NoSuchTableError:
# No 'dataset' table means a completely uninitialized database. If we have an app, we'll
# set its new_installation setting to True so the tool migration process will be skipped.
if app:
app.new_installation = True
log.info("No database, initializing")
migrate()
return
try:
hda_table = Table("history_dataset_association", meta, autoload=True)
except NoSuchTableError:
raise Exception(
"Your database is older than hg revision 1464:c7acaa1bb88f and will need to be updated manually"
)
# There is a 'history_dataset_association' table, so we (hopefully) have
# version 1 of the database, but without the migrate_version table. This
# happens if the user has a build from right before migration was added.
# Verify that this is true, if it is any older they'll have to update
# manually
if 'copied_from_history_dataset_association_id' not in hda_table.c:
# The 'copied_from_history_dataset_association_id' column was added in
# rev 1464:c7acaa1bb88f. This is the oldest revision we currently do
# automated versioning for, so stop here
raise Exception(
"Your database is older than hg revision 1464:c7acaa1bb88f and will need to be updated manually"
)
# At revision 1464:c7acaa1bb88f or greater (database version 1), make sure
# that the db has version information. This is the trickiest case -- we
# have a database but no version control, and are assuming it is a certain
# version. If the user has postion version 1 changes this could cause
# problems
try:
Table("migrate_version", meta, autoload=True)
except NoSuchTableError:
# The database exists but is not yet under migrate version control, so init with version 1
log.info("Adding version control to existing database")
try:
Table("metadata_file", meta, autoload=True)
schema.ControlledSchema.create(engine,
migrate_repository,
version=2)
except NoSuchTableError:
schema.ControlledSchema.create(engine,
migrate_repository,
version=1)
# Verify that the code and the DB are in sync
db_schema = schema.ControlledSchema(engine, migrate_repository)
if migrate_repository.versions.latest != db_schema.version:
config_arg = ''
if os.path.abspath(os.path.join(os.getcwd(), 'config',
'galaxy.ini')) != galaxy_config_file:
config_arg = ' -c %s' % galaxy_config_file.replace(
os.path.abspath(os.getcwd()), '.')
raise Exception(
"Your database has version '%d' but this code expects version '%d'. Please backup your database and then migrate the schema by running 'sh manage_db.sh%s upgrade'."
% (db_schema.version, migrate_repository.versions.latest,
config_arg))
else:
log.info("At database version %d" % db_schema.version)
"""Module for searching the toolshed repositories"""
import datetime
from galaxy import exceptions
from galaxy import eggs
from galaxy.webapps.tool_shed import model
import logging
log = logging.getLogger(__name__)
# Whoosh is compatible with Python 2.5+
# Try to import Whoosh and set flag to indicate whether
# the tool search is ready.
try:
eggs.require("Whoosh")
import whoosh.index
from whoosh import scoring
from whoosh.fields import Schema, STORED, ID, KEYWORD, TEXT, STORED
from whoosh.scoring import BM25F
from whoosh.qparser import MultifieldParser
from whoosh.index import Index
search_ready = True
schema = Schema(id=STORED,
name=TEXT(field_boost=1.7, stored=True),
description=TEXT(field_boost=1.5, stored=True),
long_description=TEXT(stored=True),
homepage_url=TEXT(stored=True),
remote_repository_url=TEXT(stored=True),
repo_owner_username=TEXT(stored=True),
times_downloaded=STORED,
approved=STORED,
last_updated=STORED,
"""
Code to support database helper scripts (create_db.py, manage_db.py, etc...).
"""
import logging
import os.path
from ConfigParser import SafeConfigParser
from galaxy import eggs
eggs.require("decorator")
eggs.require("Tempita")
eggs.require("SQLAlchemy")
eggs.require("sqlalchemy_migrate")
from galaxy.model.orm import dialect_to_egg
import pkg_resources
log = logging.getLogger(__name__)
DEFAULT_CONFIG_FILE = 'universe_wsgi.ini'
DEFAULT_CONFIG_PREFIX = ''
DEFAULT_DATABASE = 'galaxy'
DATABASE = {
"galaxy": {
'repo': 'lib/galaxy/model/migrate',
'default_sqlite_file': './database/universe.sqlite',
},
"tool_shed": {
'repo': 'lib/galaxy/webapps/tool_shed/model/migrate',
from __future__ import absolute_import
try:
from galaxy import eggs
eggs.require("requests")
except ImportError:
pass
try:
import requests
except ImportError:
requests = None
requests_multipart_post_available = False
try:
import requests_toolbelt
requests_multipart_post_available = True
except ImportError:
requests_toolbelt = None
REQUESTS_UNAVAILABLE_MESSAGE = "Pulsar configured to use requests module - but it is unavailable. Please install requests."
REQUESTS_TOOLBELT_UNAVAILABLE_MESSAGE = "Pulsar configured to use requests_toolbelt module - but it is unavailable. Please install requests_toolbelt."
import logging
log = logging.getLogger(__name__)
def post_file(url, path):
if requests_toolbelt is None:
raise ImportError(REQUESTS_TOOLBELT_UNAVAILABLE_MESSAGE)
__ensure_requests()
import tempfile
import re
from ConfigParser import SafeConfigParser
# Assume we are run from the galaxy root directory, add lib to the python path
cwd = os.getcwd()
new_path = [os.path.join(cwd, "lib"), os.path.join(cwd, "test")]
new_path.extend(sys.path[1:])
sys.path = new_path
from base.tool_shed_util import parse_tool_panel_config
from galaxy import eggs
from galaxy.util.properties import load_app_properties
eggs.require("nose")
eggs.require("NoseHTML")
eggs.require("NoseTestDiff")
eggs.require("twill==0.9")
eggs.require("Paste")
eggs.require("PasteDeploy")
eggs.require("Cheetah")
# this should not be required, but it is under certain conditions, thanks to this bug:
# http://code.google.com/p/python-nose/issues/detail?id=284
eggs.require("pysqlite")
import atexit
import logging
import os.path
import twill
import smtplib import stat import string import sys import tempfile import threading from email.MIMEText import MIMEText from os.path import relpath from hashlib import md5 from itertools import izip from galaxy import eggs eggs.require( 'docutils' ) import docutils.core import docutils.writers.html4css1 eggs.require( 'elementtree' ) from elementtree import ElementTree, ElementInclude eggs.require( "wchartype" ) import wchartype from .inflection import Inflector, English inflector = Inflector(English) log = logging.getLogger(__name__) _lock = threading.RLock()
"""
Galaxy web framework helpers
"""
import time
from cgi import escape
from datetime import datetime, timedelta
from galaxy import eggs
from galaxy.util import hash_util
from galaxy.util.json import dumps
eggs.require( "MarkupSafe" ) #required by WebHelpers
eggs.require( "WebHelpers" )
from webhelpers import date
from webhelpers.html.tags import stylesheet_link, javascript_link
eggs.require( "Routes" )
from routes import url_for
server_starttime = int(time.time())
def time_ago( x ):
"""
Convert a datetime to a string.
"""
delta = timedelta(weeks=1)
# If the date is more than one week ago, then display the actual date instead of in words
if (datetime.utcnow() - x) > delta: # Greater than a week difference
return x.strftime("%b %d, %Y")
else:
date_array = date.distance_of_time_in_words( x, datetime.utcnow() ).replace(",", "").split(" ")
from __future__ import absolute_import
import logging
from galaxy import eggs
eggs.require('SQLAlchemy')
from sqlalchemy import false
eggs.require( "MarkupSafe" )
from markupsafe import escape
from galaxy import model, util
from galaxy.web.base.controller import BaseUIController, UsesFormDefinitionsMixin, web
from galaxy.web.form_builder import build_select_field, TextField
from galaxy.web.framework.helpers import iff, grids
from .requests_common import invalid_id_redirect
log = logging.getLogger( __name__ )
class RequestTypeGrid( grids.Grid ):
# Custom column types
class NameColumn( grids.TextColumn ):
def get_value(self, trans, grid, request_type):
return escape(request_type.name)
class DescriptionColumn( grids.TextColumn ):
def get_value(self, trans, grid, request_type):
return escape(request_type.desc)
class RequestFormColumn( grids.TextColumn ):
def get_value(self, trans, grid, request_type):
"""
import copy
import cPickle
import json
import os
import shutil
import sys
import tempfile
import weakref
from os.path import abspath
from galaxy import eggs
eggs.require("SQLAlchemy >= 0.4")
from sqlalchemy.orm import object_session
import galaxy.model
from galaxy.util import listify
from galaxy.util.object_wrapper import sanitize_lists_to_string
from galaxy.util import stringify_dictionary_keys
from galaxy.util import string_as_bool
from galaxy.util import in_directory
from galaxy.util.odict import odict
from galaxy.web import form_builder
import logging
log = logging.getLogger(__name__)
STATEMENTS = "__galaxy_statements__" #this is the name of the property in a Datatype class where new metadata spec element Statements are stored
import os
import sys
import shutil
import logging
import inspect
import datetime
from ConfigParser import ConfigParser
from optparse import OptionParser
galaxy_root = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
sys.path.insert(0, os.path.join(galaxy_root, "lib"))
from galaxy import eggs
eggs.require("psycopg2")
eggs.require("SQLAlchemy")
import psycopg2
from sqlalchemy.engine.url import make_url
import galaxy.config
from galaxy.exceptions import ObjectNotFound
from galaxy.objectstore import build_object_store_from_config
from galaxy.util.bunch import Bunch
log = logging.getLogger()
class MetadataFile(Bunch):
pass
from __future__ import absolute_import
from galaxy import eggs
eggs.require("Mako")
import mako.exceptions
def build_template_error_formatters():
"""
Build a list of template error formatters for WebError. When an error
occurs, WebError pass the exception to each function in this list until
one returns a value, which will be displayed on the error page.
"""
formatters = []
# Formatter for mako
def mako_html_data(exc_value):
if isinstance(exc_value, (mako.exceptions.CompileException,
mako.exceptions.SyntaxException)):
return mako.exceptions.html_error_template().render(full=False,
css=False)
if isinstance(exc_value,
AttributeError) and exc_value.args[0].startswith(
"'Undefined' object has no attribute"):
return mako.exceptions.html_error_template().render(full=False,
css=False)
formatters.append(mako_html_data)
return formatters
import logging import os import shutil import sys from string import Template from galaxy.util import unicodify, nice_size from galaxy import eggs eggs.require( 'MarkupSafe' ) import markupsafe log = logging.getLogger( __name__ ) CHUNK_SIZE = 2**20 # 1Mb INSTALLATION_LOG = 'INSTALLATION.log' # Set no activity timeout to 20 minutes. NO_OUTPUT_TIMEOUT = 3600.0 MAXDIFFSIZE = 8000 MAX_DISPLAY_SIZE = 32768 DOCKER_IMAGE_TEMPLATE = ''' # Galaxy Docker image FROM bgruening/galaxy-stable MAINTAINER Bjoern A. Gruning, [email protected] WORKDIR /galaxy-central
#!/usr/bin/env python
"""
Small library with cheminformatic functions based on openbabel and pgchem.
Copyright 2012, Bjoern Gruening and Xavier Lucas
"""
import os, sys
try:
from galaxy import eggs
eggs.require('psycopg2')
except:
print(
'psycopg2 is not available. It is currently used in the pgchem wrappers, that are not shipped with default CTB'
)
try:
import pybel
import openbabel
except:
print(
'OpenBabel could not be found. A few functions are not available without OpenBabel.'
)
from multiprocessing import Pool
import glob, tempfile, re
import subprocess
def CountLines(path):
out = subprocess.Popen(['wc', '-l', path],
# -*- coding: utf-8 -*-
import os
import imp
import unittest
test_utils = imp.load_source( 'test_utils',
os.path.join( os.path.dirname( __file__), '../unittest_utils/utility.py' ) )
from galaxy import eggs
eggs.require( 'SQLAlchemy >= 0.4' )
import sqlalchemy
from galaxy import model
from galaxy import exceptions
from base import BaseTestCase
from galaxy.managers.histories import HistoryManager
from galaxy.managers.datasets import DatasetManager
from galaxy.managers import hdas
# =============================================================================
default_password = '******'
user2_data = dict( email='[email protected]', username='******', password=default_password )
user3_data = dict( email='[email protected]', username='******', password=default_password )
# =============================================================================
class HDATestCase( BaseTestCase ):
import tempfile import threading from galaxy.util import json from email.MIMEText import MIMEText from os.path import relpath from hashlib import md5 from itertools import izip from urlparse import urlparse from galaxy import eggs eggs.require( 'docutils' ) import docutils.core import docutils.writers.html4css1 eggs.require( 'elementtree' ) from elementtree import ElementTree, ElementInclude eggs.require( "wchartype" ) import wchartype from .inflection import Inflector, English inflector = Inflector(English) log = logging.getLogger(__name__) _lock = threading.RLock()
from galaxy import eggs
eggs.require( "SVGFig" )
import svgfig
MARGIN = 5
LINE_SPACING = 15
class WorkflowCanvas( object ):
def __init__( self ):
self.canvas = svgfig.canvas( style="stroke:black; fill:none; stroke-width:1px; stroke-linejoin:round; text-anchor:left" )
self.text = svgfig.SVG( "g" )
self.connectors = svgfig.SVG( "g" )
self.boxes = svgfig.SVG( "g" )
svgfig.Text.defaults[ "font-size" ] = "10px"
self.in_pos = {}
self.out_pos = {}
self.widths = {}
self.max_x = 0
self.max_y = 0
self.max_width = 0
self.data = []
def finish( self ):
max_x, max_y, max_width = self.max_x, self.max_y, self.max_width
import json
import logging
import os
import string
import subprocess
import sys
import time
from galaxy import eggs
from galaxy import model
from galaxy.jobs import JobDestination
from galaxy.jobs.handler import DEFAULT_JOB_PUT_FAILURE_MESSAGE
from galaxy.jobs.runners import AsynchronousJobState, AsynchronousJobRunner
eggs.require("drmaa")
log = logging.getLogger(__name__)
__all__ = ['DRMAAJobRunner']
drmaa = None
DRMAA_jobTemplate_attributes = [
'args', 'remoteCommand', 'outputPath', 'errorPath', 'nativeSpecification',
'jobName', 'email', 'project'
]
class DRMAAJobRunner(AsynchronousJobRunner):
"""
""" """ import datetime import inspect import os import hashlib import random import socket import string import time from Cookie import CookieError from galaxy import eggs eggs.require( "Cheetah" ) from Cheetah.Template import Template eggs.require( "Mako" ) import mako.runtime import mako.lookup # pytz is used by Babel. eggs.require( "pytz" ) eggs.require( "Babel" ) from babel.support import Translations from babel import Locale eggs.require( "SQLAlchemy >= 0.4" ) from sqlalchemy import and_ from sqlalchemy.orm.exc import NoResultFound from sqlalchemy.orm import joinedload from galaxy.exceptions import MessageException from galaxy import util
"""
Actions to be run at job completion (or output hda creation, as in the case of
immediate_actions listed below. Currently only used in workflows.
"""
import datetime
import logging
import socket
from galaxy.util import send_mail
from galaxy.util.json import dumps
from galaxy import eggs
eggs.require("MarkupSafe")
from markupsafe import escape
log = logging.getLogger(__name__)
def get_form_template(action_type, title, content, help, on_output=True):
if on_output:
form = """
if (pja.action_type == "%s"){
p_str = "<div class='pjaForm toolForm'><span class='action_tag' style='display:none'>"+ pja.action_type + pja.output_name + "</span><div class='toolFormTitle'> %s <br/> on " + pja.output_name + "\
<div style='float: right;' class='buttons'><img src='/static/images/history-buttons/delete_icon.png'></div></div><div class='toolFormBody'>";
%s
p_str += "</div><div class='toolParamHelp'>%s</div></div>";
}""" % (action_type, title, content, help)
else:
form = """
if (pja.action_type == "%s"){
p_str = "<div class='pjaForm toolForm'><span class='action_tag' style='display:none'>"+ pja.action_type + "</span><div class='toolFormTitle'> %s \
<div style='float: right;' class='buttons'><img src='/static/images/history-buttons/delete_icon.png'></div></div><div class='toolFormBody'>";
import logging
import os
import subprocess
import sys
from galaxy import eggs
eggs.require( "decorator" )
eggs.require( "Tempita" )
eggs.require( "six" ) # Required by sqlalchemy-migrate
eggs.require( "sqlparse" ) # Required by sqlalchemy-migrate
eggs.require( "SQLAlchemy" )
eggs.require( "sqlalchemy_migrate" )
from migrate.versioning import repository
from migrate.versioning import schema
from sqlalchemy import create_engine
from sqlalchemy import MetaData
from sqlalchemy import Table
from galaxy.util.odict import odict
from galaxy.model.orm import dialect_to_egg
from tool_shed.util import common_util
log = logging.getLogger( __name__ )
# Path relative to galaxy
migrate_repository_directory = os.path.dirname( __file__ ).replace( os.getcwd() + os.path.sep, '', 1 )
migrate_repository = repository.Repository( migrate_repository_directory )
def verify_tools( app, url, galaxy_config_file, engine_options={} ):
# Check the value in the migrate_tools.version database table column to verify that the number is in
# sync with the number of version scripts in ~/lib/galaxy/tools/migrate/versions.
dialect = ( url.split( ':', 1 ) )[0]
try:
#!/usr/bin/env python
# Guruprasad Ananda
# Refactored 2011 to use numpy instead of rpy, Kanwei Li
"""
This tool provides the SQL "group by" functionality.
"""
import sys, commands, tempfile, random
try:
import numpy
except:
from galaxy import eggs
eggs.require( "numpy" )
import numpy
from itertools import groupby
def stop_err(msg):
sys.stderr.write(msg)
sys.exit()
def mode(data):
counts = {}
for x in data:
counts[x] = counts.get(x,0) + 1
maxcount = max(counts.values())
modelist = []
for x in counts:
if counts[x] == maxcount:
modelist.append( str(x) )
return ','.join(modelist)
"""
Module for building and searching the index of tools
installed within this Galaxy.
"""
from galaxy import eggs
from galaxy.web.framework.helpers import to_unicode
eggs.require( "Whoosh" )
from whoosh.filedb.filestore import RamStorage
from whoosh.fields import Schema, STORED, TEXT
from whoosh.scoring import BM25F
from whoosh.qparser import MultifieldParser
schema = Schema( id=STORED,
name=TEXT,
description=TEXT,
section=TEXT,
help=TEXT )
import logging
log = logging.getLogger( __name__ )
class ToolBoxSearch( object ):
"""
Support searching tools in a toolbox. This implementation uses
the Whoosh search library.
"""
def __init__( self, toolbox, index_help=True ):
"""
Create a searcher for `toolbox`.
"""
import logging
import os
from datetime import datetime, timedelta
from decimal import Decimal
from galaxy import eggs
eggs.require('SQLAlchemy')
from sqlalchemy import and_, desc, false, null, true
from sqlalchemy.orm import eagerload
from galaxy.web.base.controller import BaseUIController, web
from galaxy import model, util
log = logging.getLogger( __name__ )
class System( BaseUIController ):
@web.expose
def index( self, trans, **kwd ):
params = util.Params( kwd )
message = ''
if params.userless_histories_days:
userless_histories_days = params.userless_histories_days
else:
userless_histories_days = '60'
if params.deleted_histories_days:
deleted_histories_days = params.deleted_histories_days
else:
deleted_histories_days = '60'
if params.deleted_datasets_days:
deleted_datasets_days = params.deleted_datasets_days
import logging
from tool_shed.repository_types.metadata import TipOnly
import tool_shed.repository_types.util as rt_util
from tool_shed.util import basic_util
from galaxy import eggs
eggs.require( 'mercurial' )
from mercurial import hg
from mercurial import ui
log = logging.getLogger( __name__ )
class ToolDependencyDefinition( TipOnly ):
def __init__( self ):
self.type = rt_util.TOOL_DEPENDENCY_DEFINITION
self.label = 'Tool dependency definition'
self.valid_file_names = [ 'tool_dependencies.xml' ]
def is_valid_for_type( self, app, repository, revisions_to_check=None ):
"""
Inspect the received repository's contents to determine if they abide by the rules defined for the contents of this type.
If the received revisions_to_check is a list of changeset revisions, then inspection will be restricted to the revisions
in the list.
"""
repo = hg.repository( ui.ui(), repository.repo_path( app ) )
if revisions_to_check:
changeset_revisions = revisions_to_check
else:
changeset_revisions = repo.changelog
for changeset in changeset_revisions:
#!/usr/bin/env python
"""
Convert from VCF file to interval index file.
"""
from __future__ import division
import optparse
from galaxy import eggs
eggs.require("bx-python")
from bx.interval_index_file import Indexes
import galaxy_utils.sequence.vcf
def main():
# Read options, args.
parser = optparse.OptionParser()
(options, args) = parser.parse_args()
in_file, out_file = args
# Do conversion.
index = Indexes()
reader = galaxy_utils.sequence.vcf.Reader(open(in_file))
offset = reader.metadata_len
for vcf_line in reader:
# VCF format provides a chrom and 1-based position for each variant.
# IntervalIndex expects 0-based coordinates.
index.add(vcf_line.chrom, vcf_line.pos - 1, vcf_line.pos, offset)
offset += len(vcf_line.raw_line)
from galaxy.eggs import require
from galaxy.web.framework.helpers import to_unicode
# Whoosh is compatible with Python 2.5+ Try to import Whoosh and set flag to indicate whether tool search is enabled.
try:
require( "Whoosh" )
from whoosh.filedb.filestore import RamStorage
from whoosh.fields import Schema, STORED, ID, KEYWORD, TEXT
from whoosh.index import Index
from whoosh.scoring import BM25F
from whoosh.qparser import MultifieldParser
tool_search_enabled = True
schema = Schema( id = STORED, title = TEXT, description = TEXT, help = TEXT )
except ImportError, e:
tool_search_enabled = False
schema = None
class ToolBoxSearch( object ):
"""
Support searching tools in a toolbox. This implementation uses
the "whoosh" search library.
"""
def __init__( self, toolbox ):
"""
Create a searcher for `toolbox`.
"""
self.toolbox = toolbox
self.enabled = tool_search_enabled
if tool_search_enabled:
self.build_index()
This parallelizes the task over available cores using multiprocessing.
Code mostly taken form CloudBioLinux.
"""
import contextlib
import functools
import glob
import multiprocessing
import os
import subprocess
from multiprocessing.pool import IMapIterator
try:
from galaxy import eggs
eggs.require('boto')
except ImportError:
pass
try:
import boto
from boto.s3.connection import S3Connection
except ImportError:
boto = None
def map_wrap(f):
@functools.wraps(f)
def wrapper(*args, **kwargs):
return apply(f, *args, **kwargs)
# # You can also use this script as a library, for instance see https://gist.github.com/1979583 # TODO: This script overlaps a lot with manage_db.py and create_db.py, # these should maybe be refactored to remove duplication. import sys import os.path db_shell_path = __file__ new_path = [ os.path.join( os.path.dirname( db_shell_path ), os.path.pardir, "lib" ) ] new_path.extend( sys.path[1:] ) # remove scripts/ from the path sys.path = new_path from galaxy.model.orm.scripts import get_config from galaxy import eggs eggs.require( "decorator" ) eggs.require( "Tempita" ) eggs.require( "SQLAlchemy" ) db_url = get_config( sys.argv )['db_url'] # Setup DB scripting environment from sqlalchemy import * from sqlalchemy.orm import * from sqlalchemy.exc import * from galaxy.model.mapping import init sa_session = init( '/tmp/', db_url ).context from galaxy.model import *
import os, logging
from galaxy.web.base.controller import *
from galaxy.webapps.community.controllers.common import *
from galaxy import eggs
eggs.require('mercurial')
from mercurial.hgweb.hgwebdir_mod import hgwebdir
from mercurial.hgweb.request import wsgiapplication
log = logging.getLogger(__name__)
class HgController(BaseUIController):
@web.expose
def handle_request(self, trans, **kwd):
# The os command that results in this method being called will look something like
# hg clone http://[email protected]:9009/repos/test/convert_characters1
cmd = kwd.get('cmd', None)
wsgi_app = wsgiapplication(make_web_app)
# Hack: Add a parameter to requests for which we do not want all repository metadata reset.
reset_metadata = not (kwd.get('no_reset', False))
if cmd == 'listkeys' and reset_metadata:
# This possibly results from an "hg push" from the command line. When doing this, the following 7 commands, in order,
# will be retrieved from environ: between -> capabilities -> heads -> branchmap -> unbundle -> unbundle -> listkeys
path_info = kwd.get('path_info', None)
if path_info:
owner, name = path_info.split('/')
repository = get_repository_by_name_and_owner(
trans, name, owner)
if repository:
import sys import tempfile from ConfigParser import SafeConfigParser # Assume we are run from the galaxy root directory, add lib to the python path cwd = os.getcwd() new_path = [ os.path.join( cwd, "lib" ), os.path.join( cwd, "test" ) ] new_path.extend( sys.path[1:] ) sys.path = new_path from base.tool_shed_util import parse_tool_panel_config from galaxy import eggs from galaxy.util.properties import load_app_properties eggs.require( "nose" ) eggs.require( "NoseHTML" ) eggs.require( "NoseTestDiff" ) eggs.require( "Paste" ) eggs.require( "PasteDeploy" ) eggs.require( "Cheetah" ) # this should not be required, but it is under certain conditions, thanks to this bug: # http://code.google.com/p/python-nose/issues/detail?id=284 eggs.require( "pysqlite" ) import logging import os.path import time import threading import random
from __future__ import absolute_import
import logging
try:
from galaxy import eggs
eggs.require("requests")
except ImportError:
pass
try:
import requests
except ImportError:
requests = None
try:
import requests_toolbelt
requests_multipart_post_available = True
except ImportError:
requests_multipart_post_available = False
requests_toolbelt = None
REQUESTS_UNAVAILABLE_MESSAGE = "Pulsar configured to use requests module - but it is unavailable. Please install requests."
REQUESTS_TOOLBELT_UNAVAILABLE_MESSAGE = "Pulsar configured to use requests_toolbelt module - but it is unavailable. Please install requests_toolbelt."
log = logging.getLogger(__name__)
def post_file(url, path):
if requests_toolbelt is None:
raise ImportError(REQUESTS_TOOLBELT_UNAVAILABLE_MESSAGE)
