def load_galaxy_app(config_builder, config_env=False, log=None, **kwds):
# Allow specification of log so daemon can reuse properly configured one.
if log is None:
log = logging.getLogger(__name__)
# If called in daemon mode, set the ROOT directory and ensure Galaxy is on
# sys.path.
if config_env:
try:
os.chdir(GALAXY_ROOT_DIR)
except Exception:
log.exception("Failed to chdir")
raise
try:
sys.path.insert(1, GALAXY_LIB_DIR)
except Exception:
log.exception("Failed to add Galaxy to sys.path")
raise
config_builder.setup_logging()
from galaxy.util.properties import load_app_properties
kwds = config_builder.app_kwds()
kwds = load_app_properties(**kwds)
from galaxy.app import UniverseApplication
app = UniverseApplication(
global_conf={"__file__": config_builder.ini_path}, **kwds)
app.setup_control_queue()
return app
def build_galaxy_app(simple_kwargs) -> GalaxyUniverseApplication:
"""Build a Galaxy app object from a simple keyword arguments.
Construct paste style complex dictionary and use load_app_properties so
Galaxy override variables are respected. Also setup "global" references
to sqlalchemy database context for Galaxy and install databases.
"""
log.info("Galaxy database connection: %s",
simple_kwargs["database_connection"])
simple_kwargs['global_conf'] = get_webapp_global_conf()
simple_kwargs['global_conf'][
'__file__'] = "lib/galaxy/config/sample/galaxy.yml.sample"
simple_kwargs = load_app_properties(kwds=simple_kwargs)
# Build the Universe Application
app = GalaxyUniverseApplication(**simple_kwargs)
if not simple_kwargs.get("enable_celery_tasks"):
rebind_container_to_task(app)
log.info("Embedded Galaxy application started")
global galaxy_context
global install_context
galaxy_context = app.model.context
install_context = app.install_model.context
# Toolbox indexing happens via the work queue out of band recently, and,
# beyond potentially running async after tests execute doesn't execute
# without building a uwsgi app (app.is_webapp = False for this test kit).
# We need to ensure to build an index for the test galaxy app -- this is
# pretty fast with the limited toolset
app.reindex_tool_search()
return app
def load_galaxy_app(
config_builder,
config_env=False,
log=None,
**kwds
):
# Allow specification of log so daemon can reuse properly configured one.
if log is None:
log = logging.getLogger(__name__)
# If called in daemon mode, set the ROOT directory and ensure Galaxy is on
# sys.path.
if config_env:
try:
os.chdir(GALAXY_ROOT_DIR)
except Exception:
log.exception("Failed to chdir")
raise
try:
sys.path.insert(1, GALAXY_LIB_DIR)
except Exception:
log.exception("Failed to add Galaxy to sys.path")
raise
config_builder.setup_logging()
from galaxy.util.properties import load_app_properties
kwds = config_builder.app_kwds()
kwds = load_app_properties(**kwds)
from galaxy.app import UniverseApplication
app = UniverseApplication(
global_conf={"__file__": config_builder.ini_path},
**kwds
)
app.setup_control_queue()
return app
def load_galaxy_app(config_builder,
config_env=False,
log=None,
attach_to_pools=None,
**kwds):
# Allow specification of log so daemon can reuse properly configured one.
if log is None:
log = logging.getLogger(__name__)
# If called in daemon mode, set the ROOT directory and ensure Galaxy is on
# sys.path.
if config_env:
try:
os.chdir(GALAXY_ROOT_DIR)
except Exception:
log.exception("Failed to chdir")
raise
config_builder.setup_logging()
from galaxy.util.properties import load_app_properties
kwds = config_builder.app_kwds()
kwds = load_app_properties(**kwds)
from galaxy.app import UniverseApplication
app = UniverseApplication(global_conf=config_builder.global_conf(),
attach_to_pools=attach_to_pools,
**kwds)
app.database_heartbeat.start()
app.application_stack.log_startup()
return app
def app_factory( global_conf, **kwargs ):
"""
Return a wsgi application serving the root object
"""
# Create the Galaxy application unless passed in
if 'app' in kwargs:
app = kwargs.pop( 'app' )
else:
try:
from galaxy.app import UniverseApplication
app = UniverseApplication( global_conf = global_conf, **kwargs )
except:
import traceback, sys
traceback.print_exc()
sys.exit( 1 )
atexit.register( app.shutdown )
# Create the universe WSGI application
if app.config.log_memory_usage:
from galaxy.web.framework.memdebug import MemoryLoggingWebApplication
webapp = MemoryLoggingWebApplication( app, session_cookie='galaxysession' )
else:
webapp = galaxy.web.framework.WebApplication( app, session_cookie='galaxysession' )
# Find controllers
add_controllers( webapp, app )
# Force /history to go to /root/history -- needed since the tests assume this
webapp.add_route( '/history', controller='root', action='history' )
# These two routes handle our simple needs at the moment
webapp.add_route( '/async/:tool_id/:data_id/:data_secret', controller='async', action='index', tool_id=None, data_id=None, data_secret=None )
webapp.add_route( '/:controller/:action', action='index' )
webapp.add_route( '/:action', controller='root', action='index' )
webapp.add_route( '/datasets/:dataset_id/:action/:filename', controller='dataset', action='index', dataset_id=None, filename=None)
webapp.add_route( '/u/:username/p/:slug', controller='page', action='display_by_username_and_slug' )
webapp.finalize_config()
# Wrap the webapp in some useful middleware
if kwargs.get( 'middleware', True ):
webapp = wrap_in_middleware( webapp, global_conf, **kwargs )
if asbool( kwargs.get( 'static_enabled', True ) ):
webapp = wrap_in_static( webapp, global_conf, **kwargs )
# Close any pooled database connections before forking
try:
galaxy.model.mapping.metadata.engine.connection_provider._pool.dispose()
except:
pass
# Return
return webapp
def app_factory(global_conf, **kwargs):
"""
Return a wsgi application serving the root object
"""
# Create the application
if 'app' in kwargs:
app = kwargs.pop('app')
else:
app = UniverseApplication(**kwargs)
atexit.register(app.shutdown)
# Create the universe WSGI application
webapp = web.framework.WebApplication()
# Transaction that provides access to unvierse history
webapp.set_transaction_factory(
lambda e: web.UniverseWebTransaction(e, app))
# Add controllers to the web application
webapp.add_controller('root', root.Universe(app))
webapp.add_controller('tool_runner', tool_runner.ToolRunner(app))
webapp.add_controller('ucsc_proxy', proxy.UCSCProxy(app))
webapp.add_controller('async', async .ASync(app))
webapp.add_controller('admin', admin.Admin(app))
webapp.add_controller('user', user.User(app))
webapp.add_controller('error', error.Error(app))
# These two routes handle our simple needs at the moment
webapp.add_route('/async/:tool_id/:data_id',
controller='async',
action='index',
tool_id=None,
data_id=None)
webapp.add_route('/:controller/:action', action='index')
webapp.add_route('/:action', controller='root', action='index')
webapp.finalize_config()
# Wrap the webapp in some useful middleware
if kwargs.get('middleware', True):
webapp = middleware.make_middleware(webapp, global_conf, **kwargs)
return webapp
def main():
# ---- Configuration ------------------------------------------------------
galaxy_test_host = os.environ.get( 'GALAXY_TEST_HOST', default_galaxy_test_host )
galaxy_test_port = os.environ.get( 'GALAXY_TEST_PORT', None )
galaxy_test_save = os.environ.get( 'GALAXY_TEST_SAVE', None)
tool_path = os.environ.get( 'GALAXY_TEST_TOOL_PATH', 'tools' )
if 'HTTP_ACCEPT_LANGUAGE' not in os.environ:
os.environ[ 'HTTP_ACCEPT_LANGUAGE' ] = default_galaxy_locales
testing_migrated_tools = __check_arg( '-migrated' )
testing_installed_tools = __check_arg( '-installed' )
datatypes_conf_override = None
if testing_migrated_tools or testing_installed_tools:
# Store a jsonified dictionary of tool_id : GALAXY_TEST_FILE_DIR pairs.
galaxy_tool_shed_test_file = 'shed_tools_dict'
# We need the upload tool for functional tests, so we'll create a temporary tool panel config that defines it.
fd, tmp_tool_panel_conf = tempfile.mkstemp()
os.write( fd, '<?xml version="1.0"?>\n' )
os.write( fd, '<toolbox>\n' )
os.write( fd, '<tool file="data_source/upload.xml"/>\n' )
os.write( fd, '</toolbox>\n' )
os.close( fd )
tool_config_file = tmp_tool_panel_conf
galaxy_test_file_dir = None
library_import_dir = None
user_library_import_dir = None
# Exclude all files except test_toolbox.py.
ignore_files = ( re.compile( r'^test_[adghlmsu]*' ), re.compile( r'^test_ta*' ) )
else:
framework_tool_dir = os.path.join('test', 'functional', 'tools')
framework_test = __check_arg( '-framework' ) # Run through suite of tests testing framework.
if framework_test:
tool_conf = os.path.join( framework_tool_dir, 'samples_tool_conf.xml' )
datatypes_conf_override = os.path.join( framework_tool_dir, 'sample_datatypes_conf.xml' )
else:
# Use tool_conf.xml toolbox.
tool_conf = None
if __check_arg( '-with_framework_test_tools' ):
tool_conf = "%s,%s" % ( 'config/tool_conf.xml.sample', os.path.join( framework_tool_dir, 'samples_tool_conf.xml' ) )
test_dir = default_galaxy_test_file_dir
tool_config_file = os.environ.get( 'GALAXY_TEST_TOOL_CONF', tool_conf )
galaxy_test_file_dir = os.environ.get( 'GALAXY_TEST_FILE_DIR', test_dir )
first_test_file_dir = galaxy_test_file_dir.split(",")[0]
if not os.path.isabs( first_test_file_dir ):
first_test_file_dir = os.path.join( os.getcwd(), first_test_file_dir )
library_import_dir = first_test_file_dir
import_dir = os.path.join( first_test_file_dir, 'users' )
if os.path.exists(import_dir):
user_library_import_dir = import_dir
else:
user_library_import_dir = None
ignore_files = ()
start_server = 'GALAXY_TEST_EXTERNAL' not in os.environ
tool_data_table_config_path = None
if os.path.exists( 'tool_data_table_conf.test.xml' ):
# If explicitly defined tables for test, use those.
tool_data_table_config_path = 'tool_data_table_conf.test.xml'
else:
# ... otherise find whatever Galaxy would use as the default and
# the sample data for fucntional tests to that.
default_tool_data_config = 'config/tool_data_table_conf.xml.sample'
for tool_data_config in ['config/tool_data_table_conf.xml', 'tool_data_table_conf.xml' ]:
if os.path.exists( tool_data_config ):
default_tool_data_config = tool_data_config
tool_data_table_config_path = '%s,test/functional/tool-data/sample_tool_data_tables.xml' % default_tool_data_config
default_data_manager_config = 'config/data_manager_conf.xml.sample'
for data_manager_config in ['config/data_manager_conf.xml', 'data_manager_conf.xml' ]:
if os.path.exists( data_manager_config ):
default_data_manager_config = data_manager_config
data_manager_config_file = "%s,test/functional/tools/sample_data_manager_conf.xml" % default_data_manager_config
shed_tool_data_table_config = 'config/shed_tool_data_table_conf.xml'
tool_dependency_dir = os.environ.get( 'GALAXY_TOOL_DEPENDENCY_DIR', None )
use_distributed_object_store = os.environ.get( 'GALAXY_USE_DISTRIBUTED_OBJECT_STORE', False )
galaxy_test_tmp_dir = os.environ.get( 'GALAXY_TEST_TMP_DIR', None )
if galaxy_test_tmp_dir is None:
galaxy_test_tmp_dir = tempfile.mkdtemp()
galaxy_job_conf_file = os.environ.get( 'GALAXY_TEST_JOB_CONF',
os.path.join( galaxy_test_tmp_dir, 'test_job_conf.xml' ) )
# Generate the job_conf.xml file.
file( galaxy_job_conf_file, 'w' ).write( job_conf_xml )
database_auto_migrate = False
galaxy_test_proxy_port = None
if start_server:
tempdir = tempfile.mkdtemp( dir=galaxy_test_tmp_dir )
# Configure the database path.
if 'GALAXY_TEST_DBPATH' in os.environ:
galaxy_db_path = os.environ[ 'GALAXY_TEST_DBPATH' ]
else:
galaxy_db_path = os.path.join( tempdir, 'database' )
# Configure the paths Galaxy needs to test tools.
file_path = os.path.join( galaxy_db_path, 'files' )
new_file_path = tempfile.mkdtemp( prefix='new_files_path_', dir=tempdir )
job_working_directory = tempfile.mkdtemp( prefix='job_working_directory_', dir=tempdir )
install_database_connection = os.environ.get( 'GALAXY_TEST_INSTALL_DBURI', None )
if 'GALAXY_TEST_DBURI' in os.environ:
database_connection = os.environ['GALAXY_TEST_DBURI']
else:
db_path = os.path.join( galaxy_db_path, 'universe.sqlite' )
if 'GALAXY_TEST_DB_TEMPLATE' in os.environ:
# Middle ground between recreating a completely new
# database and pointing at existing database with
# GALAXY_TEST_DBURI. The former requires a lot of setup
# time, the latter results in test failures in certain
# cases (namely tool shed tests expecting clean database).
log.debug( "Copying database template from %s.", os.environ['GALAXY_TEST_DB_TEMPLATE'] )
__copy_database_template(os.environ['GALAXY_TEST_DB_TEMPLATE'], db_path)
database_auto_migrate = True
database_connection = 'sqlite:///%s' % db_path
kwargs = {}
for dir in file_path, new_file_path:
try:
if not os.path.exists( dir ):
os.makedirs( dir )
except OSError:
pass
# Data Manager testing temp path
# For storing Data Manager outputs and .loc files so that real ones don't get clobbered
data_manager_test_tmp_path = tempfile.mkdtemp( prefix='data_manager_test_tmp', dir=galaxy_test_tmp_dir )
galaxy_data_manager_data_path = tempfile.mkdtemp( prefix='data_manager_tool-data', dir=data_manager_test_tmp_path )
# ---- Build Application --------------------------------------------------
master_api_key = get_master_api_key()
app = None
if start_server:
kwargs = dict( admin_users='*****@*****.**',
api_allow_run_as='*****@*****.**',
allow_library_path_paste=True,
allow_user_creation=True,
allow_user_deletion=True,
database_connection=database_connection,
database_auto_migrate=database_auto_migrate,
datatype_converters_config_file="datatype_converters_conf.xml.sample",
file_path=file_path,
id_secret='changethisinproductiontoo',
job_queue_workers=5,
job_working_directory=job_working_directory,
library_import_dir=library_import_dir,
log_destination="stdout",
new_file_path=new_file_path,
running_functional_tests=True,
shed_tool_data_table_config=shed_tool_data_table_config,
template_path="templates",
test_conf="test.conf",
tool_config_file=tool_config_file,
tool_data_table_config_path=tool_data_table_config_path,
tool_path=tool_path,
galaxy_data_manager_data_path=galaxy_data_manager_data_path,
tool_parse_help=False,
update_integrated_tool_panel=False,
use_heartbeat=False,
user_library_import_dir=user_library_import_dir,
master_api_key=master_api_key,
use_tasked_jobs=True,
cleanup_job='onsuccess',
enable_beta_tool_formats=True,
data_manager_config_file=data_manager_config_file )
if install_database_connection is not None:
kwargs[ 'install_database_connection' ] = install_database_connection
if not database_connection.startswith( 'sqlite://' ):
kwargs[ 'database_engine_option_max_overflow' ] = '20'
kwargs[ 'database_engine_option_pool_size' ] = '10'
if tool_dependency_dir is not None:
kwargs[ 'tool_dependency_dir' ] = tool_dependency_dir
if use_distributed_object_store:
kwargs[ 'object_store' ] = 'distributed'
kwargs[ 'distributed_object_store_config_file' ] = 'distributed_object_store_conf.xml.sample'
if datatypes_conf_override:
kwargs[ 'datatypes_config_file' ] = datatypes_conf_override
# If the user has passed in a path for the .ini file, do not overwrite it.
galaxy_config_file = os.environ.get( 'GALAXY_TEST_INI_FILE', None )
if not galaxy_config_file:
galaxy_config_file = os.path.join( galaxy_test_tmp_dir, 'functional_tests_wsgi.ini' )
config_items = []
for label in kwargs:
config_tuple = label, kwargs[ label ]
config_items.append( config_tuple )
# Write a temporary file, based on config/galaxy.ini.sample, using the configuration options defined above.
generate_config_file( 'config/galaxy.ini.sample', galaxy_config_file, config_items )
# Set the global_conf[ '__file__' ] option to the location of the temporary .ini file, which gets passed to set_metadata.sh.
kwargs[ 'global_conf' ] = get_webapp_global_conf()
kwargs[ 'global_conf' ][ '__file__' ] = galaxy_config_file
kwargs[ 'config_file' ] = galaxy_config_file
kwargs = load_app_properties(
kwds=kwargs
)
# Build the Universe Application
app = UniverseApplication( **kwargs )
database_contexts.galaxy_context = app.model.context
log.info( "Embedded Universe application started" )
# ---- Run webserver ------------------------------------------------------
server = None
if start_server:
webapp = buildapp.app_factory( kwargs[ 'global_conf' ], app=app,
use_translogger=False, static_enabled=STATIC_ENABLED )
if galaxy_test_port is not None:
server = httpserver.serve( webapp, host=galaxy_test_host, port=galaxy_test_port, start_loop=False )
else:
random.seed()
for i in range( 0, 9 ):
try:
galaxy_test_port = str( random.randint( default_galaxy_test_port_min, default_galaxy_test_port_max ) )
log.debug( "Attempting to serve app on randomly chosen port: %s" % galaxy_test_port )
server = httpserver.serve( webapp, host=galaxy_test_host, port=galaxy_test_port, start_loop=False )
break
except socket.error, e:
if e[0] == 98:
continue
raise
else:
raise Exception( "Unable to open a port between %s and %s to start Galaxy server" % ( default_galaxy_test_port_min, default_galaxy_test_port_max ) )
kwargs[ 'database_engine_option_max_overflow' ] = '20'
galaxyapp = GalaxyUniverseApplication( allow_user_creation = True,
allow_user_deletion = True,
admin_users = '*****@*****.**',
allow_library_path_paste = True,
database_connection = galaxy_database_connection,
datatype_converters_config_file = "datatype_converters_conf.xml.sample",
enable_tool_shed_check = True,
file_path = galaxy_file_path,
global_conf = global_conf,
hours_between_check = 0.001,
id_secret = 'changethisinproductiontoo',
job_queue_workers = 5,
log_destination = "stdout",
migrated_tools_config = galaxy_migrated_tool_conf_file,
new_file_path = galaxy_tempfiles,
running_functional_tests=True,
shed_tool_data_table_config = shed_tool_data_table_conf_file,
shed_tool_path = galaxy_shed_tool_path,
template_path = "templates",
tool_data_path = tool_data_path,
tool_dependency_dir = galaxy_tool_dependency_dir,
tool_path = tool_path,
tool_config_file = [ galaxy_tool_conf_file, galaxy_shed_tool_conf_file ],
tool_sheds_config_file = galaxy_tool_sheds_conf_file,
tool_parse_help = False,
tool_data_table_config_path = galaxy_tool_data_table_conf_file,
update_integrated_tool_panel = False,
use_heartbeat = False,
**kwargs )
def main():
if install_and_test_base_util.tool_shed_api_key is None:
# If the tool shed URL specified in any dict is not present in the tool_sheds_conf.xml, the installation will fail.
log.debug(
'Cannot proceed without a valid tool shed API key set in the enviroment variable GALAXY_INSTALL_TEST_TOOL_SHED_API_KEY.'
)
return 1
if install_and_test_base_util.galaxy_tool_shed_url is None:
log.debug(
'Cannot proceed without a valid Tool Shed base URL set in the environment variable GALAXY_INSTALL_TEST_TOOL_SHED_URL.'
)
return 1
# ---- Configuration ------------------------------------------------------
galaxy_test_host = os.environ.get(
'GALAXY_INSTALL_TEST_HOST',
install_and_test_base_util.default_galaxy_test_host)
# Set the GALAXY_INSTALL_TEST_HOST variable so that Twill will have the Galaxy url to which to
# install repositories.
os.environ['GALAXY_INSTALL_TEST_HOST'] = galaxy_test_host
# Set the GALAXY_TEST_HOST environment variable so that the toolbox tests will have the Galaxy url
# on which to to run tool functional tests.
os.environ['GALAXY_TEST_HOST'] = galaxy_test_host
galaxy_test_port = os.environ.get(
'GALAXY_INSTALL_TEST_PORT',
str(install_and_test_base_util.default_galaxy_test_port_max))
os.environ['GALAXY_TEST_PORT'] = galaxy_test_port
tool_path = os.environ.get('GALAXY_INSTALL_TEST_TOOL_PATH', 'tools')
if 'HTTP_ACCEPT_LANGUAGE' not in os.environ:
os.environ['HTTP_ACCEPT_LANGUAGE'] = default_galaxy_locales
galaxy_test_file_dir = os.environ.get('GALAXY_INSTALL_TEST_FILE_DIR',
default_galaxy_test_file_dir)
if not os.path.isabs(galaxy_test_file_dir):
galaxy_test_file_dir = os.path.abspath(galaxy_test_file_dir)
use_distributed_object_store = os.environ.get(
'GALAXY_INSTALL_TEST_USE_DISTRIBUTED_OBJECT_STORE', False)
if not os.path.isdir(galaxy_test_tmp_dir):
os.mkdir(galaxy_test_tmp_dir)
# Set up the configuration files for the Galaxy instance.
galaxy_shed_tool_path = os.environ.get(
'GALAXY_INSTALL_TEST_SHED_TOOL_PATH',
tempfile.mkdtemp(dir=galaxy_test_tmp_dir, prefix='shed_tools'))
shed_tool_data_table_conf_file = os.environ.get(
'GALAXY_INSTALL_TEST_SHED_TOOL_DATA_TABLE_CONF',
os.path.join(galaxy_test_tmp_dir,
'test_shed_tool_data_table_conf.xml'))
galaxy_tool_data_table_conf_file = os.environ.get(
'GALAXY_INSTALL_TEST_TOOL_DATA_TABLE_CONF',
install_and_test_base_util.tool_data_table_conf)
galaxy_tool_conf_file = os.environ.get(
'GALAXY_INSTALL_TEST_TOOL_CONF',
os.path.join(galaxy_test_tmp_dir, 'test_tool_conf.xml'))
galaxy_job_conf_file = os.environ.get(
'GALAXY_INSTALL_TEST_JOB_CONF',
os.path.join(galaxy_test_tmp_dir, 'test_job_conf.xml'))
galaxy_shed_tool_conf_file = os.environ.get(
'GALAXY_INSTALL_TEST_SHED_TOOL_CONF',
os.path.join(galaxy_test_tmp_dir, 'test_shed_tool_conf.xml'))
galaxy_migrated_tool_conf_file = os.environ.get(
'GALAXY_INSTALL_TEST_MIGRATED_TOOL_CONF',
os.path.join(galaxy_test_tmp_dir, 'test_migrated_tool_conf.xml'))
galaxy_tool_sheds_conf_file = os.environ.get(
'GALAXY_INSTALL_TEST_TOOL_SHEDS_CONF',
os.path.join(galaxy_test_tmp_dir, 'test_tool_sheds_conf.xml'))
galaxy_shed_tools_dict_file = os.environ.get(
'GALAXY_INSTALL_TEST_SHED_TOOL_DICT_FILE',
os.path.join(galaxy_test_tmp_dir, 'shed_tool_dict'))
install_and_test_base_util.populate_galaxy_shed_tools_dict_file(
galaxy_shed_tools_dict_file, shed_tools_dict=None)
# Set the GALAXY_TOOL_SHED_TEST_FILE environment variable to the path of the shed_tools_dict file so that
# test.base.twilltestcase.setUp will find and parse it properly.
os.environ['GALAXY_TOOL_SHED_TEST_FILE'] = galaxy_shed_tools_dict_file
if 'GALAXY_INSTALL_TEST_TOOL_DATA_PATH' in os.environ:
tool_data_path = os.environ.get('GALAXY_INSTALL_TEST_TOOL_DATA_PATH')
else:
tool_data_path = tempfile.mkdtemp(dir=galaxy_test_tmp_dir)
os.environ['GALAXY_INSTALL_TEST_TOOL_DATA_PATH'] = tool_data_path
# Configure the database connection and path.
if 'GALAXY_INSTALL_TEST_DBPATH' in os.environ:
galaxy_db_path = os.environ['GALAXY_INSTALL_TEST_DBPATH']
else:
tempdir = tempfile.mkdtemp(dir=galaxy_test_tmp_dir)
galaxy_db_path = os.path.join(tempdir, 'database')
# Configure the paths Galaxy needs to install and test tools.
galaxy_file_path = os.path.join(galaxy_db_path, 'files')
new_repos_path = tempfile.mkdtemp(dir=galaxy_test_tmp_dir)
galaxy_tempfiles = tempfile.mkdtemp(dir=galaxy_test_tmp_dir)
galaxy_migrated_tool_path = tempfile.mkdtemp(dir=galaxy_test_tmp_dir)
# Set up the tool dependency path for the Galaxy instance.
tool_dependency_dir = os.environ.get(
'GALAXY_INSTALL_TEST_TOOL_DEPENDENCY_DIR', None)
if tool_dependency_dir is None:
tool_dependency_dir = tempfile.mkdtemp(dir=galaxy_test_tmp_dir)
os.environ[
'GALAXY_INSTALL_TEST_TOOL_DEPENDENCY_DIR'] = tool_dependency_dir
os.environ['GALAXY_TOOL_DEPENDENCY_DIR'] = tool_dependency_dir
if 'GALAXY_INSTALL_TEST_DBURI' in os.environ:
database_connection = os.environ['GALAXY_INSTALL_TEST_DBURI']
else:
database_connection = 'sqlite:///' + os.path.join(
galaxy_db_path, 'install_and_test_repositories.sqlite')
if 'GALAXY_INSTALL_TEST_INSTALL_DBURI' in os.environ:
install_database_connection = os.environ[
'GALAXY_INSTALL_TEST_INSTALL_DBURI']
elif asbool(
os.environ.get('GALAXY_TEST_INSTALL_DB_MERGED',
default_install_db_merged)):
install_database_connection = database_connection
else:
install_galaxy_db_path = os.path.join(galaxy_db_path, 'install.sqlite')
install_database_connection = 'sqlite:///%s' % install_galaxy_db_path
kwargs = {}
for dir in [galaxy_test_tmp_dir]:
try:
os.makedirs(dir)
except OSError:
pass
print "Database connection: ", database_connection
print "Install database connection: ", install_database_connection
# Generate the shed_tool_data_table_conf.xml file.
file(shed_tool_data_table_conf_file, 'w').write(
install_and_test_base_util.tool_data_table_conf_xml_template)
os.environ[
'GALAXY_INSTALL_TEST_SHED_TOOL_DATA_TABLE_CONF'] = shed_tool_data_table_conf_file
# ---- Start up a Galaxy instance ------------------------------------------------------
# Generate the tool_conf.xml file.
file(galaxy_tool_conf_file,
'w').write(install_and_test_base_util.tool_conf_xml)
# Generate the job_conf.xml file.
file(galaxy_job_conf_file,
'w').write(install_and_test_base_util.job_conf_xml)
# Generate the tool_sheds_conf.xml file, but only if a the user has not specified an existing one in the environment.
if 'GALAXY_INSTALL_TEST_TOOL_SHEDS_CONF' not in os.environ:
file(galaxy_tool_sheds_conf_file,
'w').write(install_and_test_base_util.tool_sheds_conf_xml)
# Generate the shed_tool_conf.xml file.
install_and_test_base_util.populate_shed_conf_file(
galaxy_shed_tool_conf_file, galaxy_shed_tool_path, xml_elems=None)
os.environ[
'GALAXY_INSTALL_TEST_SHED_TOOL_CONF'] = galaxy_shed_tool_conf_file
# Generate the migrated_tool_conf.xml file.
install_and_test_base_util.populate_shed_conf_file(
galaxy_migrated_tool_conf_file,
galaxy_migrated_tool_path,
xml_elems=None)
# Write the embedded web application's specific configuration to a temporary file. This is necessary in order for
# the external metadata script to find the right datasets.
kwargs = dict(
admin_users='*****@*****.**',
master_api_key=install_and_test_base_util.
default_galaxy_master_api_key,
allow_user_creation=True,
allow_user_deletion=True,
allow_library_path_paste=True,
database_connection=database_connection,
datatype_converters_config_file="datatype_converters_conf.xml.sample",
file_path=galaxy_file_path,
id_secret=install_and_test_base_util.galaxy_encode_secret,
install_database_connection=install_database_connection,
job_config_file=galaxy_job_conf_file,
job_queue_workers=5,
log_destination="stdout",
migrated_tools_config=galaxy_migrated_tool_conf_file,
new_file_path=galaxy_tempfiles,
running_functional_tests=True,
shed_tool_data_table_config=shed_tool_data_table_conf_file,
shed_tool_path=galaxy_shed_tool_path,
template_path="templates",
tool_config_file=','.join(
[galaxy_tool_conf_file, galaxy_shed_tool_conf_file]),
tool_data_path=tool_data_path,
tool_dependency_dir=tool_dependency_dir,
tool_path=tool_path,
tool_parse_help=False,
tool_sheds_config_file=galaxy_tool_sheds_conf_file,
update_integrated_tool_panel=False,
use_heartbeat=False)
if os.path.exists(galaxy_tool_data_table_conf_file):
kwargs[
'tool_data_table_config_path'] = galaxy_tool_data_table_conf_file
galaxy_config_file = os.environ.get('GALAXY_INSTALL_TEST_INI_FILE', None)
# If the user has passed in a path for the .ini file, do not overwrite it.
if not galaxy_config_file:
galaxy_config_file = os.path.join(
galaxy_test_tmp_dir,
'install_test_tool_shed_repositories_wsgi.ini')
config_items = []
for label in kwargs:
config_tuple = label, kwargs[label]
config_items.append(config_tuple)
# Write a temporary file, based on galaxy.ini.sample, using the configuration options defined above.
generate_config_file('config/galaxy.ini.sample', galaxy_config_file,
config_items)
kwargs['tool_config_file'] = [
galaxy_tool_conf_file, galaxy_shed_tool_conf_file
]
# Set the global_conf[ '__file__' ] option to the location of the temporary .ini file, which gets passed to set_metadata.sh.
kwargs['global_conf'] = install_and_test_base_util.get_webapp_global_conf()
kwargs['global_conf']['__file__'] = galaxy_config_file
# ---- Build Galaxy Application --------------------------------------------------
if not database_connection.startswith('sqlite://'):
kwargs['database_engine_option_max_overflow'] = '20'
kwargs['database_engine_option_pool_size'] = '10'
kwargs['config_file'] = galaxy_config_file
app = UniverseApplication(**kwargs)
database_contexts.galaxy_context = app.model.context
database_contexts.install_context = app.install_model.context
log.debug("Embedded Galaxy application started...")
# ---- Run galaxy webserver ------------------------------------------------------
server = None
global_conf = install_and_test_base_util.get_webapp_global_conf()
global_conf['database_file'] = database_connection
webapp = buildapp.app_factory(
global_conf,
use_translogger=False,
static_enabled=install_and_test_base_util.STATIC_ENABLED,
app=app)
# Serve the app on a specified or random port.
if galaxy_test_port is not None:
server = httpserver.serve(webapp,
host=galaxy_test_host,
port=galaxy_test_port,
start_loop=False)
else:
random.seed()
for i in range(0, 9):
try:
galaxy_test_port = str(
random.randint(
install_and_test_base_util.
default_galaxy_test_port_min,
install_and_test_base_util.default_galaxy_test_port_max
))
log.debug(
"Attempting to serve app on randomly chosen port: %s",
galaxy_test_port)
server = httpserver.serve(webapp,
host=galaxy_test_host,
port=galaxy_test_port,
start_loop=False)
break
except socket.error, e:
if e[0] == 98:
continue
raise
else:
def app_factory(global_conf, **kwargs):
"""
Return a wsgi application serving the root object
"""
kwargs = load_app_properties(kwds=kwargs)
# Create the Galaxy application unless passed in
if 'app' in kwargs:
app = kwargs.pop('app')
else:
try:
from galaxy.app import UniverseApplication
app = UniverseApplication(global_conf=global_conf, **kwargs)
except:
import traceback
traceback.print_exc()
sys.exit(1)
# Call app's shutdown method when the interpeter exits, this cleanly stops
# the various Galaxy application daemon threads
atexit.register(app.shutdown)
# Create the universe WSGI application
webapp = GalaxyWebApplication(app,
session_cookie='galaxysession',
name='galaxy')
webapp.add_ui_controllers('galaxy.webapps.galaxy.controllers', app)
# Force /history to go to /root/history -- needed since the tests assume this
webapp.add_route('/history', controller='root', action='history')
# Force /activate to go to the controller
webapp.add_route('/activate', controller='user', action='activate')
# These two routes handle our simple needs at the moment
webapp.add_route('/async/:tool_id/:data_id/:data_secret',
controller='async',
action='index',
tool_id=None,
data_id=None,
data_secret=None)
webapp.add_route('/:controller/:action', action='index')
webapp.add_route('/:action', controller='root', action='index')
# allow for subdirectories in extra_files_path
webapp.add_route('/datasets/:dataset_id/display/{filename:.+?}',
controller='dataset',
action='display',
dataset_id=None,
filename=None)
webapp.add_route('/datasets/:dataset_id/:action/:filename',
controller='dataset',
action='index',
dataset_id=None,
filename=None)
webapp.add_route(
'/display_application/:dataset_id/:app_name/:link_name/:user_id/:app_action/:action_param',
controller='dataset',
action='display_application',
dataset_id=None,
user_id=None,
app_name=None,
link_name=None,
app_action=None,
action_param=None)
webapp.add_route('/u/:username/d/:slug/:filename',
controller='dataset',
action='display_by_username_and_slug',
filename=None)
webapp.add_route('/u/:username/p/:slug',
controller='page',
action='display_by_username_and_slug')
webapp.add_route('/u/:username/h/:slug',
controller='history',
action='display_by_username_and_slug')
webapp.add_route('/u/:username/w/:slug',
controller='workflow',
action='display_by_username_and_slug')
webapp.add_route('/u/:username/w/:slug/:format',
controller='workflow',
action='display_by_username_and_slug')
webapp.add_route('/u/:username/v/:slug',
controller='visualization',
action='display_by_username_and_slug')
webapp.add_route('/search', controller='search', action='index')
# TODO: Refactor above routes into external method to allow testing in
# isolation as well.
populate_api_routes(webapp, app)
# ==== Done
# Indicate that all configuration settings have been provided
webapp.finalize_config()
# Wrap the webapp in some useful middleware
if kwargs.get('middleware', True):
webapp = wrap_in_middleware(webapp, global_conf, **kwargs)
if asbool(kwargs.get('static_enabled', True)):
webapp = wrap_in_static(
webapp,
global_conf,
plugin_frameworks=[app.visualizations_registry],
**kwargs)
#webapp = wrap_in_static( webapp, global_conf, plugin_frameworks=None, **kwargs )
if asbool(kwargs.get('pack_scripts', False)):
pack_scripts()
# Close any pooled database connections before forking
try:
galaxy.model.mapping.metadata.engine.connection_provider._pool.dispose(
)
except:
pass
# Close any pooled database connections before forking
try:
galaxy.model.tool_shed_install.mapping.metadata.engine.connection_provider._pool.dispose(
)
except:
pass
# Return
return webapp
tool_config_file=[
galaxy_tool_conf_file, galaxy_shed_tool_conf_file
],
tool_sheds_config_file=galaxy_tool_sheds_conf_file,
tool_parse_help=False,
tool_data_table_config_path=galaxy_tool_data_table_conf_file,
update_integrated_tool_panel=False,
use_heartbeat=False)
# ---- Build Galaxy Application --------------------------------------------------
if not galaxy_database_connection.startswith(
'sqlite://'
) and not install_galaxy_database_connection.startswith('sqlite://'):
kwargs['database_engine_option_pool_size'] = '10'
kwargs['database_engine_option_max_overflow'] = '20'
galaxyapp = GalaxyUniverseApplication(**kwargs)
log.info("Embedded Galaxy application started")
# ---- Run galaxy webserver ------------------------------------------------------
galaxy_server = None
galaxy_global_conf['database_file'] = galaxy_database_connection
galaxywebapp = galaxybuildapp.app_factory(galaxy_global_conf,
use_translogger=False,
static_enabled=True,
app=galaxyapp)
database_contexts.galaxy_context = galaxyapp.model.context
database_contexts.install_context = galaxyapp.install_model.context
if galaxy_test_port is not None:
galaxy_server = httpserver.serve(galaxywebapp,
host=galaxy_test_host,
def main():
# ---- Configuration ------------------------------------------------------
galaxy_test_host = os.environ.get('GALAXY_TEST_HOST',
default_galaxy_test_host)
galaxy_test_port = os.environ.get('GALAXY_TEST_PORT', None)
if 'HTTP_ACCEPT_LANGUAGE' not in os.environ:
os.environ['HTTP_ACCEPT_LANGUAGE'] = default_galaxy_locales
galaxy_test_file_dir = os.environ.get('GALAXY_TEST_FILE_DIR',
default_galaxy_test_file_dir)
if not os.path.isabs(galaxy_test_file_dir):
galaxy_test_file_dir = os.path.join(os.getcwd(), galaxy_test_file_dir)
start_server = 'GALAXY_TEST_EXTERNAL' not in os.environ
if start_server:
if 'GALAXY_TEST_DBPATH' in os.environ:
db_path = os.environ['GALAXY_TEST_DBPATH']
else:
tempdir = tempfile.mkdtemp()
db_path = os.path.join(tempdir, 'database')
file_path = os.path.join(db_path, 'files')
try:
os.makedirs(file_path)
except OSError:
pass
if 'GALAXY_TEST_DBURI' in os.environ:
database_connection = os.environ['GALAXY_TEST_DBURI']
else:
database_connection = 'sqlite:///' + os.path.join(
db_path, 'universe.sqlite')
if 'GALAXY_TEST_RUNNERS' in os.environ:
start_job_runners = os.environ['GALAXY_TEST_RUNNERS']
else:
start_job_runners = None
if 'GALAXY_TEST_DEF_RUNNER' in os.environ:
default_cluster_job_runner = os.environ['GALAXY_TEST_DEF_RUNNER']
else:
default_cluster_job_runner = 'local:///'
set_metadata_externally = False
if 'GALAXY_SET_METADATA_EXTERNALLY' in os.environ:
set_metadata_externally = True
print "Database connection:", database_connection
# What requires these?
os.environ['GALAXY_TEST_HOST'] = galaxy_test_host
os.environ['GALAXY_TEST_FILE_DIR'] = galaxy_test_file_dir
# ---- Build Application --------------------------------------------------
app = None
if start_server:
# Build the Universe Application
app = UniverseApplication(
job_queue_workers=5,
start_job_runners=start_job_runners,
default_cluster_job_runner=default_cluster_job_runner,
id_secret='changethisinproductiontoo',
template_path="templates",
database_connection=database_connection,
file_path=file_path,
tool_config_file="tool_conf.xml.sample",
datatype_converters_config_file=
"datatype_converters_conf.xml.sample",
tool_path="tools",
tool_parse_help=False,
test_conf="test.conf",
log_destination="stdout",
use_heartbeat=False,
allow_user_creation=True,
allow_user_deletion=True,
admin_users='*****@*****.**',
library_import_dir=galaxy_test_file_dir,
user_library_import_dir=os.path.join(galaxy_test_file_dir,
'users'),
set_metadata_externally=set_metadata_externally,
global_conf={"__file__": "universe_wsgi.ini.sample"})
log.info("Embedded Universe application started")
# ---- Run webserver ------------------------------------------------------
server = None
if start_server:
webapp = buildapp.app_factory(dict(),
use_translogger=False,
static_enabled=False,
app=app)
if galaxy_test_port is not None:
server = httpserver.serve(webapp,
host=galaxy_test_host,
port=galaxy_test_port,
start_loop=False)
else:
random.seed()
for i in range(0, 9):
try:
galaxy_test_port = str(
random.randint(default_galaxy_test_port_min,
default_galaxy_test_port_max))
log.debug(
"Attempting to serve app on randomly chosen port: %s" %
galaxy_test_port)
server = httpserver.serve(webapp,
host=galaxy_test_host,
port=galaxy_test_port,
start_loop=False)
break
except socket.error, e:
if e[0] == 98:
continue
raise
else:
raise Exception(
"Unable to open a port between %s and %s to start Galaxy server"
% (default_galaxy_test_port_min,
default_galaxy_test_port_max))
def app_factory(global_conf, **kwargs):
"""
Return a wsgi application serving the root object
"""
# Create the Galaxy application unless passed in
if 'app' in kwargs:
app = kwargs.pop('app')
else:
try:
from galaxy.app import UniverseApplication
app = UniverseApplication(global_conf=global_conf, **kwargs)
except:
import traceback
traceback.print_exc()
sys.exit(1)
# Call app's shutdown method when the interpeter exits, this cleanly stops
# the various Galaxy application daemon threads
atexit.register(app.shutdown)
# Create the universe WSGI application
webapp = GalaxyWebApplication(app,
session_cookie='galaxysession',
name='galaxy')
# The following route will handle displaying tool shelp images for tools contained in repositories installed from the tool shed.
webapp.add_route('/tool_runner/static/images/:repository_id/:image_file',
controller='tool_runner',
action='display_tool_help_image_in_repository',
repository_id=None,
image_file=None)
webapp.add_ui_controllers('galaxy.webapps.galaxy.controllers', app)
# Force /history to go to /root/history -- needed since the tests assume this
webapp.add_route('/history', controller='root', action='history')
# These two routes handle our simple needs at the moment
webapp.add_route('/async/:tool_id/:data_id/:data_secret',
controller='async',
action='index',
tool_id=None,
data_id=None,
data_secret=None)
webapp.add_route('/:controller/:action', action='index')
webapp.add_route('/:action', controller='root', action='index')
# allow for subdirectories in extra_files_path
webapp.add_route('/datasets/:dataset_id/display/{filename:.+?}',
controller='dataset',
action='display',
dataset_id=None,
filename=None)
webapp.add_route('/datasets/:dataset_id/:action/:filename',
controller='dataset',
action='index',
dataset_id=None,
filename=None)
webapp.add_route(
'/display_application/:dataset_id/:app_name/:link_name/:user_id/:app_action/:action_param',
controller='dataset',
action='display_application',
dataset_id=None,
user_id=None,
app_name=None,
link_name=None,
app_action=None,
action_param=None)
webapp.add_route('/u/:username/d/:slug/:filename',
controller='dataset',
action='display_by_username_and_slug',
filename=None)
webapp.add_route('/u/:username/p/:slug',
controller='page',
action='display_by_username_and_slug')
webapp.add_route('/u/:username/h/:slug',
controller='history',
action='display_by_username_and_slug')
webapp.add_route('/u/:username/w/:slug',
controller='workflow',
action='display_by_username_and_slug')
webapp.add_route('/u/:username/v/:slug',
controller='visualization',
action='display_by_username_and_slug')
webapp.add_route('/search', controller='search', action='index')
# Add the web API
webapp.add_api_controllers('galaxy.webapps.galaxy.api', app)
# The /folders section is experimental at this point:
log.debug("app.config.api_folders: %s" % app.config.api_folders)
webapp.mapper.resource('folder', 'folders', path_prefix='/api')
webapp.mapper.resource('content',
'contents',
controller='folder_contents',
name_prefix='folder_',
path_prefix='/api/folders/:folder_id',
parent_resources=dict(member_name='folder',
collection_name='folders'))
webapp.mapper.resource('content',
'contents',
controller='library_contents',
name_prefix='library_',
path_prefix='/api/libraries/:library_id',
parent_resources=dict(member_name='library',
collection_name='libraries'))
webapp.mapper.resource('content',
'contents',
controller='history_contents',
name_prefix='history_',
path_prefix='/api/histories/:history_id',
parent_resources=dict(member_name='history',
collection_name='histories'))
webapp.mapper.connect(
"history_contents_display",
"/api/histories/:history_id/contents/:history_content_id/display",
controller="datasets",
action="display",
conditions=dict(method=["GET"]))
webapp.mapper.resource('permission',
'permissions',
path_prefix='/api/libraries/:library_id',
parent_resources=dict(member_name='library',
collection_name='libraries'))
webapp.mapper.resource('user',
'users',
controller='group_users',
name_prefix='group_',
path_prefix='/api/groups/:group_id',
parent_resources=dict(member_name='group',
collection_name='groups'))
webapp.mapper.resource('role',
'roles',
controller='group_roles',
name_prefix='group_',
path_prefix='/api/groups/:group_id',
parent_resources=dict(member_name='group',
collection_name='groups'))
_add_item_tags_controller(
webapp,
name_prefix="history_content_",
path_prefix='/api/histories/:history_id/contents/:history_content_id')
_add_item_tags_controller(webapp,
name_prefix="history_",
path_prefix='/api/histories/:history_id')
_add_item_tags_controller(webapp,
name_prefix="workflow_",
path_prefix='/api/workflows/:workflow_id')
_add_item_annotation_controller(
webapp,
name_prefix="history_content_",
path_prefix='/api/histories/:history_id/contents/:history_content_id')
_add_item_annotation_controller(webapp,
name_prefix="history_",
path_prefix='/api/histories/:history_id')
_add_item_annotation_controller(webapp,
name_prefix="workflow_",
path_prefix='/api/workflows/:workflow_id')
_add_item_provenance_controller(
webapp,
name_prefix="history_content_",
path_prefix='/api/histories/:history_id/contents/:history_content_id')
webapp.mapper.resource('dataset', 'datasets', path_prefix='/api')
webapp.mapper.resource_with_deleted('library',
'libraries',
path_prefix='/api')
webapp.mapper.resource('sample', 'samples', path_prefix='/api')
webapp.mapper.resource('request', 'requests', path_prefix='/api')
webapp.mapper.resource('form', 'forms', path_prefix='/api')
webapp.mapper.resource('request_type', 'request_types', path_prefix='/api')
webapp.mapper.resource('role', 'roles', path_prefix='/api')
webapp.mapper.resource('group', 'groups', path_prefix='/api')
webapp.mapper.resource_with_deleted('quota', 'quotas', path_prefix='/api')
webapp.mapper.resource('tool', 'tools', path_prefix='/api')
webapp.mapper.resource_with_deleted('user', 'users', path_prefix='/api')
webapp.mapper.resource('genome', 'genomes', path_prefix='/api')
webapp.mapper.resource('visualization',
'visualizations',
path_prefix='/api')
webapp.mapper.resource('workflow', 'workflows', path_prefix='/api')
webapp.mapper.resource_with_deleted('history',
'histories',
path_prefix='/api')
webapp.mapper.resource('configuration',
'configuration',
path_prefix='/api')
#webapp.mapper.connect( 'run_workflow', '/api/workflow/{workflow_id}/library/{library_id}', controller='workflows', action='run', workflow_id=None, library_id=None, conditions=dict(method=["GET"]) )
webapp.mapper.resource('search', 'search', path_prefix='/api')
# visualizations registry generic template renderer
webapp.add_route('/visualization/show/:visualization_name',
controller='visualization',
action='render',
visualization_name=None)
# "POST /api/workflows/import" => ``workflows.import_workflow()``.
# Defines a named route "import_workflow".
webapp.mapper.connect("import_workflow",
"/api/workflows/upload",
controller="workflows",
action="import_new_workflow",
conditions=dict(method=["POST"]))
webapp.mapper.connect("workflow_dict",
'/api/workflows/{workflow_id}/download',
controller='workflows',
action='workflow_dict',
conditions=dict(method=['GET']))
# Preserve the following download route for now for dependent applications -- deprecate at some point
webapp.mapper.connect("workflow_dict",
'/api/workflows/download/{workflow_id}',
controller='workflows',
action='workflow_dict',
conditions=dict(method=['GET']))
# Galaxy API for tool shed features.
webapp.mapper.resource('tool_shed_repository',
'tool_shed_repositories',
member={'repair_repository_revision': 'POST'},
controller='tool_shed_repositories',
name_prefix='tool_shed_repository_',
path_prefix='/api',
new={'install_repository_revision': 'POST'},
parent_resources=dict(
member_name='tool_shed_repository',
collection_name='tool_shed_repositories'))
# Connect logger from app
if app.trace_logger:
webapp.trace_logger = app.trace_logger
# Indicate that all configuration settings have been provided
webapp.finalize_config()
# Wrap the webapp in some useful middleware
if kwargs.get('middleware', True):
webapp = wrap_in_middleware(webapp, global_conf, **kwargs)
if asbool(kwargs.get('static_enabled', True)):
webapp = wrap_in_static(webapp, global_conf, **kwargs)
if asbool(kwargs.get('pack_scripts', False)):
pack_scripts()
# Close any pooled database connections before forking
try:
galaxy.model.mapping.metadata.engine.connection_provider._pool.dispose(
)
except:
pass
# Return
return webapp
def app_factory( global_conf, **kwargs ):
"""
Return a wsgi application serving the root object
"""
# Create the Galaxy application unless passed in
if 'app' in kwargs:
app = kwargs.pop( 'app' )
else:
try:
from galaxy.app import UniverseApplication
app = UniverseApplication( global_conf = global_conf, **kwargs )
except:
import traceback, sys
traceback.print_exc()
sys.exit( 1 )
atexit.register( app.shutdown )
# Create the universe WSGI application
webapp = galaxy.web.framework.WebApplication( app, session_cookie='galaxysession' )
add_ui_controllers( webapp, app )
# Force /history to go to /root/history -- needed since the tests assume this
webapp.add_route( '/history', controller='root', action='history' )
# These two routes handle our simple needs at the moment
webapp.add_route( '/async/:tool_id/:data_id/:data_secret', controller='async', action='index', tool_id=None, data_id=None, data_secret=None )
webapp.add_route( '/:controller/:action', action='index' )
webapp.add_route( '/:action', controller='root', action='index' )
# allow for subdirectories in extra_files_path
webapp.add_route( '/datasets/:dataset_id/display/{filename:.+?}', controller='dataset', action='display', dataset_id=None, filename=None)
webapp.add_route( '/datasets/:dataset_id/:action/:filename', controller='dataset', action='index', dataset_id=None, filename=None)
webapp.add_route( '/display_application/:dataset_id/:app_name/:link_name/:user_id/:app_action/:action_param', controller='dataset', action='display_application', dataset_id=None, user_id=None, app_name = None, link_name = None, app_action = None, action_param = None )
webapp.add_route( '/u/:username/d/:slug/:filename', controller='dataset', action='display_by_username_and_slug', filename=None )
webapp.add_route( '/u/:username/p/:slug', controller='page', action='display_by_username_and_slug' )
webapp.add_route( '/u/:username/h/:slug', controller='history', action='display_by_username_and_slug' )
webapp.add_route( '/u/:username/w/:slug', controller='workflow', action='display_by_username_and_slug' )
webapp.add_route( '/u/:username/v/:slug', controller='visualization', action='display_by_username_and_slug' )
# Add the web API
add_api_controllers( webapp, app )
webapp.api_mapper.resource( 'content',
'contents',
controller='library_contents',
name_prefix='library_',
path_prefix='/api/libraries/:library_id',
parent_resources=dict( member_name='library', collection_name='libraries' ) )
webapp.api_mapper.resource( 'content',
'contents',
controller='history_contents',
name_prefix='history_',
path_prefix='/api/histories/:history_id',
parent_resources=dict( member_name='history', collection_name='histories' ) )
webapp.api_mapper.resource( 'permission',
'permissions',
path_prefix='/api/libraries/:library_id',
parent_resources=dict( member_name='library', collection_name='libraries' ) )
webapp.api_mapper.resource( 'dataset', 'datasets', path_prefix='/api' )
webapp.api_mapper.resource_with_deleted( 'library', 'libraries', path_prefix='/api' )
webapp.api_mapper.resource( 'sample', 'samples', path_prefix='/api' )
webapp.api_mapper.resource( 'request', 'requests', path_prefix='/api' )
webapp.api_mapper.resource( 'form', 'forms', path_prefix='/api' )
webapp.api_mapper.resource( 'request_type', 'request_types', path_prefix='/api' )
webapp.api_mapper.resource( 'role', 'roles', path_prefix='/api' )
webapp.api_mapper.resource_with_deleted( 'quota', 'quotas', path_prefix='/api' )
webapp.api_mapper.resource( 'tool', 'tools', path_prefix='/api' )
webapp.api_mapper.resource_with_deleted( 'user', 'users', path_prefix='/api' )
webapp.api_mapper.resource( 'workflow', 'workflows', path_prefix='/api' )
webapp.api_mapper.resource_with_deleted( 'history', 'histories', path_prefix='/api' )
#webapp.api_mapper.connect( 'run_workflow', '/api/workflow/{workflow_id}/library/{library_id}', controller='workflows', action='run', workflow_id=None, library_id=None, conditions=dict(method=["GET"]) )
webapp.finalize_config()
# Wrap the webapp in some useful middleware
if kwargs.get( 'middleware', True ):
webapp = wrap_in_middleware( webapp, global_conf, **kwargs )
if asbool( kwargs.get( 'static_enabled', True ) ):
webapp = wrap_in_static( webapp, global_conf, **kwargs )
if asbool(kwargs.get('pack_scripts', False)):
pack_scripts()
# Close any pooled database connections before forking
try:
galaxy.model.mapping.metadata.engine.connection_provider._pool.dispose()
except:
pass
# Return
return webapp
def main():
# ---- Configuration ------------------------------------------------------
galaxy_test_host = os.environ.get('GALAXY_TEST_HOST',
default_galaxy_test_host)
galaxy_test_port = os.environ.get('GALAXY_TEST_PORT', None)
galaxy_test_save = os.environ.get('GALAXY_TEST_SAVE', None)
tool_path = os.environ.get('GALAXY_TEST_TOOL_PATH', 'tools')
if 'HTTP_ACCEPT_LANGUAGE' not in os.environ:
os.environ['HTTP_ACCEPT_LANGUAGE'] = default_galaxy_locales
testing_migrated_tools = '-migrated' in sys.argv
testing_installed_tools = '-installed' in sys.argv
if testing_migrated_tools or testing_installed_tools:
sys.argv.pop()
# Store a jsonified dictionary of tool_id : GALAXY_TEST_FILE_DIR pairs.
galaxy_tool_shed_test_file = 'shed_tools_dict'
# We need the upload tool for functional tests, so we'll create a temporary tool panel config that defines it.
fd, tmp_tool_panel_conf = tempfile.mkstemp()
os.write(fd, '<?xml version="1.0"?>\n')
os.write(fd, '<toolbox>\n')
os.write(fd, '<tool file="data_source/upload.xml"/>\n')
os.write(fd, '</toolbox>\n')
os.close(fd)
tool_config_file = tmp_tool_panel_conf
galaxy_test_file_dir = None
library_import_dir = None
user_library_import_dir = None
# Exclude all files except test_toolbox.py.
ignore_files = (re.compile(r'^test_[adghlmsu]*'),
re.compile(r'^test_ta*'))
else:
tool_config_file = os.environ.get('GALAXY_TEST_TOOL_CONF',
'tool_conf.xml.sample')
galaxy_test_file_dir = os.environ.get('GALAXY_TEST_FILE_DIR',
default_galaxy_test_file_dir)
if not os.path.isabs(galaxy_test_file_dir):
galaxy_test_file_dir = os.path.join(os.getcwd(),
galaxy_test_file_dir)
library_import_dir = galaxy_test_file_dir
user_library_import_dir = os.path.join(galaxy_test_file_dir, 'users')
ignore_files = ()
start_server = 'GALAXY_TEST_EXTERNAL' not in os.environ
if os.path.exists('tool_data_table_conf.test.xml'):
tool_data_table_config_path = 'tool_data_table_conf.test.xml'
else:
tool_data_table_config_path = 'tool_data_table_conf.xml'
tool_dependency_dir = os.environ.get('GALAXY_TOOL_DEPENDENCY_DIR', None)
use_distributed_object_store = os.environ.get(
'GALAXY_USE_DISTRIBUTED_OBJECT_STORE', False)
if start_server:
psu_production = False
galaxy_test_proxy_port = None
if 'GALAXY_TEST_PSU_PRODUCTION' in os.environ:
if not galaxy_test_port:
raise Exception(
'Please set GALAXY_TEST_PORT to the port to which the proxy server will proxy'
)
galaxy_test_proxy_port = os.environ.get('GALAXY_TEST_PROXY_PORT',
None)
if not galaxy_test_proxy_port:
raise Exception(
'Please set GALAXY_TEST_PROXY_PORT to the port on which the proxy server is listening'
)
base_file_path = os.environ.get('GALAXY_TEST_BASE_FILE_PATH', None)
if not base_file_path:
raise Exception(
'Please set GALAXY_TEST_BASE_FILE_PATH to the directory which will contain the dataset files directory'
)
base_new_file_path = os.environ.get(
'GALAXY_TEST_BASE_NEW_FILE_PATH', None)
if not base_new_file_path:
raise Exception(
'Please set GALAXY_TEST_BASE_NEW_FILE_PATH to the directory which will contain the temporary directory'
)
database_connection = os.environ.get('GALAXY_TEST_DBURI', None)
if not database_connection:
raise Exception(
'Please set GALAXY_TEST_DBURI to the URI of the database to be used for tests'
)
nginx_upload_store = os.environ.get(
'GALAXY_TEST_NGINX_UPLOAD_STORE', None)
if not nginx_upload_store:
raise Exception(
'Please set GALAXY_TEST_NGINX_UPLOAD_STORE to the path where the nginx upload module places uploaded files'
)
tool_config_file = 'tool_conf.xml.main'
default_cluster_job_runner = os.environ.get(
'GALAXY_TEST_DEFAULT_CLUSTER_JOB_RUNNER', 'pbs:///')
file_path = tempfile.mkdtemp(dir=base_file_path)
new_file_path = tempfile.mkdtemp(dir=base_new_file_path)
cluster_files_directory = os.path.join(new_file_path, 'pbs')
job_working_directory = os.path.join(new_file_path,
'job_working_directory')
os.mkdir(cluster_files_directory)
os.mkdir(job_working_directory)
kwargs = dict(
database_engine_option_pool_size='10',
database_engine_option_max_overflow='20',
database_engine_option_strategy='threadlocal',
nginx_x_accel_redirect_base='/_x_accel_redirect',
nginx_upload_store=nginx_upload_store,
nginx_upload_path='/_upload',
allow_library_path_paste='True',
cluster_files_directory=cluster_files_directory,
job_working_directory=job_working_directory,
outputs_to_working_directory='True',
set_metadata_externally='True',
static_enabled='False',
debug='False',
track_jobs_in_database='True',
job_scheduler_policy='FIFO',
start_job_runners='pbs',
default_cluster_job_runner=default_cluster_job_runner)
psu_production = True
else:
if 'GALAXY_TEST_DBPATH' in os.environ:
db_path = os.environ['GALAXY_TEST_DBPATH']
else:
tempdir = tempfile.mkdtemp()
db_path = os.path.join(tempdir, 'database')
file_path = os.path.join(db_path, 'files')
new_file_path = os.path.join(db_path, 'tmp')
if 'GALAXY_TEST_DBURI' in os.environ:
database_connection = os.environ['GALAXY_TEST_DBURI']
else:
database_connection = 'sqlite:///' + os.path.join(
db_path, 'universe.sqlite')
kwargs = {}
for dir in file_path, new_file_path:
try:
os.makedirs(dir)
except OSError:
pass
print "Database connection:", database_connection
# ---- Build Application --------------------------------------------------
app = None
if start_server:
global_conf = {'__file__': 'universe_wsgi.ini.sample'}
if psu_production:
global_conf = None
if not database_connection.startswith('sqlite://'):
kwargs['database_engine_option_max_overflow'] = '20'
if tool_dependency_dir is not None:
kwargs['tool_dependency_dir'] = tool_dependency_dir
if use_distributed_object_store:
kwargs['object_store'] = 'distributed'
kwargs[
'distributed_object_store_config_file'] = 'distributed_object_store_conf.xml.sample'
# Build the Universe Application
app = UniverseApplication(
job_queue_workers=5,
id_secret='changethisinproductiontoo',
template_path="templates",
database_connection=database_connection,
database_engine_option_pool_size='10',
file_path=file_path,
new_file_path=new_file_path,
tool_path=tool_path,
update_integrated_tool_panel=False,
tool_config_file=tool_config_file,
datatype_converters_config_file=
"datatype_converters_conf.xml.sample",
tool_parse_help=False,
test_conf="test.conf",
tool_data_table_config_path=tool_data_table_config_path,
log_destination="stdout",
use_heartbeat=False,
allow_user_creation=True,
allow_user_deletion=True,
admin_users='*****@*****.**',
allow_library_path_paste=True,
library_import_dir=library_import_dir,
user_library_import_dir=user_library_import_dir,
global_conf=global_conf,
running_functional_tests=True,
**kwargs)
log.info("Embedded Universe application started")
# ---- Run webserver ------------------------------------------------------
server = None
if start_server:
webapp = buildapp.app_factory(dict(),
use_translogger=False,
static_enabled=False,
app=app)
if galaxy_test_port is not None:
server = httpserver.serve(webapp,
host=galaxy_test_host,
port=galaxy_test_port,
start_loop=False)
else:
random.seed()
for i in range(0, 9):
try:
galaxy_test_port = str(
random.randint(default_galaxy_test_port_min,
default_galaxy_test_port_max))
log.debug(
"Attempting to serve app on randomly chosen port: %s" %
galaxy_test_port)
server = httpserver.serve(webapp,
host=galaxy_test_host,
port=galaxy_test_port,
start_loop=False)
break
except socket.error, e:
if e[0] == 98:
continue
raise
else:
raise Exception(
"Unable to open a port between %s and %s to start Galaxy server"
% (default_galaxy_test_port_min,
default_galaxy_test_port_max))
def setup():
"""Start the web server for the tests"""
global server, app
galaxy_test_host = os.environ.get('GALAXY_TEST_HOST',
default_galaxy_test_host)
galaxy_test_port = os.environ.get('GALAXY_TEST_PORT',
default_galaxy_test_port)
start_server = 'GALAXY_TEST_EXTERNAL' not in os.environ
if start_server:
tempdir = tempfile.mkdtemp()
file_path = os.path.join(tempdir, 'database', 'files')
os.makedirs(file_path)
if 'GALAXY_TEST_DBURI' in os.environ:
database_connection = os.environ['GALAXY_TEST_DBURI']
else:
database_connection = 'sqlite:///' + os.path.join(
tempdir, 'database', 'universe.sqlite')
app = UniverseApplication(job_queue_workers=5,
template_path="templates",
database_connection=database_connection,
file_path=file_path,
tool_config_file="tool_conf.xml",
tool_path="tools",
test_conf="test.conf",
log_destination="stdout",
use_heartbeat=True)
log.info("Embedded Universe application started")
webapp = universe_wsgi.app_factory(dict(),
use_translogger=False,
app=app)
server = galaxy.web.server.serve(webapp,
dict(),
host=galaxy_test_host,
port=galaxy_test_port,
start_loop=False)
atexit.register(teardown)
import threading
t = threading.Thread(target=server.serve_forever)
t.start()
time.sleep(2)
log.info("Embedded web server started")
if app:
# TODO: provisions for loading toolbox from file when using external server
import test_toolbox
test_toolbox.toolbox = app.toolbox
else:
from galaxy import tools
import test_toolbox
test_toolbox.toolbox = tools.ToolBox('tool_conf.xml', 'tools')
# Test if the server is up
import httplib
conn = httplib.HTTPConnection(galaxy_test_host, galaxy_test_port)
conn.request("GET", "/")
assert conn.getresponse(
).status == 200, "Test HTTP server did not return '200 OK'"
os.environ['GALAXY_TEST_HOST'] = galaxy_test_host
os.environ['GALAXY_TEST_PORT'] = galaxy_test_port
os.environ['GALAXY_TEST_FILE_DIR'] = galaxy_test_file_dir
