def install_repository(self, repository_elem, tool_shed_repository, install_dependencies, is_repository_dependency=False):
"""Install a single repository, loading contained tools into the tool panel."""
# Install path is of the form: <tool path>/<tool shed>/repos/<repository owner>/<repository name>/<installed changeset revision>
relative_clone_dir = os.path.join(tool_shed_repository.tool_shed,
'repos',
tool_shed_repository.owner,
tool_shed_repository.name,
tool_shed_repository.installed_changeset_revision)
clone_dir = os.path.join(self.tool_path, relative_clone_dir)
cloned_ok = self.__iscloned(clone_dir)
is_installed = False
# Any of the following states should count as installed in this context.
if tool_shed_repository.status in [self.app.install_model.ToolShedRepository.installation_status.INSTALLED,
self.app.install_model.ToolShedRepository.installation_status.ERROR,
self.app.install_model.ToolShedRepository.installation_status.UNINSTALLED,
self.app.install_model.ToolShedRepository.installation_status.DEACTIVATED]:
is_installed = True
if cloned_ok and is_installed:
log.info("Skipping automatic install of repository '%s' because it has already been installed in location %s",
tool_shed_repository.name, clone_dir)
else:
irm = install_manager.InstallRepositoryManager(self.app, self.tpm)
repository_clone_url = os.path.join(self.tool_shed_url, 'repos', tool_shed_repository.owner, tool_shed_repository.name)
relative_install_dir = os.path.join(relative_clone_dir, tool_shed_repository.name)
install_dir = os.path.join(clone_dir, tool_shed_repository.name)
ctx_rev = suc.get_ctx_rev(self.app,
self.tool_shed_url,
tool_shed_repository.name,
tool_shed_repository.owner,
tool_shed_repository.installed_changeset_revision)
if not cloned_ok:
irm.update_tool_shed_repository_status(tool_shed_repository,
self.app.install_model.ToolShedRepository.installation_status.CLONING)
cloned_ok, error_message = hg_util.clone_repository(repository_clone_url, os.path.abspath(install_dir), ctx_rev)
if cloned_ok and not is_installed:
self.handle_repository_contents(tool_shed_repository=tool_shed_repository,
repository_clone_url=repository_clone_url,
relative_install_dir=relative_install_dir,
repository_elem=repository_elem,
install_dependencies=install_dependencies,
is_repository_dependency=is_repository_dependency)
self.app.install_model.context.refresh(tool_shed_repository)
irm.update_tool_shed_repository_status(tool_shed_repository,
self.app.install_model.ToolShedRepository.installation_status.INSTALLED)
else:
log.error('Error attempting to clone repository %s: %s', str(tool_shed_repository.name), str(error_message))
irm.update_tool_shed_repository_status(tool_shed_repository,
self.app.install_model.ToolShedRepository.installation_status.ERROR,
error_message=error_message)
def install_repository( self, repository_elem, tool_shed_repository, install_dependencies, is_repository_dependency=False ):
"""Install a single repository, loading contained tools into the tool panel."""
# Install path is of the form: <tool path>/<tool shed>/repos/<repository owner>/<repository name>/<installed changeset revision>
relative_clone_dir = os.path.join( tool_shed_repository.tool_shed,
'repos',
tool_shed_repository.owner,
tool_shed_repository.name,
tool_shed_repository.installed_changeset_revision )
clone_dir = os.path.join( self.tool_path, relative_clone_dir )
cloned_ok = self.__iscloned( clone_dir )
is_installed = False
# Any of the following states should count as installed in this context.
if tool_shed_repository.status in [ self.app.install_model.ToolShedRepository.installation_status.INSTALLED,
self.app.install_model.ToolShedRepository.installation_status.ERROR,
self.app.install_model.ToolShedRepository.installation_status.UNINSTALLED,
self.app.install_model.ToolShedRepository.installation_status.DEACTIVATED ]:
is_installed = True
if cloned_ok and is_installed:
print "Skipping automatic install of repository '", tool_shed_repository.name, "' because it has already been installed in location ", clone_dir
else:
irm = install_manager.InstallRepositoryManager( self.app, self.tpm )
repository_clone_url = os.path.join( self.tool_shed_url, 'repos', tool_shed_repository.owner, tool_shed_repository.name )
relative_install_dir = os.path.join( relative_clone_dir, tool_shed_repository.name )
install_dir = os.path.join( clone_dir, tool_shed_repository.name )
ctx_rev = suc.get_ctx_rev( self.app,
self.tool_shed_url,
tool_shed_repository.name,
tool_shed_repository.owner,
tool_shed_repository.installed_changeset_revision )
if not cloned_ok:
irm.update_tool_shed_repository_status( tool_shed_repository,
self.app.install_model.ToolShedRepository.installation_status.CLONING )
cloned_ok, error_message = hg_util.clone_repository( repository_clone_url, os.path.abspath( install_dir ), ctx_rev )
if cloned_ok and not is_installed:
self.handle_repository_contents( tool_shed_repository=tool_shed_repository,
repository_clone_url=repository_clone_url,
relative_install_dir=relative_install_dir,
repository_elem=repository_elem,
install_dependencies=install_dependencies,
is_repository_dependency=is_repository_dependency )
self.app.install_model.context.refresh( tool_shed_repository )
metadata_dict = tool_shed_repository.metadata
if 'tools' in metadata_dict:
# Initialize the ToolVersionManager.
tvm = tool_version_manager.ToolVersionManager( self.app )
irm.update_tool_shed_repository_status( tool_shed_repository,
self.app.install_model.ToolShedRepository.installation_status.SETTING_TOOL_VERSIONS )
# Get the tool_versions from the tool shed for each tool in the installed change set.
url = '%s/repository/get_tool_versions?name=%s&owner=%s&changeset_revision=%s' % \
( self.tool_shed_url, tool_shed_repository.name, self.repository_owner, tool_shed_repository.installed_changeset_revision )
text = common_util.tool_shed_get( self.app, self.tool_shed_url, url )
if text:
tool_version_dicts = json.loads( text )
tvm.handle_tool_versions( tool_version_dicts, tool_shed_repository )
else:
# Set the tool versions since they seem to be missing for this repository in the tool shed.
# CRITICAL NOTE: These default settings may not properly handle all parent/child associations.
for tool_dict in metadata_dict[ 'tools' ]:
flush_needed = False
tool_id = tool_dict[ 'guid' ]
old_tool_id = tool_dict[ 'id' ]
tool_version = tool_dict[ 'version' ]
tool_version_using_old_id = tvm.get_tool_version( old_tool_id )
tool_version_using_guid = tvm.get_tool_version( tool_id )
if not tool_version_using_old_id:
tool_version_using_old_id = self.app.install_model.ToolVersion( tool_id=old_tool_id,
tool_shed_repository=tool_shed_repository )
self.app.install_model.context.add( tool_version_using_old_id )
self.app.install_model.context.flush()
if not tool_version_using_guid:
tool_version_using_guid = self.app.install_model.ToolVersion( tool_id=tool_id,
tool_shed_repository=tool_shed_repository )
self.app.install_model.context.add( tool_version_using_guid )
self.app.install_model.context.flush()
# Associate the two versions as parent / child.
tool_version_association = tvm.get_tool_version_association( tool_version_using_old_id,
tool_version_using_guid )
if not tool_version_association:
tool_version_association = \
self.app.install_model.ToolVersionAssociation( tool_id=tool_version_using_guid.id,
parent_id=tool_version_using_old_id.id )
self.app.install_model.context.add( tool_version_association )
self.app.install_model.context.flush()
irm.update_tool_shed_repository_status( tool_shed_repository,
self.app.install_model.ToolShedRepository.installation_status.INSTALLED )
else:
print 'Error attempting to clone repository %s: %s' % ( str( tool_shed_repository.name ), str( error_message ) )
irm.update_tool_shed_repository_status( tool_shed_repository,
self.app.install_model.ToolShedRepository.installation_status.ERROR,
error_message=error_message )
def check_and_update_repository_metadata( app, info_only=False, verbosity=1 ):
"""
This method will iterate through all records in the repository_metadata
table, checking each one for tool metadata, then checking the tool
metadata for tests. Each tool's metadata should look something like:
{
"add_to_tool_panel": true,
"description": "",
"guid": "toolshed.url:9009/repos/owner/name/tool_id/1.2.3",
"id": "tool_wrapper",
"name": "Map with Tool Wrapper",
"requirements": [],
"tests": [],
"tool_config": "database/community_files/000/repo_1/tool_wrapper.xml",
"tool_type": "default",
"version": "1.2.3",
"version_string_cmd": null
}
If the "tests" attribute is missing or empty, this script will mark the metadata record (which is specific to a changeset revision of a repository)
not to be tested. If each "tools" attribute has at least one valid "tests" entry, this script will do nothing, and leave it available for the install
and test repositories script to process. If the tested changeset revision does not have a test-data directory, this script will also mark the revision
not to be tested.
"""
start = time.time()
skip_metadata_ids = []
checked_repository_ids = []
tool_count = 0
has_tests = 0
no_tests = 0
valid_revisions = 0
invalid_revisions = 0
records_checked = 0
# Do not check metadata records that have an entry in the skip_tool_tests table, since they won't be tested anyway.
print '# -------------------------------------------------------------------------------------------'
print '# The skip_tool_test setting has been set for the following repository revision, so they will not be tested.'
skip_metadata_ids = []
for skip_tool_test in app.sa_session.query( app.model.SkipToolTest ):
print '# repository_metadata_id: %s, changeset_revision: %s' % \
( str( skip_tool_test.repository_metadata_id ), str( skip_tool_test.initial_changeset_revision ) )
print 'reason: %s' % str( skip_tool_test.comment )
skip_metadata_ids.append( skip_tool_test.repository_metadata_id )
# Get the list of metadata records to check for functional tests and test data. Limit this to records that have not been flagged do_not_test,
# since there's no need to check them again if they won't be tested anyway. Also filter out changeset revisions that are not downloadable,
# because it's redundant to test a revision that a user can't install.
for repository_metadata in app.sa_session.query( app.model.RepositoryMetadata ) \
.filter( and_( app.model.RepositoryMetadata.table.c.downloadable == True,
app.model.RepositoryMetadata.table.c.includes_tools == True,
app.model.RepositoryMetadata.table.c.do_not_test == False ) ):
# Initialize some items.
missing_test_components = []
revision_has_test_data = False
testable_revision = False
repository = repository_metadata.repository
records_checked += 1
# Check the next repository revision.
changeset_revision = str( repository_metadata.changeset_revision )
name = repository.name
owner = repository.user.username
metadata = repository_metadata.metadata
repository = repository_metadata.repository
if repository.id not in checked_repository_ids:
checked_repository_ids.append( repository.id )
print '# -------------------------------------------------------------------------------------------'
print '# Checking revision %s of %s owned by %s.' % ( changeset_revision, name, owner )
if repository_metadata.id in skip_metadata_ids:
print'# Skipping revision %s of %s owned by %s because the skip_tool_test setting has been set.' % ( changeset_revision, name, owner )
continue
# If this changeset revision has no tools, we don't need to do anything here, the install and test script has a filter for returning
# only repositories that contain tools.
tool_dicts = metadata.get( 'tools', None )
if tool_dicts is not None:
# Clone the repository up to the changeset revision we're checking.
repo_dir = repository.repo_path( app )
hg_util.get_repo_for_repository( app, repository=None, repo_path=repo_dir, create=False )
work_dir = tempfile.mkdtemp( prefix="tmp-toolshed-cafr" )
cloned_ok, error_message = hg_util.clone_repository( repo_dir, work_dir, changeset_revision )
if cloned_ok:
# Iterate through all the directories in the cloned changeset revision and determine whether there's a
# directory named test-data. If this directory is not present update the metadata record for the changeset
# revision we're checking.
for root, dirs, files in os.walk( work_dir ):
if '.hg' in dirs:
dirs.remove( '.hg' )
if 'test-data' in dirs:
revision_has_test_data = True
test_data_path = os.path.join( root, dirs[ dirs.index( 'test-data' ) ] )
break
if revision_has_test_data:
print '# Test data directory found in changeset revision %s of repository %s owned by %s.' % ( changeset_revision, name, owner )
else:
print '# Test data directory missing in changeset revision %s of repository %s owned by %s.' % ( changeset_revision, name, owner )
print '# Checking for functional tests in changeset revision %s of %s, owned by %s.' % \
( changeset_revision, name, owner )
# Inspect each tool_dict for defined functional tests. If there
# are no tests, this tool should not be tested, since the tool
# functional tests only report failure if the test itself fails,
# not if it's missing or undefined. Filtering out those
# repositories at this step will reduce the number of "false
# negatives" the automated functional test framework produces.
for tool_dict in tool_dicts:
failure_reason = ''
problem_found = False
tool_has_defined_tests = False
tool_has_test_files = False
missing_test_files = []
tool_count += 1
tool_id = tool_dict[ 'id' ]
tool_version = tool_dict[ 'version' ]
tool_guid = tool_dict[ 'guid' ]
if verbosity >= 1:
print "# Checking tool ID '%s' in changeset revision %s of %s." % ( tool_id, changeset_revision, name )
defined_test_dicts = tool_dict.get( 'tests', None )
if defined_test_dicts is not None:
# We need to inspect the <test> tags because the following tags...
# <tests>
# </tests>
# ...will produce the following metadata:
# "tests": []
# And the following tags...
# <tests>
# <test>
# </test>
# </tests>
# ...will produce the following metadata:
# "tests":
# [{"inputs": [], "name": "Test-1", "outputs": [], "required_files": []}]
for defined_test_dict in defined_test_dicts:
inputs = defined_test_dict.get( 'inputs', [] )
outputs = defined_test_dict.get( 'outputs', [] )
if inputs and outputs:
# At least one tool within the repository has a valid <test> tag.
tool_has_defined_tests = True
break
if tool_has_defined_tests:
print "# Tool ID '%s' in changeset revision %s of %s has one or more valid functional tests defined." % \
( tool_id, changeset_revision, name )
has_tests += 1
else:
print '# No functional tests defined for %s.' % tool_id
no_tests += 1
if tool_has_defined_tests and revision_has_test_data:
missing_test_files = check_for_missing_test_files( defined_test_dicts, test_data_path )
if missing_test_files:
print "# Tool id '%s' in changeset revision %s of %s is missing one or more required test files: %s" % \
( tool_id, changeset_revision, name, ', '.join( missing_test_files ) )
else:
tool_has_test_files = True
if not revision_has_test_data:
failure_reason += 'Repository does not have a test-data directory. '
problem_found = True
if not tool_has_defined_tests:
failure_reason += 'Functional test definitions missing for %s. ' % tool_id
problem_found = True
if missing_test_files:
failure_reason += 'One or more test files are missing for tool %s: %s' % ( tool_id, ', '.join( missing_test_files ) )
problem_found = True
test_errors = dict( tool_id=tool_id, tool_version=tool_version, tool_guid=tool_guid, missing_components=failure_reason )
# Only append this error dict if it hasn't already been added.
if problem_found:
if test_errors not in missing_test_components:
missing_test_components.append( test_errors )
if tool_has_defined_tests and tool_has_test_files:
print '# Revision %s of %s owned by %s is a testable revision.' % ( changeset_revision, name, owner )
testable_revision = True
# Remove the cloned repository path. This has to be done after the check for required test files, for obvious reasons.
if os.path.exists( work_dir ):
shutil.rmtree( work_dir )
if not missing_test_components:
valid_revisions += 1
print '# All tools have functional tests in changeset revision %s of repository %s owned by %s.' % ( changeset_revision, name, owner )
else:
invalid_revisions += 1
print '# Some tools have problematic functional tests in changeset revision %s of repository %s owned by %s.' % ( changeset_revision, name, owner )
if verbosity >= 1:
for missing_test_component in missing_test_components:
if 'missing_components' in missing_test_component:
print '# %s' % missing_test_component[ 'missing_components' ]
if not info_only:
# Get or create the list of tool_test_results dictionaries.
if repository_metadata.tool_test_results is not None:
# We'll listify the column value in case it uses the old approach of storing the results of only a single test run.
tool_test_results_dicts = listify( repository_metadata.tool_test_results )
else:
tool_test_results_dicts = []
if tool_test_results_dicts:
# Inspect the tool_test_results_dict for the last test run in case it contains only a test_environment
# entry. This will occur with multiple runs of this script without running the associated
# install_and_test_tool_sed_repositories.sh script which will further populate the tool_test_results_dict.
tool_test_results_dict = tool_test_results_dicts[ 0 ]
if len( tool_test_results_dict ) <= 1:
# We can re-use the mostly empty tool_test_results_dict for this run because it is either empty or it contains only
# a test_environment entry. If we use it we need to temporarily eliminate it from the list of tool_test_results_dicts
# since it will be re-inserted later.
tool_test_results_dict = tool_test_results_dicts.pop( 0 )
elif (len( tool_test_results_dict ) == 2 and
'test_environment' in tool_test_results_dict and 'missing_test_components' in tool_test_results_dict):
# We can re-use tool_test_results_dict if its only entries are "test_environment" and "missing_test_components".
# In this case, some tools are missing tests components while others are not.
tool_test_results_dict = tool_test_results_dicts.pop( 0 )
else:
# The latest tool_test_results_dict has been populated with the results of a test run, so it cannot be used.
tool_test_results_dict = {}
else:
# Create a new dictionary for the most recent test run.
tool_test_results_dict = {}
test_environment_dict = tool_test_results_dict.get( 'test_environment', {} )
# Add the current time as the approximate time that this test run occurs. A similar value will also be
# set to the repository_metadata.time_last_tested column, but we also store it here because the Tool Shed
# may be configured to store multiple test run results, so each must be associated with a time stamp.
now = time.strftime( "%Y-%m-%d %H:%M:%S" )
test_environment_dict[ 'time_tested' ] = now
test_environment_dict[ 'tool_shed_database_version' ] = get_database_version( app )
test_environment_dict[ 'tool_shed_mercurial_version' ] = __version__.version
test_environment_dict[ 'tool_shed_revision' ] = get_repository_current_revision( os.getcwd() )
tool_test_results_dict[ 'test_environment' ] = test_environment_dict
# The repository_metadata.time_last_tested column is not changed by this script since no testing is performed here.
if missing_test_components:
# If functional test definitions or test data are missing, set do_not_test = True if no tool with valid tests has been
# found in this revision, and:
# a) There are multiple downloadable revisions, and the revision being tested is not the most recent downloadable revision.
# In this case, the revision will never be updated with the missing components, and re-testing it would be redundant.
# b) There are one or more downloadable revisions, and the provided changeset revision is the most recent downloadable
# revision. In this case, if the repository is updated with test data or functional tests, the downloadable
# changeset revision that was tested will either be replaced with the new changeset revision, or a new downloadable
# changeset revision will be created, either of which will be automatically checked and flagged as appropriate.
# In the install and test script, this behavior is slightly different, since we do want to always run functional
# tests on the most recent downloadable changeset revision.
if should_set_do_not_test_flag( app, repository, changeset_revision, testable_revision ):
print "# Setting do_not_test to True on revision %s of %s owned by %s because it is missing test components" % (
changeset_revision, name, owner
)
print "# and it is not the latest downloadable revision."
repository_metadata.do_not_test = True
if not testable_revision:
# Even though some tools may be missing test components, it may be possible to test other tools. Since the
# install and test framework filters out repositories marked as missing test components, we'll set it only if
# no tools can be tested.
print '# Setting missing_test_components to True for revision %s of %s owned by %s because all tools are missing test components.' % (
changeset_revision, name, owner
)
repository_metadata.missing_test_components = True
print "# Setting tools_functionally_correct to False on revision %s of %s owned by %s because it is missing test components" % (
changeset_revision, name, owner
)
repository_metadata.tools_functionally_correct = False
tool_test_results_dict[ 'missing_test_components' ] = missing_test_components
# Store only the configured number of test runs.
num_tool_test_results_saved = int( app.config.num_tool_test_results_saved )
if len( tool_test_results_dicts ) >= num_tool_test_results_saved:
test_results_index = num_tool_test_results_saved - 1
new_tool_test_results_dicts = tool_test_results_dicts[ :test_results_index ]
else:
new_tool_test_results_dicts = [ d for d in tool_test_results_dicts ]
# Insert the new element into the first position in the list.
new_tool_test_results_dicts.insert( 0, tool_test_results_dict )
repository_metadata.tool_test_results = new_tool_test_results_dicts
app.sa_session.add( repository_metadata )
app.sa_session.flush()
stop = time.time()
print '# -------------------------------------------------------------------------------------------'
print '# Checked %d repositories with %d tools in %d changeset revisions.' % ( len( checked_repository_ids ), tool_count, records_checked )
print '# %d revisions found with functional tests and test data for all tools.' % valid_revisions
print '# %d revisions found with one or more tools missing functional tests and/or test data.' % invalid_revisions
print '# Found %d tools without functional tests.' % no_tests
print '# Found %d tools with functional tests.' % has_tests
if info_only:
print '# Database not updated, info_only set.'
print "# Elapsed time: ", stop - start
print "#############################################################################"
def upload(self, trans, **kwd):
message = escape(kwd.get('message', ''))
status = kwd.get('status', 'done')
commit_message = escape(kwd.get('commit_message', 'Uploaded'))
repository_id = kwd.get('repository_id', '')
repository = repository_util.get_repository_in_tool_shed(
trans.app, repository_id)
repo_dir = repository.repo_path(trans.app)
uncompress_file = util.string_as_bool(
kwd.get('uncompress_file', 'true'))
remove_repo_files_not_in_tar = util.string_as_bool(
kwd.get('remove_repo_files_not_in_tar', 'true'))
uploaded_file = None
upload_point = commit_util.get_upload_point(repository, **kwd)
tip = repository.tip()
file_data = kwd.get('file_data', '')
url = kwd.get('url', '')
# Part of the upload process is sending email notification to those that have registered to
# receive them. One scenario occurs when the first change set is produced for the repository.
# See the suc.handle_email_alerts() method for the definition of the scenarios.
new_repo_alert = repository.is_new()
uploaded_directory = None
if kwd.get('upload_button', False):
if file_data == '' and url == '':
message = 'No files were entered on the upload form.'
status = 'error'
uploaded_file = None
elif url and url.startswith('hg'):
# Use mercurial clone to fetch repository, contents will then be copied over.
uploaded_directory = tempfile.mkdtemp()
repo_url = 'http%s' % url[len('hg'):]
cloned_ok, error_message = hg_util.clone_repository(
repo_url, uploaded_directory)
if not cloned_ok:
message = 'Error uploading via mercurial clone: %s' % error_message
status = 'error'
basic_util.remove_dir(uploaded_directory)
uploaded_directory = None
elif url:
valid_url = True
try:
stream = requests.get(url, stream=True)
except Exception as e:
valid_url = False
message = 'Error uploading file via http: %s' % util.unicodify(
e)
status = 'error'
uploaded_file = None
if valid_url:
fd, uploaded_file_name = tempfile.mkstemp()
uploaded_file = open(uploaded_file_name, 'wb')
for chunk in stream.iter_content(
chunk_size=util.CHUNK_SIZE):
if chunk:
uploaded_file.write(chunk)
uploaded_file.flush()
uploaded_file_filename = url.split('/')[-1]
isempty = os.path.getsize(
os.path.abspath(uploaded_file_name)) == 0
elif file_data not in ('', None):
uploaded_file = file_data.file
uploaded_file_name = uploaded_file.name
uploaded_file_filename = os.path.split(file_data.filename)[-1]
isempty = os.path.getsize(
os.path.abspath(uploaded_file_name)) == 0
if uploaded_file or uploaded_directory:
rdah = attribute_handlers.RepositoryDependencyAttributeHandler(
trans.app, unpopulate=False)
tdah = attribute_handlers.ToolDependencyAttributeHandler(
trans.app, unpopulate=False)
stdtm = ShedToolDataTableManager(trans.app)
ok = True
isgzip = False
isbz2 = False
if uploaded_file:
if uncompress_file:
isgzip = checkers.is_gzip(uploaded_file_name)
if not isgzip:
isbz2 = checkers.is_bz2(uploaded_file_name)
if isempty:
tar = None
istar = False
else:
# Determine what we have - a single file or an archive
try:
if (isgzip or isbz2) and uncompress_file:
# Open for reading with transparent compression.
tar = tarfile.open(uploaded_file_name, 'r:*')
else:
tar = tarfile.open(uploaded_file_name)
istar = True
except tarfile.ReadError:
tar = None
istar = False
else:
# Uploaded directory
istar = False
if istar:
ok, message, files_to_remove, content_alert_str, undesirable_dirs_removed, undesirable_files_removed = \
repository_content_util.upload_tar(
trans,
rdah,
tdah,
repository,
tar,
uploaded_file,
upload_point,
remove_repo_files_not_in_tar,
commit_message,
new_repo_alert
)
elif uploaded_directory:
ok, message, files_to_remove, content_alert_str, undesirable_dirs_removed, undesirable_files_removed = \
self.upload_directory(trans,
rdah,
tdah,
repository,
uploaded_directory,
upload_point,
remove_repo_files_not_in_tar,
commit_message,
new_repo_alert)
else:
if (isgzip or isbz2) and uncompress_file:
uploaded_file_filename = commit_util.uncompress(
repository,
uploaded_file_name,
uploaded_file_filename,
isgzip=isgzip,
isbz2=isbz2)
if repository.type == rt_util.REPOSITORY_SUITE_DEFINITION and \
uploaded_file_filename != rt_util.REPOSITORY_DEPENDENCY_DEFINITION_FILENAME:
ok = False
message = 'Repositories of type <b>Repository suite definition</b> can only contain a single file named '
message += '<b>repository_dependencies.xml</b>.'
elif repository.type == rt_util.TOOL_DEPENDENCY_DEFINITION and \
uploaded_file_filename != rt_util.TOOL_DEPENDENCY_DEFINITION_FILENAME:
ok = False
message = 'Repositories of type <b>Tool dependency definition</b> can only contain a single file named '
message += '<b>tool_dependencies.xml</b>.'
if ok:
if upload_point is not None:
full_path = os.path.abspath(
os.path.join(repo_dir, upload_point,
uploaded_file_filename))
else:
full_path = os.path.abspath(
os.path.join(repo_dir, uploaded_file_filename))
# Move some version of the uploaded file to the load_point within the repository hierarchy.
if uploaded_file_filename in [
rt_util.
REPOSITORY_DEPENDENCY_DEFINITION_FILENAME
]:
# Inspect the contents of the file to see if toolshed or changeset_revision attributes
# are missing and if so, set them appropriately.
altered, root_elem, error_message = rdah.handle_tag_attributes(
uploaded_file_name)
if error_message:
ok = False
message = error_message
status = 'error'
elif altered:
tmp_filename = xml_util.create_and_write_tmp_file(
root_elem)
shutil.move(tmp_filename, full_path)
else:
shutil.move(uploaded_file_name, full_path)
elif uploaded_file_filename in [
rt_util.TOOL_DEPENDENCY_DEFINITION_FILENAME
]:
# Inspect the contents of the file to see if changeset_revision values are
# missing and if so, set them appropriately.
altered, root_elem, error_message = tdah.handle_tag_attributes(
uploaded_file_name)
if error_message:
ok = False
message = error_message
status = 'error'
if ok:
if altered:
tmp_filename = xml_util.create_and_write_tmp_file(
root_elem)
shutil.move(tmp_filename, full_path)
else:
shutil.move(uploaded_file_name, full_path)
else:
shutil.move(uploaded_file_name, full_path)
if ok:
# See if any admin users have chosen to receive email alerts when a repository is updated.
# If so, check every uploaded file to ensure content is appropriate.
check_contents = commit_util.check_file_contents_for_email_alerts(
trans.app)
if check_contents and os.path.isfile(full_path):
content_alert_str = commit_util.check_file_content_for_html_and_images(
full_path)
else:
content_alert_str = ''
hg_util.add_changeset(repo_dir, full_path)
hg_util.commit_changeset(
repo_dir,
full_path_to_changeset=full_path,
username=trans.user.username,
message=commit_message)
if full_path.endswith(
'tool_data_table_conf.xml.sample'):
# Handle the special case where a tool_data_table_conf.xml.sample file is being uploaded
# by parsing the file and adding new entries to the in-memory trans.app.tool_data_tables
# dictionary.
error, error_message = stdtm.handle_sample_tool_data_table_conf_file(
full_path, persist=False)
if error:
message = '%s<br/>%s' % (message,
error_message)
# See if the content of the change set was valid.
admin_only = len(
repository.downloadable_revisions) != 1
suc.handle_email_alerts(
trans.app,
trans.request.host,
repository,
content_alert_str=content_alert_str,
new_repo_alert=new_repo_alert,
admin_only=admin_only)
if ok:
# Update the repository files for browsing.
hg_util.update_repository(repo_dir)
# Get the new repository tip.
if tip == repository.tip():
message = 'No changes to repository. '
status = 'warning'
else:
if (isgzip or isbz2) and uncompress_file:
uncompress_str = ' uncompressed and '
else:
uncompress_str = ' '
if uploaded_directory:
source_type = "repository"
source = url
else:
source_type = "file"
source = uploaded_file_filename
message = "The %s <b>%s</b> has been successfully%suploaded to the repository. " % \
(source_type, escape(source), uncompress_str)
if istar and (undesirable_dirs_removed
or undesirable_files_removed):
items_removed = undesirable_dirs_removed + undesirable_files_removed
message += " %d undesirable items (.hg .svn .git directories, .DS_Store, hgrc files, etc) " % items_removed
message += "were removed from the archive. "
if istar and remove_repo_files_not_in_tar and files_to_remove:
if upload_point is not None:
message += " %d files were removed from the repository relative to the selected upload point '%s'. " % \
(len(files_to_remove), upload_point)
else:
message += " %d files were removed from the repository root. " % len(
files_to_remove)
rmm = repository_metadata_manager.RepositoryMetadataManager(
app=trans.app,
user=trans.user,
repository=repository)
status, error_message = \
rmm.set_repository_metadata_due_to_new_tip(trans.request.host,
content_alert_str=content_alert_str,
**kwd)
if error_message:
message = error_message
kwd['message'] = message
if repository.metadata_revisions:
# A repository's metadata revisions are order descending by update_time, so the zeroth revision
# will be the tip just after an upload.
metadata_dict = repository.metadata_revisions[
0].metadata
else:
metadata_dict = {}
dd = dependency_display.DependencyDisplayer(trans.app)
if str(repository.type) not in [
rt_util.REPOSITORY_SUITE_DEFINITION,
rt_util.TOOL_DEPENDENCY_DEFINITION
]:
change_repository_type_message = rt_util.generate_message_for_repository_type_change(
trans.app, repository)
if change_repository_type_message:
message += change_repository_type_message
status = 'warning'
else:
# Provide a warning message if a tool_dependencies.xml file is provided, but tool dependencies
# weren't loaded due to a requirement tag mismatch or some other problem. Tool dependency
# definitions can define orphan tool dependencies (no relationship to any tools contained in the
# repository), so warning messages are important because orphans are always valid. The repository
# owner must be warned in case they did not intend to define an orphan dependency, but simply
# provided incorrect information (tool shed, name owner, changeset_revision) for the definition.
orphan_message = dd.generate_message_for_orphan_tool_dependencies(
repository, metadata_dict)
if orphan_message:
message += orphan_message
status = 'warning'
# Handle messaging for invalid tool dependencies.
invalid_tool_dependencies_message = dd.generate_message_for_invalid_tool_dependencies(
metadata_dict)
if invalid_tool_dependencies_message:
message += invalid_tool_dependencies_message
status = 'error'
# Handle messaging for invalid repository dependencies.
invalid_repository_dependencies_message = \
dd.generate_message_for_invalid_repository_dependencies(metadata_dict,
error_from_tuple=True)
if invalid_repository_dependencies_message:
message += invalid_repository_dependencies_message
status = 'error'
# Reset the tool_data_tables by loading the empty tool_data_table_conf.xml file.
stdtm.reset_tool_data_tables()
if uploaded_directory:
basic_util.remove_dir(uploaded_directory)
trans.response.send_redirect(
web.url_for(controller='repository',
action='browse_repository',
id=repository_id,
commit_message='Deleted selected files',
message=message,
status=status))
else:
if uploaded_directory:
basic_util.remove_dir(uploaded_directory)
status = 'error'
# Reset the tool_data_tables by loading the empty tool_data_table_conf.xml file.
stdtm.reset_tool_data_tables()
return trans.fill_template(
'/webapps/tool_shed/repository/upload.mako',
repository=repository,
changeset_revision=tip,
url=url,
commit_message=commit_message,
uncompress_file=uncompress_file,
remove_repo_files_not_in_tar=remove_repo_files_not_in_tar,
message=message,
status=status)
def check_and_update_repository_metadata(app, info_only=False, verbosity=1):
"""
This method will iterate through all records in the repository_metadata
table, checking each one for tool metadata, then checking the tool
metadata for tests. Each tool's metadata should look something like:
{
"add_to_tool_panel": true,
"description": "",
"guid": "toolshed.url:9009/repos/owner/name/tool_id/1.2.3",
"id": "tool_wrapper",
"name": "Map with Tool Wrapper",
"requirements": [],
"tests": [],
"tool_config": "database/community_files/000/repo_1/tool_wrapper.xml",
"tool_type": "default",
"version": "1.2.3",
"version_string_cmd": null
}
If the "tests" attribute is missing or empty, this script will mark the metadata record (which is specific to a changeset revision of a repository)
not to be tested. If each "tools" attribute has at least one valid "tests" entry, this script will do nothing, and leave it available for the install
and test repositories script to process. If the tested changeset revision does not have a test-data directory, this script will also mark the revision
not to be tested.
"""
start = time.time()
skip_metadata_ids = []
checked_repository_ids = []
tool_count = 0
has_tests = 0
no_tests = 0
valid_revisions = 0
invalid_revisions = 0
records_checked = 0
# Do not check metadata records that have an entry in the skip_tool_tests table, since they won't be tested anyway.
print '# -------------------------------------------------------------------------------------------'
print '# The skip_tool_test setting has been set for the following repository revision, so they will not be tested.'
skip_metadata_ids = []
for skip_tool_test in app.sa_session.query(app.model.SkipToolTest):
print '# repository_metadata_id: %s, changeset_revision: %s' % \
( str( skip_tool_test.repository_metadata_id ), str( skip_tool_test.initial_changeset_revision ) )
print 'reason: %s' % str(skip_tool_test.comment)
skip_metadata_ids.append(skip_tool_test.repository_metadata_id)
# Get the list of metadata records to check for functional tests and test data. Limit this to records that have not been flagged do_not_test,
# since there's no need to check them again if they won't be tested anyway. Also filter out changeset revisions that are not downloadable,
# because it's redundant to test a revision that a user can't install.
for repository_metadata in app.sa_session.query( app.model.RepositoryMetadata ) \
.filter( and_( app.model.RepositoryMetadata.table.c.downloadable == True,
app.model.RepositoryMetadata.table.c.includes_tools == True,
app.model.RepositoryMetadata.table.c.do_not_test == False ) ):
# Initialize some items.
missing_test_components = []
revision_has_test_data = False
testable_revision = False
repository = repository_metadata.repository
records_checked += 1
# Check the next repository revision.
changeset_revision = str(repository_metadata.changeset_revision)
name = repository.name
owner = repository.user.username
metadata = repository_metadata.metadata
repository = repository_metadata.repository
if repository.id not in checked_repository_ids:
checked_repository_ids.append(repository.id)
print '# -------------------------------------------------------------------------------------------'
print '# Checking revision %s of %s owned by %s.' % (
changeset_revision, name, owner)
if repository_metadata.id in skip_metadata_ids:
print '# Skipping revision %s of %s owned by %s because the skip_tool_test setting has been set.' % (
changeset_revision, name, owner)
continue
# If this changeset revision has no tools, we don't need to do anything here, the install and test script has a filter for returning
# only repositories that contain tools.
tool_dicts = metadata.get('tools', None)
if tool_dicts is not None:
# Clone the repository up to the changeset revision we're checking.
repo_dir = repository.repo_path(app)
hg_util.get_repo_for_repository(app,
repository=None,
repo_path=repo_dir,
create=False)
work_dir = tempfile.mkdtemp(prefix="tmp-toolshed-cafr")
cloned_ok, error_message = hg_util.clone_repository(
repo_dir, work_dir, changeset_revision)
if cloned_ok:
# Iterate through all the directories in the cloned changeset revision and determine whether there's a
# directory named test-data. If this directory is not present update the metadata record for the changeset
# revision we're checking.
for root, dirs, files in os.walk(work_dir):
if '.hg' in dirs:
dirs.remove('.hg')
if 'test-data' in dirs:
revision_has_test_data = True
test_data_path = os.path.join(
root, dirs[dirs.index('test-data')])
break
if revision_has_test_data:
print '# Test data directory found in changeset revision %s of repository %s owned by %s.' % (
changeset_revision, name, owner)
else:
print '# Test data directory missing in changeset revision %s of repository %s owned by %s.' % (
changeset_revision, name, owner)
print '# Checking for functional tests in changeset revision %s of %s, owned by %s.' % \
( changeset_revision, name, owner )
# Inspect each tool_dict for defined functional tests. If there
# are no tests, this tool should not be tested, since the tool
# functional tests only report failure if the test itself fails,
# not if it's missing or undefined. Filtering out those
# repositories at this step will reduce the number of "false
# negatives" the automated functional test framework produces.
for tool_dict in tool_dicts:
failure_reason = ''
problem_found = False
tool_has_defined_tests = False
tool_has_test_files = False
missing_test_files = []
tool_count += 1
tool_id = tool_dict['id']
tool_version = tool_dict['version']
tool_guid = tool_dict['guid']
if verbosity >= 1:
print "# Checking tool ID '%s' in changeset revision %s of %s." % (
tool_id, changeset_revision, name)
defined_test_dicts = tool_dict.get('tests', None)
if defined_test_dicts is not None:
# We need to inspect the <test> tags because the following tags...
# <tests>
# </tests>
# ...will produce the following metadata:
# "tests": []
# And the following tags...
# <tests>
# <test>
# </test>
# </tests>
# ...will produce the following metadata:
# "tests":
# [{"inputs": [], "name": "Test-1", "outputs": [], "required_files": []}]
for defined_test_dict in defined_test_dicts:
inputs = defined_test_dict.get('inputs', [])
outputs = defined_test_dict.get('outputs', [])
if inputs and outputs:
# At least one tool within the repository has a valid <test> tag.
tool_has_defined_tests = True
break
if tool_has_defined_tests:
print "# Tool ID '%s' in changeset revision %s of %s has one or more valid functional tests defined." % \
( tool_id, changeset_revision, name )
has_tests += 1
else:
print '# No functional tests defined for %s.' % tool_id
no_tests += 1
if tool_has_defined_tests and revision_has_test_data:
missing_test_files = check_for_missing_test_files(
defined_test_dicts, test_data_path)
if missing_test_files:
print "# Tool id '%s' in changeset revision %s of %s is missing one or more required test files: %s" % \
( tool_id, changeset_revision, name, ', '.join( missing_test_files ) )
else:
tool_has_test_files = True
if not revision_has_test_data:
failure_reason += 'Repository does not have a test-data directory. '
problem_found = True
if not tool_has_defined_tests:
failure_reason += 'Functional test definitions missing for %s. ' % tool_id
problem_found = True
if missing_test_files:
failure_reason += 'One or more test files are missing for tool %s: %s' % (
tool_id, ', '.join(missing_test_files))
problem_found = True
test_errors = dict(tool_id=tool_id,
tool_version=tool_version,
tool_guid=tool_guid,
missing_components=failure_reason)
# Only append this error dict if it hasn't already been added.
if problem_found:
if test_errors not in missing_test_components:
missing_test_components.append(test_errors)
if tool_has_defined_tests and tool_has_test_files:
print '# Revision %s of %s owned by %s is a testable revision.' % (
changeset_revision, name, owner)
testable_revision = True
# Remove the cloned repository path. This has to be done after the check for required test files, for obvious reasons.
if os.path.exists(work_dir):
shutil.rmtree(work_dir)
if not missing_test_components:
valid_revisions += 1
print '# All tools have functional tests in changeset revision %s of repository %s owned by %s.' % (
changeset_revision, name, owner)
else:
invalid_revisions += 1
print '# Some tools have problematic functional tests in changeset revision %s of repository %s owned by %s.' % (
changeset_revision, name, owner)
if verbosity >= 1:
for missing_test_component in missing_test_components:
if 'missing_components' in missing_test_component:
print '# %s' % missing_test_component[
'missing_components']
if not info_only:
# Get or create the list of tool_test_results dictionaries.
if repository_metadata.tool_test_results is not None:
# We'll listify the column value in case it uses the old approach of storing the results of only a single test run.
tool_test_results_dicts = listify(
repository_metadata.tool_test_results)
else:
tool_test_results_dicts = []
if tool_test_results_dicts:
# Inspect the tool_test_results_dict for the last test run in case it contains only a test_environment
# entry. This will occur with multiple runs of this script without running the associated
# install_and_test_tool_sed_repositories.sh script which will further populate the tool_test_results_dict.
tool_test_results_dict = tool_test_results_dicts[0]
if len(tool_test_results_dict) <= 1:
# We can re-use the mostly empty tool_test_results_dict for this run because it is either empty or it contains only
# a test_environment entry. If we use it we need to temporarily eliminate it from the list of tool_test_results_dicts
# since it will be re-inserted later.
tool_test_results_dict = tool_test_results_dicts.pop(0)
elif (len(tool_test_results_dict) == 2
and 'test_environment' in tool_test_results_dict and
'missing_test_components' in tool_test_results_dict):
# We can re-use tool_test_results_dict if its only entries are "test_environment" and "missing_test_components".
# In this case, some tools are missing tests components while others are not.
tool_test_results_dict = tool_test_results_dicts.pop(0)
else:
# The latest tool_test_results_dict has been populated with the results of a test run, so it cannot be used.
tool_test_results_dict = {}
else:
# Create a new dictionary for the most recent test run.
tool_test_results_dict = {}
test_environment_dict = tool_test_results_dict.get(
'test_environment', {})
# Add the current time as the approximate time that this test run occurs. A similar value will also be
# set to the repository_metadata.time_last_tested column, but we also store it here because the Tool Shed
# may be configured to store multiple test run results, so each must be associated with a time stamp.
now = time.strftime("%Y-%m-%d %H:%M:%S")
test_environment_dict['time_tested'] = now
test_environment_dict[
'tool_shed_database_version'] = get_database_version(app)
test_environment_dict[
'tool_shed_mercurial_version'] = __version__.version
test_environment_dict[
'tool_shed_revision'] = get_repository_current_revision(
os.getcwd())
tool_test_results_dict[
'test_environment'] = test_environment_dict
# The repository_metadata.time_last_tested column is not changed by this script since no testing is performed here.
if missing_test_components:
# If functional test definitions or test data are missing, set do_not_test = True if no tool with valid tests has been
# found in this revision, and:
# a) There are multiple downloadable revisions, and the revision being tested is not the most recent downloadable revision.
# In this case, the revision will never be updated with the missing components, and re-testing it would be redundant.
# b) There are one or more downloadable revisions, and the provided changeset revision is the most recent downloadable
# revision. In this case, if the repository is updated with test data or functional tests, the downloadable
# changeset revision that was tested will either be replaced with the new changeset revision, or a new downloadable
# changeset revision will be created, either of which will be automatically checked and flagged as appropriate.
# In the install and test script, this behavior is slightly different, since we do want to always run functional
# tests on the most recent downloadable changeset revision.
if should_set_do_not_test_flag(app, repository,
changeset_revision,
testable_revision):
print "# Setting do_not_test to True on revision %s of %s owned by %s because it is missing test components" % (
changeset_revision, name, owner)
print "# and it is not the latest downloadable revision."
repository_metadata.do_not_test = True
if not testable_revision:
# Even though some tools may be missing test components, it may be possible to test other tools. Since the
# install and test framework filters out repositories marked as missing test components, we'll set it only if
# no tools can be tested.
print '# Setting missing_test_components to True for revision %s of %s owned by %s because all tools are missing test components.' % (
changeset_revision, name, owner)
repository_metadata.missing_test_components = True
print "# Setting tools_functionally_correct to False on revision %s of %s owned by %s because it is missing test components" % (
changeset_revision, name, owner)
repository_metadata.tools_functionally_correct = False
tool_test_results_dict[
'missing_test_components'] = missing_test_components
# Store only the configured number of test runs.
num_tool_test_results_saved = int(
app.config.num_tool_test_results_saved)
if len(tool_test_results_dicts) >= num_tool_test_results_saved:
test_results_index = num_tool_test_results_saved - 1
new_tool_test_results_dicts = tool_test_results_dicts[:
test_results_index]
else:
new_tool_test_results_dicts = [
d for d in tool_test_results_dicts
]
# Insert the new element into the first position in the list.
new_tool_test_results_dicts.insert(0, tool_test_results_dict)
repository_metadata.tool_test_results = new_tool_test_results_dicts
app.sa_session.add(repository_metadata)
app.sa_session.flush()
stop = time.time()
print '# -------------------------------------------------------------------------------------------'
print '# Checked %d repositories with %d tools in %d changeset revisions.' % (
len(checked_repository_ids), tool_count, records_checked)
print '# %d revisions found with functional tests and test data for all tools.' % valid_revisions
print '# %d revisions found with one or more tools missing functional tests and/or test data.' % invalid_revisions
print '# Found %d tools without functional tests.' % no_tests
print '# Found %d tools with functional tests.' % has_tests
if info_only:
print '# Database not updated, info_only set.'
print "# Elapsed time: ", stop - start
print "#############################################################################"
def install_repository(self,
repository_elem,
tool_shed_repository,
install_dependencies,
is_repository_dependency=False):
"""Install a single repository, loading contained tools into the tool panel."""
# Install path is of the form: <tool path>/<tool shed>/repos/<repository owner>/<repository name>/<installed changeset revision>
relative_clone_dir = os.path.join(
tool_shed_repository.tool_shed, 'repos',
tool_shed_repository.owner, tool_shed_repository.name,
tool_shed_repository.installed_changeset_revision)
clone_dir = os.path.join(self.tool_path, relative_clone_dir)
cloned_ok = self.__iscloned(clone_dir)
is_installed = False
# Any of the following states should count as installed in this context.
if tool_shed_repository.status in [
self.app.install_model.ToolShedRepository.installation_status.
INSTALLED, self.app.install_model.ToolShedRepository.
installation_status.ERROR, self.app.install_model.
ToolShedRepository.installation_status.UNINSTALLED,
self.app.install_model.ToolShedRepository.installation_status.
DEACTIVATED
]:
is_installed = True
if cloned_ok and is_installed:
log.info(
"Skipping automatic install of repository '%s' because it has already been installed in location %s",
tool_shed_repository.name, clone_dir)
else:
irm = install_manager.InstallRepositoryManager(self.app, self.tpm)
repository_clone_url = os.path.join(self.tool_shed_url, 'repos',
tool_shed_repository.owner,
tool_shed_repository.name)
relative_install_dir = os.path.join(relative_clone_dir,
tool_shed_repository.name)
install_dir = os.path.join(clone_dir, tool_shed_repository.name)
ctx_rev = suc.get_ctx_rev(
self.app, self.tool_shed_url, tool_shed_repository.name,
tool_shed_repository.owner,
tool_shed_repository.installed_changeset_revision)
if not cloned_ok:
irm.update_tool_shed_repository_status(
tool_shed_repository, self.app.install_model.
ToolShedRepository.installation_status.CLONING)
cloned_ok, error_message = hg_util.clone_repository(
repository_clone_url, os.path.abspath(install_dir),
ctx_rev)
if cloned_ok and not is_installed:
self.handle_repository_contents(
tool_shed_repository=tool_shed_repository,
repository_clone_url=repository_clone_url,
relative_install_dir=relative_install_dir,
repository_elem=repository_elem,
install_dependencies=install_dependencies,
is_repository_dependency=is_repository_dependency)
self.app.install_model.context.refresh(tool_shed_repository)
irm.update_tool_shed_repository_status(
tool_shed_repository, self.app.install_model.
ToolShedRepository.installation_status.INSTALLED)
else:
log.error('Error attempting to clone repository %s: %s',
str(tool_shed_repository.name), str(error_message))
irm.update_tool_shed_repository_status(
tool_shed_repository,
self.app.install_model.ToolShedRepository.
installation_status.ERROR,
error_message=error_message)
def install_repository(self,
repository_elem,
tool_shed_repository,
install_dependencies,
is_repository_dependency=False):
"""Install a single repository, loading contained tools into the tool panel."""
# Install path is of the form: <tool path>/<tool shed>/repos/<repository owner>/<repository name>/<installed changeset revision>
relative_clone_dir = os.path.join(
tool_shed_repository.tool_shed, 'repos',
tool_shed_repository.owner, tool_shed_repository.name,
tool_shed_repository.installed_changeset_revision)
clone_dir = os.path.join(self.tool_path, relative_clone_dir)
cloned_ok = self.__iscloned(clone_dir)
is_installed = False
# Any of the following states should count as installed in this context.
if tool_shed_repository.status in [
self.app.install_model.ToolShedRepository.installation_status.
INSTALLED, self.app.install_model.ToolShedRepository.
installation_status.ERROR, self.app.install_model.
ToolShedRepository.installation_status.UNINSTALLED,
self.app.install_model.ToolShedRepository.installation_status.
DEACTIVATED
]:
is_installed = True
if cloned_ok and is_installed:
log.info(
"Skipping automatic install of repository '%s' because it has already been installed in location %s",
tool_shed_repository.name, clone_dir)
else:
irm = install_manager.InstallRepositoryManager(self.app, self.tpm)
repository_clone_url = os.path.join(self.tool_shed_url, 'repos',
tool_shed_repository.owner,
tool_shed_repository.name)
relative_install_dir = os.path.join(relative_clone_dir,
tool_shed_repository.name)
install_dir = os.path.join(clone_dir, tool_shed_repository.name)
ctx_rev = suc.get_ctx_rev(
self.app, self.tool_shed_url, tool_shed_repository.name,
tool_shed_repository.owner,
tool_shed_repository.installed_changeset_revision)
if not cloned_ok:
irm.update_tool_shed_repository_status(
tool_shed_repository, self.app.install_model.
ToolShedRepository.installation_status.CLONING)
cloned_ok, error_message = hg_util.clone_repository(
repository_clone_url, os.path.abspath(install_dir),
ctx_rev)
if cloned_ok and not is_installed:
self.handle_repository_contents(
tool_shed_repository=tool_shed_repository,
repository_clone_url=repository_clone_url,
relative_install_dir=relative_install_dir,
repository_elem=repository_elem,
install_dependencies=install_dependencies,
is_repository_dependency=is_repository_dependency)
self.app.install_model.context.refresh(tool_shed_repository)
metadata_dict = tool_shed_repository.metadata
if 'tools' in metadata_dict:
# Initialize the ToolVersionManager.
tvm = tool_version_manager.ToolVersionManager(self.app)
irm.update_tool_shed_repository_status(
tool_shed_repository,
self.app.install_model.ToolShedRepository.
installation_status.SETTING_TOOL_VERSIONS)
# Get the tool_versions from the tool shed for each tool in the installed change set.
params = dict(name=tool_shed_repository.name,
owner=self.repository_owner,
changeset_revision=tool_shed_repository.
installed_changeset_revision)
pathspec = ['repository', 'get_tool_versions']
text = util.url_get(
self.tool_shed_url,
password_mgr=self.app.tool_shed_registry.url_auth(
self.tool_shed_url),
pathspec=pathspec,
params=params)
if text:
tool_version_dicts = json.loads(text)
tvm.handle_tool_versions(tool_version_dicts,
tool_shed_repository)
else:
# Set the tool versions since they seem to be missing
# for this repository in the tool shed. CRITICAL NOTE:
# These default settings may not properly handle all
# parent/child associations.
for tool_dict in metadata_dict['tools']:
tool_id = tool_dict['guid']
old_tool_id = tool_dict['id']
tool_version_using_old_id = tvm.get_tool_version(
old_tool_id)
tool_version_using_guid = tvm.get_tool_version(
tool_id)
if not tool_version_using_old_id:
tool_version_using_old_id = self.app.install_model.ToolVersion(
tool_id=old_tool_id,
tool_shed_repository=tool_shed_repository)
self.app.install_model.context.add(
tool_version_using_old_id)
self.app.install_model.context.flush()
if not tool_version_using_guid:
tool_version_using_guid = self.app.install_model.ToolVersion(
tool_id=tool_id,
tool_shed_repository=tool_shed_repository)
self.app.install_model.context.add(
tool_version_using_guid)
self.app.install_model.context.flush()
# Associate the two versions as parent / child.
tool_version_association = tvm.get_tool_version_association(
tool_version_using_old_id,
tool_version_using_guid)
if not tool_version_association:
tool_version_association = \
self.app.install_model.ToolVersionAssociation( tool_id=tool_version_using_guid.id,
parent_id=tool_version_using_old_id.id )
self.app.install_model.context.add(
tool_version_association)
self.app.install_model.context.flush()
irm.update_tool_shed_repository_status(
tool_shed_repository, self.app.install_model.
ToolShedRepository.installation_status.INSTALLED)
else:
log.error('Error attempting to clone repository %s: %s',
str(tool_shed_repository.name), str(error_message))
irm.update_tool_shed_repository_status(
tool_shed_repository,
self.app.install_model.ToolShedRepository.
installation_status.ERROR,
error_message=error_message)
def upload(self, trans, **kwd):
message = escape(kwd.get('message', ''))
status = kwd.get('status', 'done')
commit_message = escape(kwd.get('commit_message', 'Uploaded'))
repository_id = kwd.get('repository_id', '')
repository = repository_util.get_repository_in_tool_shed(trans.app, repository_id)
repo_dir = repository.repo_path(trans.app)
uncompress_file = util.string_as_bool(kwd.get('uncompress_file', 'true'))
remove_repo_files_not_in_tar = util.string_as_bool(kwd.get('remove_repo_files_not_in_tar', 'true'))
uploaded_file = None
upload_point = commit_util.get_upload_point(repository, **kwd)
tip = repository.tip(trans.app)
file_data = kwd.get('file_data', '')
url = kwd.get('url', '')
# Part of the upload process is sending email notification to those that have registered to
# receive them. One scenario occurs when the first change set is produced for the repository.
# See the suc.handle_email_alerts() method for the definition of the scenarios.
new_repo_alert = repository.is_new(trans.app)
uploaded_directory = None
if kwd.get('upload_button', False):
if file_data == '' and url == '':
message = 'No files were entered on the upload form.'
status = 'error'
uploaded_file = None
elif url and url.startswith('hg'):
# Use mercurial clone to fetch repository, contents will then be copied over.
uploaded_directory = tempfile.mkdtemp()
repo_url = 'http%s' % url[len('hg'):]
cloned_ok, error_message = hg_util.clone_repository(repo_url, uploaded_directory)
if not cloned_ok:
message = 'Error uploading via mercurial clone: %s' % error_message
status = 'error'
basic_util.remove_dir(uploaded_directory)
uploaded_directory = None
elif url:
valid_url = True
try:
stream = requests.get(url, stream=True)
except Exception as e:
valid_url = False
message = 'Error uploading file via http: %s' % str(e)
status = 'error'
uploaded_file = None
if valid_url:
fd, uploaded_file_name = tempfile.mkstemp()
uploaded_file = open(uploaded_file_name, 'wb')
for chunk in stream.iter_content(chunk_size=util.CHUNK_SIZE):
if chunk:
uploaded_file.write(chunk)
uploaded_file.flush()
uploaded_file_filename = url.split('/')[-1]
isempty = os.path.getsize(os.path.abspath(uploaded_file_name)) == 0
elif file_data not in ('', None):
uploaded_file = file_data.file
uploaded_file_name = uploaded_file.name
uploaded_file_filename = os.path.split(file_data.filename)[-1]
isempty = os.path.getsize(os.path.abspath(uploaded_file_name)) == 0
if uploaded_file or uploaded_directory:
rdah = attribute_handlers.RepositoryDependencyAttributeHandler(trans.app, unpopulate=False)
tdah = attribute_handlers.ToolDependencyAttributeHandler(trans.app, unpopulate=False)
stdtm = ShedToolDataTableManager(trans.app)
ok = True
isgzip = False
isbz2 = False
if uploaded_file:
if uncompress_file:
isgzip = checkers.is_gzip(uploaded_file_name)
if not isgzip:
isbz2 = checkers.is_bz2(uploaded_file_name)
if isempty:
tar = None
istar = False
else:
# Determine what we have - a single file or an archive
try:
if (isgzip or isbz2) and uncompress_file:
# Open for reading with transparent compression.
tar = tarfile.open(uploaded_file_name, 'r:*')
else:
tar = tarfile.open(uploaded_file_name)
istar = True
except tarfile.ReadError as e:
tar = None
istar = False
else:
# Uploaded directory
istar = False
if istar:
ok, message, files_to_remove, content_alert_str, undesirable_dirs_removed, undesirable_files_removed = \
repository_content_util.upload_tar(
trans,
rdah,
tdah,
repository,
tar,
uploaded_file,
upload_point,
remove_repo_files_not_in_tar,
commit_message,
new_repo_alert
)
elif uploaded_directory:
ok, message, files_to_remove, content_alert_str, undesirable_dirs_removed, undesirable_files_removed = \
self.upload_directory(trans,
rdah,
tdah,
repository,
uploaded_directory,
upload_point,
remove_repo_files_not_in_tar,
commit_message,
new_repo_alert)
else:
if (isgzip or isbz2) and uncompress_file:
uploaded_file_filename = commit_util.uncompress(repository,
uploaded_file_name,
uploaded_file_filename,
isgzip=isgzip,
isbz2=isbz2)
if repository.type == rt_util.REPOSITORY_SUITE_DEFINITION and \
uploaded_file_filename != rt_util.REPOSITORY_DEPENDENCY_DEFINITION_FILENAME:
ok = False
message = 'Repositories of type <b>Repository suite definition</b> can only contain a single file named '
message += '<b>repository_dependencies.xml</b>.'
elif repository.type == rt_util.TOOL_DEPENDENCY_DEFINITION and \
uploaded_file_filename != rt_util.TOOL_DEPENDENCY_DEFINITION_FILENAME:
ok = False
message = 'Repositories of type <b>Tool dependency definition</b> can only contain a single file named '
message += '<b>tool_dependencies.xml</b>.'
if ok:
if upload_point is not None:
full_path = os.path.abspath(os.path.join(repo_dir, upload_point, uploaded_file_filename))
else:
full_path = os.path.abspath(os.path.join(repo_dir, uploaded_file_filename))
# Move some version of the uploaded file to the load_point within the repository hierarchy.
if uploaded_file_filename in [rt_util.REPOSITORY_DEPENDENCY_DEFINITION_FILENAME]:
# Inspect the contents of the file to see if toolshed or changeset_revision attributes
# are missing and if so, set them appropriately.
altered, root_elem, error_message = rdah.handle_tag_attributes(uploaded_file_name)
if error_message:
ok = False
message = error_message
status = 'error'
elif altered:
tmp_filename = xml_util.create_and_write_tmp_file(root_elem)
shutil.move(tmp_filename, full_path)
else:
shutil.move(uploaded_file_name, full_path)
elif uploaded_file_filename in [rt_util.TOOL_DEPENDENCY_DEFINITION_FILENAME]:
# Inspect the contents of the file to see if changeset_revision values are
# missing and if so, set them appropriately.
altered, root_elem, error_message = tdah.handle_tag_attributes(uploaded_file_name)
if error_message:
ok = False
message = error_message
status = 'error'
if ok:
if altered:
tmp_filename = xml_util.create_and_write_tmp_file(root_elem)
shutil.move(tmp_filename, full_path)
else:
shutil.move(uploaded_file_name, full_path)
else:
shutil.move(uploaded_file_name, full_path)
if ok:
# See if any admin users have chosen to receive email alerts when a repository is updated.
# If so, check every uploaded file to ensure content is appropriate.
check_contents = commit_util.check_file_contents_for_email_alerts(trans.app)
if check_contents and os.path.isfile(full_path):
content_alert_str = commit_util.check_file_content_for_html_and_images(full_path)
else:
content_alert_str = ''
hg_util.add_changeset(repo_dir, full_path)
hg_util.commit_changeset(repo_dir,
full_path_to_changeset=full_path,
username=trans.user.username,
message=commit_message)
if full_path.endswith('tool_data_table_conf.xml.sample'):
# Handle the special case where a tool_data_table_conf.xml.sample file is being uploaded
# by parsing the file and adding new entries to the in-memory trans.app.tool_data_tables
# dictionary.
error, error_message = stdtm.handle_sample_tool_data_table_conf_file(full_path, persist=False)
if error:
message = '%s<br/>%s' % (message, error_message)
# See if the content of the change set was valid.
admin_only = len(repository.downloadable_revisions) != 1
suc.handle_email_alerts(trans.app,
trans.request.host,
repository,
content_alert_str=content_alert_str,
new_repo_alert=new_repo_alert,
admin_only=admin_only)
if ok:
# Update the repository files for browsing.
hg_util.update_repository(repo_dir)
# Get the new repository tip.
if tip == repository.tip(trans.app):
message = 'No changes to repository. '
status = 'warning'
else:
if (isgzip or isbz2) and uncompress_file:
uncompress_str = ' uncompressed and '
else:
uncompress_str = ' '
if uploaded_directory:
source_type = "repository"
source = url
else:
source_type = "file"
source = uploaded_file_filename
message = "The %s <b>%s</b> has been successfully%suploaded to the repository. " % \
(source_type, escape(source), uncompress_str)
if istar and (undesirable_dirs_removed or undesirable_files_removed):
items_removed = undesirable_dirs_removed + undesirable_files_removed
message += " %d undesirable items (.hg .svn .git directories, .DS_Store, hgrc files, etc) " % items_removed
message += "were removed from the archive. "
if istar and remove_repo_files_not_in_tar and files_to_remove:
if upload_point is not None:
message += " %d files were removed from the repository relative to the selected upload point '%s'. " % \
(len(files_to_remove), upload_point)
else:
message += " %d files were removed from the repository root. " % len(files_to_remove)
rmm = repository_metadata_manager.RepositoryMetadataManager(app=trans.app,
user=trans.user,
repository=repository)
status, error_message = \
rmm.set_repository_metadata_due_to_new_tip(trans.request.host,
content_alert_str=content_alert_str,
**kwd)
if error_message:
message = error_message
kwd['message'] = message
if repository.metadata_revisions:
# A repository's metadata revisions are order descending by update_time, so the zeroth revision
# will be the tip just after an upload.
metadata_dict = repository.metadata_revisions[0].metadata
else:
metadata_dict = {}
dd = dependency_display.DependencyDisplayer(trans.app)
if str(repository.type) not in [rt_util.REPOSITORY_SUITE_DEFINITION,
rt_util.TOOL_DEPENDENCY_DEFINITION]:
change_repository_type_message = rt_util.generate_message_for_repository_type_change(trans.app,
repository)
if change_repository_type_message:
message += change_repository_type_message
status = 'warning'
else:
# Provide a warning message if a tool_dependencies.xml file is provided, but tool dependencies
# weren't loaded due to a requirement tag mismatch or some other problem. Tool dependency
# definitions can define orphan tool dependencies (no relationship to any tools contained in the
# repository), so warning messages are important because orphans are always valid. The repository
# owner must be warned in case they did not intend to define an orphan dependency, but simply
# provided incorrect information (tool shed, name owner, changeset_revision) for the definition.
orphan_message = dd.generate_message_for_orphan_tool_dependencies(repository, metadata_dict)
if orphan_message:
message += orphan_message
status = 'warning'
# Handle messaging for invalid tool dependencies.
invalid_tool_dependencies_message = dd.generate_message_for_invalid_tool_dependencies(metadata_dict)
if invalid_tool_dependencies_message:
message += invalid_tool_dependencies_message
status = 'error'
# Handle messaging for invalid repository dependencies.
invalid_repository_dependencies_message = \
dd.generate_message_for_invalid_repository_dependencies(metadata_dict,
error_from_tuple=True)
if invalid_repository_dependencies_message:
message += invalid_repository_dependencies_message
status = 'error'
# Reset the tool_data_tables by loading the empty tool_data_table_conf.xml file.
stdtm.reset_tool_data_tables()
if uploaded_directory:
basic_util.remove_dir(uploaded_directory)
trans.response.send_redirect(web.url_for(controller='repository',
action='browse_repository',
id=repository_id,
commit_message='Deleted selected files',
message=message,
status=status))
else:
if uploaded_directory:
basic_util.remove_dir(uploaded_directory)
status = 'error'
# Reset the tool_data_tables by loading the empty tool_data_table_conf.xml file.
stdtm.reset_tool_data_tables()
return trans.fill_template('/webapps/tool_shed/repository/upload.mako',
repository=repository,
changeset_revision=tip,
url=url,
commit_message=commit_message,
uncompress_file=uncompress_file,
remove_repo_files_not_in_tar=remove_repo_files_not_in_tar,
message=message,
status=status)