def upload( self, trans, **kwd ):
message = kwd.get( 'message', '' )
status = kwd.get( 'status', 'done' )
commit_message = kwd.get( 'commit_message', 'Uploaded' )
category_ids = util.listify( kwd.get( 'category_id', '' ) )
categories = suc.get_categories( trans )
repository_id = kwd.get( 'repository_id', '' )
repository = suc.get_repository_in_tool_shed( trans, repository_id )
repo_dir = repository.repo_path( trans.app )
repo = hg.repository( suc.get_configured_ui(), repo_dir )
uncompress_file = util.string_as_bool( kwd.get( 'uncompress_file', 'true' ) )
remove_repo_files_not_in_tar = util.string_as_bool( kwd.get( 'remove_repo_files_not_in_tar', 'true' ) )
uploaded_file = None
upload_point = commit_util.get_upload_point( repository, **kwd )
tip = repository.tip( trans.app )
file_data = kwd.get( 'file_data', '' )
url = kwd.get( 'url', '' )
# Part of the upload process is sending email notification to those that have registered to
# receive them. One scenario occurs when the first change set is produced for the repository.
# See the suc.handle_email_alerts() method for the definition of the scenarios.
new_repo_alert = repository.is_new( trans.app )
uploaded_directory = None
if kwd.get( 'upload_button', False ):
if file_data == '' and url == '':
message = 'No files were entered on the upload form.'
status = 'error'
uploaded_file = None
elif url and url.startswith( 'hg' ):
# Use mercurial clone to fetch repository, contents will then be copied over.
uploaded_directory = tempfile.mkdtemp()
repo_url = 'http%s' % url[ len( 'hg' ): ]
repo_url = repo_url.encode( 'ascii', 'replace' )
try:
commands.clone( suc.get_configured_ui(), repo_url, uploaded_directory )
except Exception, e:
message = 'Error uploading via mercurial clone: %s' % suc.to_html_string( str( e ) )
status = 'error'
suc.remove_dir( uploaded_directory )
uploaded_directory = None
elif url:
valid_url = True
try:
stream = urllib.urlopen( url )
except Exception, e:
valid_url = False
message = 'Error uploading file via http: %s' % str( e )
status = 'error'
uploaded_file = None
if valid_url:
fd, uploaded_file_name = tempfile.mkstemp()
uploaded_file = open( uploaded_file_name, 'wb' )
while 1:
chunk = stream.read( util.CHUNK_SIZE )
if not chunk:
break
uploaded_file.write( chunk )
uploaded_file.flush()
uploaded_file_filename = url.split( '/' )[ -1 ]
isempty = os.path.getsize( os.path.abspath( uploaded_file_name ) ) == 0
if converter_path or display_path:
# Create a dictionary of tool shed repository related information.
repository_dict = datatype_util.create_repository_dict_for_proprietary_datatypes( tool_shed=self.tool_shed,
name=tool_shed_repository.name,
owner=self.repository_owner,
installed_changeset_revision=tool_shed_repository.installed_changeset_revision,
tool_dicts=metadata_dict.get( 'tools', [] ),
converter_path=converter_path,
display_path=display_path )
if converter_path:
# Load proprietary datatype converters
self.app.datatypes_registry.load_datatype_converters( self.toolbox, installed_repository_dict=repository_dict )
if display_path:
# Load proprietary datatype display applications
self.app.datatypes_registry.load_display_applications( installed_repository_dict=repository_dict )
suc.remove_dir( work_dir )
def install_repository( self, repository_elem, tool_shed_repository, install_dependencies, is_repository_dependency=False ):
"""Install a single repository, loading contained tools into the tool panel."""
# Install path is of the form: <tool path>/<tool shed>/repos/<repository owner>/<repository name>/<installed changeset revision>
relative_clone_dir = os.path.join( tool_shed_repository.tool_shed,
'repos',
tool_shed_repository.owner,
tool_shed_repository.name,
tool_shed_repository.installed_changeset_revision )
clone_dir = os.path.join( self.tool_path, relative_clone_dir )
cloned_ok = self.__iscloned( clone_dir )
is_installed = False
# Any of the following states should count as installed in this context.
if tool_shed_repository.status in [ self.app.install_model.ToolShedRepository.installation_status.INSTALLED,
self.app.install_model.ToolShedRepository.installation_status.ERROR,
def export_repository( trans, tool_shed_url, repository_id, repository_name, changeset_revision, file_type, export_repository_dependencies, api=False ):
repository = suc.get_repository_in_tool_shed( trans, repository_id )
repositories_archive_filename = generate_repository_archive_filename( tool_shed_url,
str( repository.name ),
str( repository.user.username ),
changeset_revision,
file_type,
export_repository_dependencies=export_repository_dependencies,
use_tmp_archive_dir=True )
if export_repository_dependencies:
repo_info_dicts = get_repo_info_dicts( trans, tool_shed_url, repository_id, changeset_revision )
repository_ids = get_repository_ids( trans, repo_info_dicts )
ordered_repository_ids, ordered_repositories, ordered_changeset_revisions = order_components_for_import( trans, repository_ids, repo_info_dicts )
else:
ordered_repository_ids = []
ordered_repositories = []
ordered_changeset_revisions = []
if repository:
repository_metadata = suc.get_current_repository_metadata_for_changeset_revision( trans, repository, changeset_revision )
if repository_metadata:
ordered_repository_ids = [ repository_id ]
ordered_repositories = [ repository ]
ordered_changeset_revisions = [ repository_metadata.changeset_revision ]
repositories_archive = None
error_messages = ''
lock = threading.Lock()
lock.acquire( True )
try:
repositories_archive = tarfile.open( repositories_archive_filename, "w:%s" % file_type )
exported_repository_registry = ExportedRepositoryRegistry()
for index, repository_id in enumerate( ordered_repository_ids ):
work_dir = tempfile.mkdtemp( prefix="tmp-toolshed-export-er" )
ordered_repository = ordered_repositories[ index ]
ordered_changeset_revision = ordered_changeset_revisions[ index ]
repository_archive, error_message = generate_repository_archive( trans,
work_dir,
tool_shed_url,
ordered_repository,
ordered_changeset_revision,
file_type )
if error_message:
error_messages = '%s %s' % ( error_messages, error_message )
else:
archive_name = str( os.path.basename( repository_archive.name ) )
repositories_archive.add( repository_archive.name, arcname=archive_name )
attributes, sub_elements = get_repository_attributes_and_sub_elements( ordered_repository, archive_name )
elem = xml_util.create_element( 'repository', attributes=attributes, sub_elements=sub_elements )
exported_repository_registry.exported_repository_elems.append( elem )
suc.remove_dir( work_dir )
# Keep information about the export in a file name export_info.xml in the archive.
sub_elements = generate_export_elem( tool_shed_url, repository, changeset_revision, export_repository_dependencies, api )
export_elem = xml_util.create_element( 'export_info', attributes=None, sub_elements=sub_elements )
tmp_export_info = xml_util.create_and_write_tmp_file( export_elem, use_indent=True )
repositories_archive.add( tmp_export_info, arcname='export_info.xml' )
# Write the manifest, which must preserve the order in which the repositories should be imported.
exported_repository_root = xml_util.create_element( 'repositories' )
for exported_repository_elem in exported_repository_registry.exported_repository_elems:
exported_repository_root.append( exported_repository_elem )
tmp_manifest = xml_util.create_and_write_tmp_file( exported_repository_root, use_indent=True )
repositories_archive.add( tmp_manifest, arcname='manifest.xml' )
except Exception, e:
log.exception( str( e ) )
def sync_database_with_file_system( app, tool_shed_repository, tool_dependency_name, tool_dependency_version, tool_dependency_install_dir,
tool_dependency_type='package' ):
"""
The installation directory defined by the received tool_dependency_install_dir exists, so check for the presence
of fabric_util.INSTALLATION_LOG. If the files exists, we'll assume the tool dependency is installed, but not
necessarily successfully (it could be in an error state on disk. However, we can justifiably assume here that no
matter the state, an associated database record will exist.
"""
# This method should be reached very rarely. It implies that either the Galaxy environment became corrupted (i.e.,
# the database records for installed tool dependencies is not synchronized with tool dependencies on disk) or the Tool
# Shed's install and test framework is running.
sa_session = app.install_model.context.current
can_install_tool_dependency = False
tool_dependency = get_tool_dependency_by_name_version_type_repository( app,
tool_shed_repository,
tool_dependency_name,
tool_dependency_version,
tool_dependency_type )
if tool_dependency.status == app.install_model.ToolDependency.installation_status.INSTALLING:
# The tool dependency is in an Installing state, so we don't want to do anything to it. If the tool
# dependency is being installed by someone else, we don't want to interfere with that. This assumes
# the installation by "someone else" is not hung in an Installing state, which is a weakness if that
# "someone else" never repaired it.
log.debug( 'Skipping installation of tool dependency %s version %s because it has a status of %s' % \
( str( tool_dependency.name ), str( tool_dependency.version ), str( tool_dependency.status ) ) )
else:
# We have a pre-existing installation directory on the file system, but our associated database record is
# in a state that allowed us to arrive here - see the comment in common_install_util.handle_tool_dependencies().
# At this point, we'll inspect the installation directory to see if we have a "proper installation" and
# if so, synchronize the database record rather than reinstalling the dependency if we're "running_functional_tests".
# If we're not "running_functional_tests, we'll set the tool dependency's installation status to ERROR.
tool_dependency_installation_directory_contents = os.listdir( tool_dependency_install_dir )
if fabric_util.INSTALLATION_LOG in tool_dependency_installation_directory_contents:
# Since this tool dependency's installation directory contains an installation log, we consider it to be
# installed. In some cases the record may be missing from the database due to some activity outside of
# the control of the Tool Shed. Since a new record was created for it and we don't know the state of the
# files on disk, we will set it to an error state (unless we are running Tool Shed functional tests - see
# below).
log.debug( 'Skipping installation of tool dependency %s version %s because it is installed in %s' % \
( str( tool_dependency.name ), str( tool_dependency.version ), str( tool_dependency_install_dir ) ) )
if app.config.running_functional_tests:
# If we are running functional tests, the state will be set to Installed because previously compiled
# tool dependencies are not deleted by default, from the "install and test" framework..
tool_dependency.status = app.install_model.ToolDependency.installation_status.INSTALLED
else:
error_message = 'The installation directory for this tool dependency had contents, but the database had no record. '
error_message += 'The installation log may show this tool dependency to be correctly installed, but due to the '
error_message += 'missing database record, it is automatically set to Error.'
tool_dependency.status = app.install_model.ToolDependency.installation_status.ERROR
tool_dependency.error_message = error_message
else:
error_message = '\nInstallation path %s for tool dependency %s version %s exists, but the expected file %s' % \
( str( tool_dependency_install_dir ),
str( tool_dependency_name ),
str( tool_dependency_version ),
str( fabric_util.INSTALLATION_LOG ) )
error_message += ' is missing. This indicates an installation error so the tool dependency is being'
error_message += ' prepared for re-installation.'
print error_message
tool_dependency.status = app.install_model.ToolDependency.installation_status.NEVER_INSTALLED
suc.remove_dir( tool_dependency_install_dir )
can_install_tool_dependency = True
sa_session.add( tool_dependency )
sa_session.flush()
return tool_dependency, can_install_tool_dependency
def upload( self, trans, **kwd ):
message = kwd.get( 'message', '' )
status = kwd.get( 'status', 'done' )
commit_message = kwd.get( 'commit_message', 'Uploaded' )
category_ids = util.listify( kwd.get( 'category_id', '' ) )
categories = suc.get_categories( trans )
repository_id = kwd.get( 'repository_id', '' )
repository = suc.get_repository_in_tool_shed( trans, repository_id )
repo_dir = repository.repo_path( trans.app )
repo = hg.repository( suc.get_configured_ui(), repo_dir )
uncompress_file = util.string_as_bool( kwd.get( 'uncompress_file', 'true' ) )
remove_repo_files_not_in_tar = util.string_as_bool( kwd.get( 'remove_repo_files_not_in_tar', 'true' ) )
uploaded_file = None
upload_point = commit_util.get_upload_point( repository, **kwd )
tip = repository.tip( trans.app )
file_data = kwd.get( 'file_data', '' )
url = kwd.get( 'url', '' )
# Part of the upload process is sending email notification to those that have registered to
# receive them. One scenario occurs when the first change set is produced for the repository.
# See the suc.handle_email_alerts() method for the definition of the scenarios.
new_repo_alert = repository.is_new( trans.app )
uploaded_directory = None
if kwd.get( 'upload_button', False ):
if file_data == '' and url == '':
message = 'No files were entered on the upload form.'
status = 'error'
uploaded_file = None
elif url and url.startswith( 'hg' ):
# Use mercurial clone to fetch repository, contents will then be copied over.
uploaded_directory = tempfile.mkdtemp()
repo_url = 'http%s' % url[ len( 'hg' ): ]
repo_url = repo_url.encode( 'ascii', 'replace' )
commands.clone( suc.get_configured_ui(), repo_url, uploaded_directory )
elif url:
valid_url = True
try:
stream = urllib.urlopen( url )
except Exception, e:
valid_url = False
message = 'Error uploading file via http: %s' % str( e )
status = 'error'
uploaded_file = None
if valid_url:
fd, uploaded_file_name = tempfile.mkstemp()
uploaded_file = open( uploaded_file_name, 'wb' )
while 1:
chunk = stream.read( util.CHUNK_SIZE )
if not chunk:
break
uploaded_file.write( chunk )
uploaded_file.flush()
uploaded_file_filename = url.split( '/' )[ -1 ]
isempty = os.path.getsize( os.path.abspath( uploaded_file_name ) ) == 0
elif file_data not in ( '', None ):
uploaded_file = file_data.file
uploaded_file_name = uploaded_file.name
uploaded_file_filename = os.path.split( file_data.filename )[ -1 ]
isempty = os.path.getsize( os.path.abspath( uploaded_file_name ) ) == 0
if uploaded_file or uploaded_directory:
ok = True
isgzip = False
isbz2 = False
if uploaded_file:
if uncompress_file:
isgzip = checkers.is_gzip( uploaded_file_name )
if not isgzip:
isbz2 = checkers.is_bz2( uploaded_file_name )
if isempty:
tar = None
istar = False
else:
# Determine what we have - a single file or an archive
try:
if ( isgzip or isbz2 ) and uncompress_file:
# Open for reading with transparent compression.
tar = tarfile.open( uploaded_file_name, 'r:*' )
else:
tar = tarfile.open( uploaded_file_name )
istar = True
except tarfile.ReadError, e:
tar = None
istar = False
else:
# Uploaded directory
istar = False
if istar:
ok, message, files_to_remove, content_alert_str, undesirable_dirs_removed, undesirable_files_removed = \
self.upload_tar( trans, repository, tar, uploaded_file, upload_point, remove_repo_files_not_in_tar, commit_message, new_repo_alert )
elif uploaded_directory:
ok, message, files_to_remove, content_alert_str, undesirable_dirs_removed, undesirable_files_removed = \
self.upload_directory( trans, repository, uploaded_directory, upload_point, remove_repo_files_not_in_tar, commit_message, new_repo_alert )
else:
if ( isgzip or isbz2 ) and uncompress_file:
uploaded_file_filename = commit_util.uncompress( repository, uploaded_file_name, uploaded_file_filename, isgzip=isgzip, isbz2=isbz2 )
if repository.type == rt_util.TOOL_DEPENDENCY_DEFINITION and uploaded_file_filename != suc.TOOL_DEPENDENCY_DEFINITION_FILENAME:
ok = False
message = 'Repositories of type <b>Tool dependency definition</b> can only contain a single file named <b>tool_dependencies.xml</b>.'
if ok:
if upload_point is not None:
full_path = os.path.abspath( os.path.join( repo_dir, upload_point, uploaded_file_filename ) )
else:
full_path = os.path.abspath( os.path.join( repo_dir, uploaded_file_filename ) )
# Move some version of the uploaded file to the load_point within the repository hierarchy.
if uploaded_file_filename in [ suc.REPOSITORY_DEPENDENCY_DEFINITION_FILENAME ]:
# Inspect the contents of the file to see if changeset_revision values are missing and if so, set them appropriately.
altered, root_elem, error_message = commit_util.handle_repository_dependencies_definition( trans,
uploaded_file_name,
unpopulate=False )
if error_message:
ok = False
message = error_message
status = 'error'
elif altered:
tmp_filename = xml_util.create_and_write_tmp_file( root_elem )
shutil.move( tmp_filename, full_path )
else:
shutil.move( uploaded_file_name, full_path )
elif uploaded_file_filename in [ suc.TOOL_DEPENDENCY_DEFINITION_FILENAME ]:
# Inspect the contents of the file to see if it defines a complex repository dependency definition whose changeset_revision values
# are missing and if so, set them appropriately.
altered, root_elem, error_message = commit_util.handle_tool_dependencies_definition( trans, uploaded_file_name )
if error_message:
ok = False
message = error_message
status = 'error'
if ok:
if altered:
tmp_filename = xml_util.create_and_write_tmp_file( root_elem )
shutil.move( tmp_filename, full_path )
else:
shutil.move( uploaded_file_name, full_path )
else:
shutil.move( uploaded_file_name, full_path )
if ok:
# See if any admin users have chosen to receive email alerts when a repository is updated. If so, check every uploaded file to ensure
# content is appropriate.
check_contents = commit_util.check_file_contents_for_email_alerts( trans )
if check_contents and os.path.isfile( full_path ):
content_alert_str = commit_util.check_file_content_for_html_and_images( full_path )
else:
content_alert_str = ''
commands.add( repo.ui, repo, full_path )
# Convert from unicode to prevent "TypeError: array item must be char"
full_path = full_path.encode( 'ascii', 'replace' )
commands.commit( repo.ui, repo, full_path, user=trans.user.username, message=commit_message )
if full_path.endswith( 'tool_data_table_conf.xml.sample' ):
# Handle the special case where a tool_data_table_conf.xml.sample file is being uploaded by parsing the file and adding new entries
# to the in-memory trans.app.tool_data_tables dictionary.
error, error_message = tool_util.handle_sample_tool_data_table_conf_file( trans.app, full_path )
if error:
message = '%s<br/>%s' % ( message, error_message )
# See if the content of the change set was valid.
admin_only = len( repository.downloadable_revisions ) != 1
suc.handle_email_alerts( trans,
repository,
content_alert_str=content_alert_str,
new_repo_alert=new_repo_alert,
admin_only=admin_only )
if ok:
# Update the repository files for browsing.
suc.update_repository( repo )
# Get the new repository tip.
if tip == repository.tip( trans.app ):
message = 'No changes to repository. '
status = 'warning'
else:
if ( isgzip or isbz2 ) and uncompress_file:
uncompress_str = ' uncompressed and '
else:
uncompress_str = ' '
if uploaded_directory:
source_type = "repository"
source = url
else:
source_type = "file"
source = uploaded_file_filename
message = "The %s <b>%s</b> has been successfully%suploaded to the repository. " % ( source_type, source, uncompress_str )
if istar and ( undesirable_dirs_removed or undesirable_files_removed ):
items_removed = undesirable_dirs_removed + undesirable_files_removed
message += " %d undesirable items (.hg .svn .git directories, .DS_Store, hgrc files, etc) were removed from the archive. " % items_removed
if istar and remove_repo_files_not_in_tar and files_to_remove:
if upload_point is not None:
message += " %d files were removed from the repository relative to the selected upload point '%s'. " % ( len( files_to_remove ), upload_point )
else:
message += " %d files were removed from the repository root. " % len( files_to_remove )
kwd[ 'message' ] = message
metadata_util.set_repository_metadata_due_to_new_tip( trans, repository, content_alert_str=content_alert_str, **kwd )
if repository.metadata_revisions:
# A repository's metadata revisions are order descending by update_time, so the zeroth revision will be the tip just after an upload.
metadata_dict = repository.metadata_revisions[0].metadata
else:
metadata_dict = {}
if str( repository.type ) != rt_util.TOOL_DEPENDENCY_DEFINITION:
change_repository_type_message = tool_dependency_util.generate_message_for_repository_type_change( trans, repository )
if change_repository_type_message:
message += change_repository_type_message
status = 'warning'
else:
# Provide a warning message if a tool_dependencies.xml file is provided, but tool dependencies weren't loaded due to a requirement tag mismatch
# or some other problem. Tool dependency definitions can define orphan tool dependencies (no relationship to any tools contained in the repository),
# so warning messages are important because orphans are always valid. The repository owner must be warned in case they did not intend to define an
# orphan dependency, but simply provided incorrect information (tool shed, name owner, changeset_revision) for the definition.
orphan_message = tool_dependency_util.generate_message_for_orphan_tool_dependencies( trans, repository, metadata_dict )
if orphan_message:
message += orphan_message
status = 'warning'
# Handle messaging for invalid tool dependencies.
invalid_tool_dependencies_message = tool_dependency_util.generate_message_for_invalid_tool_dependencies( metadata_dict )
if invalid_tool_dependencies_message:
message += invalid_tool_dependencies_message
status = 'error'
# Handle messaging for invalid repository dependencies.
invalid_repository_dependencies_message = repository_dependency_util.generate_message_for_invalid_repository_dependencies( metadata_dict )
if invalid_repository_dependencies_message:
message += invalid_repository_dependencies_message
status = 'error'
# Reset the tool_data_tables by loading the empty tool_data_table_conf.xml file.
tool_util.reset_tool_data_tables( trans.app )
if uploaded_directory:
suc.remove_dir( uploaded_directory )
trans.response.send_redirect( web.url_for( controller='repository',
action='browse_repository',
id=repository_id,
commit_message='Deleted selected files',
message=message,
status=status ) )
else:
if uploaded_directory:
suc.remove_dir( uploaded_directory )
status = 'error'
# Reset the tool_data_tables by loading the empty tool_data_table_conf.xml file.
tool_util.reset_tool_data_tables( trans.app )
message += orphan_message
status = 'warning'
# Handle messaging for invalid tool dependencies.
invalid_tool_dependencies_message = tool_dependency_util.generate_message_for_invalid_tool_dependencies( metadata_dict )
if invalid_tool_dependencies_message:
message += invalid_tool_dependencies_message
status = 'error'
# Handle messaging for invalid repository dependencies.
invalid_repository_dependencies_message = repository_dependency_util.generate_message_for_invalid_repository_dependencies( metadata_dict )
if invalid_repository_dependencies_message:
message += invalid_repository_dependencies_message
status = 'error'
# Reset the tool_data_tables by loading the empty tool_data_table_conf.xml file.
tool_util.reset_tool_data_tables( trans.app )
if uploaded_directory:
suc.remove_dir( uploaded_directory )
trans.response.send_redirect( web.url_for( controller='repository',
action='browse_repository',
id=repository_id,
commit_message='Deleted selected files',
message=message,
status=status ) )
else:
if uploaded_directory:
suc.remove_dir( uploaded_directory )
status = 'error'
# Reset the tool_data_tables by loading the empty tool_data_table_conf.xml file.
tool_util.reset_tool_data_tables( trans.app )
selected_categories = [ trans.security.decode_id( id ) for id in category_ids ]
return trans.fill_template( '/webapps/tool_shed/repository/upload.mako',
repository=repository,
# The manifest.xml file has already been validated, so no error_message should be returned here.
repository_info_dicts, error_message = import_util.get_repository_info_from_manifest( manifest_file_path )
# Determine the status for each exported repository archive contained within the capsule.
repository_status_info_dicts = import_util.get_repository_status_from_tool_shed( trans, repository_info_dicts )
# Generate a list of repository name / import results message tuples for display after the capsule is imported.
import_results_tups = []
# Only create repositories that do not yet exist and that the current user is authorized to create. The
# status will be None for repositories that fall into the intersection of these 2 categories.
for repository_status_info_dict in repository_status_info_dicts:
# Add the capsule_file_name and encoded_file_path to the repository_status_info_dict.
repository_status_info_dict[ 'capsule_file_name' ] = capsule_file_name
repository_status_info_dict[ 'encoded_file_path' ] = encoded_file_path
import_results_tups = repository_maintenance_util.create_repository_and_import_archive( trans,
repository_status_info_dict,
import_results_tups )
suc.remove_dir( file_path )
# NOTE: the order of installation is defined in import_results_tups, but order will be lost when transferred to return_dict.
return_dict = {}
for import_results_tup in import_results_tups:
name_owner, message = import_results_tup
name, owner = name_owner
key = 'Archive of repository "%s" owned by "%s"' % ( str( name ), str( owner ) )
val = message.replace( '<b>', '"' ).replace( '</b>', '"' )
return_dict[ key ] = val
return return_dict
@web.expose_api_anonymous
def index( self, trans, deleted=False, **kwd ):
"""
GET /api/repositories
Displays a collection (list) of repositories.
