def show( self, trans, id, **kwd ):
"""
GET /api/repositories/{encoded_repository_id}
Returns information about a repository in the Tool Shed.
Example URL: http://localhost:9009/api/repositories/f9cad7b01a472135
:param id: the encoded id of the Repository object
:type id: encoded str
:returns: detailed repository information
:rtype: dict
:raises: ObjectNotFound, MalformedId
"""
try:
trans.security.decode_id( id )
except Exception:
raise MalformedId( 'The given id is invalid.' )
repository = suc.get_repository_in_tool_shed( trans.app, id )
if repository is None:
raise ObjectNotFound( 'Unable to locate repository for the given id.' )
repository_dict = repository.to_dict( view='element',
value_mapper=self.__get_value_mapper( trans ) )
# TODO the following property would be better suited in the to_dict method
repository_dict[ 'category_ids' ] = \
[ trans.security.encode_id( x.category.id ) for x in repository.categories ]
return repository_dict
def manage_repository_reviews(self, trans, mine=False, **kwd):
# The value of the received id is the encoded repository id.
message = escape(kwd.get('message', ''))
status = kwd.get('status', 'done')
repository_id = kwd.get('id', None)
if repository_id:
repository = suc.get_repository_in_tool_shed(
trans.app, repository_id)
repo = hg_util.get_repo_for_repository(trans.app,
repository=repository)
metadata_revision_hashes = [
metadata_revision.changeset_revision
for metadata_revision in repository.metadata_revisions
]
reviewed_revision_hashes = [
review.changeset_revision for review in repository.reviews
]
reviews_dict = OrderedDict()
for changeset in hg_util.get_reversed_changelog_changesets(repo):
changeset_revision = str(repo[changeset])
if changeset_revision in metadata_revision_hashes or changeset_revision in reviewed_revision_hashes:
rev, changeset_revision_label = hg_util.get_rev_label_from_changeset_revision(
repo, changeset_revision)
if changeset_revision in reviewed_revision_hashes:
# Find the review for this changeset_revision
repository_reviews = \
review_util.get_reviews_by_repository_id_changeset_revision(trans.app,
repository_id,
changeset_revision)
# Determine if the current user can add a review to this revision.
can_add_review = trans.user not in [
repository_review.user
for repository_review in repository_reviews
]
repository_metadata = metadata_util.get_repository_metadata_by_changeset_revision(
trans.app, repository_id, changeset_revision)
if repository_metadata:
repository_metadata_reviews = util.listify(
repository_metadata.reviews)
else:
repository_metadata_reviews = []
else:
repository_reviews = []
repository_metadata_reviews = []
can_add_review = True
installable = changeset_revision in metadata_revision_hashes
revision_dict = dict(
changeset_revision_label=changeset_revision_label,
repository_reviews=repository_reviews,
repository_metadata_reviews=repository_metadata_reviews,
installable=installable,
can_add_review=can_add_review)
reviews_dict[changeset_revision] = revision_dict
return trans.fill_template(
'/webapps/tool_shed/repository_review/reviews_of_repository.mako',
repository=repository,
reviews_dict=reviews_dict,
mine=mine,
message=message,
status=status)
def show( self, trans, id, **kwd ):
"""
GET /api/repositories/{encoded_repository_id}
Returns information about a repository in the Tool Shed.
Example URL: http://localhost:9009/api/repositories/f9cad7b01a472135
:param id: the encoded id of the Repository object
:type id: encoded str
:returns: detailed repository information
:rtype: dict
:raises: ObjectNotFound, MalformedId
"""
try:
trans.security.decode_id( id )
except Exception:
raise MalformedId( 'The given id is invalid.' )
repository = suc.get_repository_in_tool_shed( trans.app, id )
if repository is None:
raise ObjectNotFound( 'Unable to locate repository for the given id.' )
repository_dict = repository.to_dict( view='element',
value_mapper=self.__get_value_mapper( trans ) )
# TODO the following property would be better suited in the to_dict method
repository_dict[ 'category_ids' ] = \
[ trans.security.encode_id( x.category.id ) for x in repository.categories ]
return repository_dict
def delete_repository( self, trans, **kwd ):
message = kwd.get( 'message', '' )
status = kwd.get( 'status', 'done' )
id = kwd.get( 'id', None )
if id:
# Deleting multiple items is currently not allowed (allow_multiple=False), so there will only be 1 id.
ids = util.listify( id )
count = 0
deleted_repositories = ""
for repository_id in ids:
repository = suc.get_repository_in_tool_shed( trans, repository_id )
if repository:
if not repository.deleted:
# Mark all installable repository_metadata records as not installable.
for repository_metadata in repository.downloadable_revisions:
repository_metadata.downloadable = False
trans.sa_session.add( repository_metadata )
repository.deleted = True
trans.sa_session.add( repository )
trans.sa_session.flush()
count += 1
deleted_repositories += " %s " % repository.name
if count:
message = "Deleted %d %s: %s" % ( count, inflector.cond_plural( len( ids ), "repository" ), deleted_repositories )
else:
message = "All selected repositories were already marked deleted."
else:
message = "No repository ids received for deleting."
status = 'error'
trans.response.send_redirect( web.url_for( controller='admin',
action='browse_repositories',
message=util.sanitize_text( message ),
status=status ) )
def get_repo_info_dict(app, user, repository_id, changeset_revision):
repository = suc.get_repository_in_tool_shed(app, repository_id)
repo = hg_util.get_repo_for_repository(app,
repository=repository,
repo_path=None,
create=False)
repository_clone_url = common_util.generate_clone_url_for_repository_in_tool_shed(
user, repository)
repository_metadata = suc.get_repository_metadata_by_changeset_revision(
app, repository_id, changeset_revision)
if not repository_metadata:
# The received changeset_revision is no longer installable, so get the next changeset_revision
# in the repository's changelog. This generally occurs only with repositories of type
# repository_suite_definition or tool_dependency_definition.
next_downloadable_changeset_revision = \
suc.get_next_downloadable_changeset_revision( repository, repo, changeset_revision )
if next_downloadable_changeset_revision:
repository_metadata = suc.get_repository_metadata_by_changeset_revision(
app, repository_id, next_downloadable_changeset_revision)
if repository_metadata:
# For now, we'll always assume that we'll get repository_metadata, but if we discover our assumption
# is not valid we'll have to enhance the callers to handle repository_metadata values of None in the
# returned repo_info_dict.
metadata = repository_metadata.metadata
if 'tools' in metadata:
includes_tools = True
else:
includes_tools = False
includes_tools_for_display_in_tool_panel = repository_metadata.includes_tools_for_display_in_tool_panel
repository_dependencies_dict = metadata.get('repository_dependencies',
{})
repository_dependencies = repository_dependencies_dict.get(
'repository_dependencies', [])
has_repository_dependencies, has_repository_dependencies_only_if_compiling_contained_td = \
suc.get_repository_dependency_types( repository_dependencies )
if 'tool_dependencies' in metadata:
includes_tool_dependencies = True
else:
includes_tool_dependencies = False
else:
# Here's where we may have to handle enhancements to the callers. See above comment.
includes_tools = False
has_repository_dependencies = False
has_repository_dependencies_only_if_compiling_contained_td = False
includes_tool_dependencies = False
includes_tools_for_display_in_tool_panel = False
ctx = hg_util.get_changectx_for_changeset(repo, changeset_revision)
repo_info_dict = create_repo_info_dict(
app=app,
repository_clone_url=repository_clone_url,
changeset_revision=changeset_revision,
ctx_rev=str(ctx.rev()),
repository_owner=repository.user.username,
repository_name=repository.name,
repository=repository,
repository_metadata=repository_metadata,
tool_dependencies=None,
repository_dependencies=None)
return repo_info_dict, includes_tools, includes_tool_dependencies, includes_tools_for_display_in_tool_panel, \
has_repository_dependencies, has_repository_dependencies_only_if_compiling_contained_td
def show(self, trans, id, **kwd):
"""
GET /api/repositories/{encoded_repository_id}
Returns information about a repository in the Tool Shed.
:param id: the encoded id of the Repository object
"""
# Example URL: http://localhost:9009/api/repositories/f9cad7b01a472135
try:
repository = suc.get_repository_in_tool_shed(trans, id)
repository_dict = repository.get_api_value(
view='element',
value_mapper=default_repository_value_mapper(
trans, repository))
repository_dict['url'] = web.url_for(controller='repositories',
action='show',
id=trans.security.encode_id(
repository.id))
return repository_dict
except Exception, e:
message = "Error in the Tool Shed repositories API in show: %s" % str(
e)
log.error(message, exc_info=True)
trans.response.status = 500
return message
def get_repo_info_dicts( trans, tool_shed_url, repository_id, changeset_revision ):
repository = suc.get_repository_in_tool_shed( trans, repository_id )
repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, repository_id, changeset_revision )
# Get a dictionary of all repositories upon which the contents of the current repository_metadata record depend.
repository_dependencies = \
repository_dependency_util.get_repository_dependencies_for_changeset_revision( trans=trans,
repository=repository,
repository_metadata=repository_metadata,
toolshed_base_url=str( web.url_for( '/', qualified=True ) ).rstrip( '/' ),
key_rd_dicts_to_be_processed=None,
all_repository_dependencies=None,
handled_key_rd_dicts=None )
repo = hg.repository( suc.get_configured_ui(), repository.repo_path( trans.app ) )
ctx = suc.get_changectx_for_changeset( repo, changeset_revision )
repo_info_dict = {}
# Cast unicode to string.
repo_info_dict[ str( repository.name ) ] = ( str( repository.description ),
suc.generate_clone_url_for_repository_in_tool_shed( trans, repository ),
str( changeset_revision ),
str( ctx.rev() ),
str( repository.user.username ),
repository_dependencies,
None )
all_required_repo_info_dict = common_install_util.get_required_repo_info_dicts( trans, tool_shed_url, [ repo_info_dict ] )
all_repo_info_dicts = all_required_repo_info_dict.get( 'all_repo_info_dicts', [] )
return all_repo_info_dicts
def manage_repository_reviews_of_revision(self, trans, **kwd):
# The value of the received id is the encoded repository id.
message = escape(kwd.get('message', ''))
status = kwd.get('status', 'done')
repository_id = kwd.get('id', None)
changeset_revision = kwd.get('changeset_revision', None)
repository = suc.get_repository_in_tool_shed(trans.app, repository_id)
repo = hg_util.get_repo_for_repository(trans.app,
repository=repository,
repo_path=None,
create=False)
installable = changeset_revision in [
metadata_revision.changeset_revision
for metadata_revision in repository.metadata_revisions
]
rev, changeset_revision_label = hg_util.get_rev_label_from_changeset_revision(
repo, changeset_revision)
reviews = review_util.get_reviews_by_repository_id_changeset_revision(
trans.app, repository_id, changeset_revision)
return trans.fill_template(
'/webapps/tool_shed/repository_review/reviews_of_changeset_revision.mako',
repository=repository,
changeset_revision=changeset_revision,
changeset_revision_label=changeset_revision_label,
reviews=reviews,
installable=installable,
message=message,
status=status)
def load_tool_from_changeset_revision(self, repository_id,
changeset_revision,
tool_config_filename):
"""
Return a loaded tool whose tool config file name (e.g., filtering.xml) is the value
of tool_config_filename. The value of changeset_revision is a valid (downloadable)
changeset revision. The tool config will be located in the repository manifest between
the received valid changeset revision and the first changeset revision in the repository,
searching backwards.
"""
original_tool_data_path = self.app.config.tool_data_path
repository = suc.get_repository_in_tool_shed(self.app, repository_id)
repo_files_dir = repository.repo_path(self.app)
repo = hg_util.get_repo_for_repository(self.app,
repository=None,
repo_path=repo_files_dir,
create=False)
message = ''
tool = None
can_use_disk_file = False
tool_config_filepath = suc.get_absolute_path_to_file_in_repository(
repo_files_dir, tool_config_filename)
work_dir = tempfile.mkdtemp(prefix="tmp-toolshed-ltfcr")
can_use_disk_file = self.can_use_tool_config_disk_file(
repository, repo, tool_config_filepath, changeset_revision)
if can_use_disk_file:
self.app.config.tool_data_path = work_dir
tool, valid, message, sample_files = \
self.handle_sample_files_and_load_tool_from_disk( repo_files_dir,
repository_id,
tool_config_filepath,
work_dir )
if tool is not None:
invalid_files_and_errors_tups = \
self.check_tool_input_params( repo_files_dir,
tool_config_filename,
tool,
sample_files )
if invalid_files_and_errors_tups:
message2 = tool_util.generate_message_for_invalid_tools(
self.app,
invalid_files_and_errors_tups,
repository,
metadata_dict=None,
as_html=True,
displaying_invalid_tool=True)
message = self.concat_messages(message, message2)
else:
tool, message, sample_files = \
self.handle_sample_files_and_load_tool_from_tmp_config( repo,
repository_id,
changeset_revision,
tool_config_filename,
work_dir )
basic_util.remove_dir(work_dir)
self.app.config.tool_data_path = original_tool_data_path
# Reset the tool_data_tables by loading the empty tool_data_table_conf.xml file.
self.tdtm.reset_tool_data_tables()
return repository, tool, message
def upload( self, trans, **kwd ):
message = escape( kwd.get( 'message', '' ) )
status = kwd.get( 'status', 'done' )
commit_message = escape( kwd.get( 'commit_message', 'Uploaded' ) )
category_ids = util.listify( kwd.get( 'category_id', '' ) )
categories = suc.get_categories( trans.app )
repository_id = kwd.get( 'repository_id', '' )
repository = suc.get_repository_in_tool_shed( trans.app, repository_id )
repo_dir = repository.repo_path( trans.app )
repo = hg_util.get_repo_for_repository( trans.app, repository=None, repo_path=repo_dir, create=False )
uncompress_file = util.string_as_bool( kwd.get( 'uncompress_file', 'true' ) )
remove_repo_files_not_in_tar = util.string_as_bool( kwd.get( 'remove_repo_files_not_in_tar', 'true' ) )
uploaded_file = None
upload_point = commit_util.get_upload_point( repository, **kwd )
tip = repository.tip( trans.app )
file_data = kwd.get( 'file_data', '' )
url = kwd.get( 'url', '' )
# Part of the upload process is sending email notification to those that have registered to
# receive them. One scenario occurs when the first change set is produced for the repository.
# See the suc.handle_email_alerts() method for the definition of the scenarios.
new_repo_alert = repository.is_new( trans.app )
uploaded_directory = None
if kwd.get( 'upload_button', False ):
if file_data == '' and url == '':
message = 'No files were entered on the upload form.'
status = 'error'
uploaded_file = None
elif url and url.startswith( 'hg' ):
# Use mercurial clone to fetch repository, contents will then be copied over.
uploaded_directory = tempfile.mkdtemp()
repo_url = 'http%s' % url[ len( 'hg' ): ]
repo_url = repo_url.encode( 'ascii', 'replace' )
try:
commands.clone( hg_util.get_configured_ui(), repo_url, uploaded_directory )
except Exception, e:
message = 'Error uploading via mercurial clone: %s' % basic_util.to_html_string( str( e ) )
status = 'error'
basic_util.remove_dir( uploaded_directory )
uploaded_directory = None
elif url:
valid_url = True
try:
stream = urllib.urlopen( url )
except Exception, e:
valid_url = False
message = 'Error uploading file via http: %s' % str( e )
status = 'error'
uploaded_file = None
if valid_url:
fd, uploaded_file_name = tempfile.mkstemp()
uploaded_file = open( uploaded_file_name, 'wb' )
while 1:
chunk = stream.read( util.CHUNK_SIZE )
if not chunk:
break
uploaded_file.write( chunk )
uploaded_file.flush()
uploaded_file_filename = url.split( '/' )[ -1 ]
isempty = os.path.getsize( os.path.abspath( uploaded_file_name ) ) == 0
def upload( self, trans, **kwd ):
message = escape( kwd.get( 'message', '' ) )
status = kwd.get( 'status', 'done' )
commit_message = escape( kwd.get( 'commit_message', 'Uploaded' ) )
repository_id = kwd.get( 'repository_id', '' )
repository = suc.get_repository_in_tool_shed( trans.app, repository_id )
repo_dir = repository.repo_path( trans.app )
repo = hg_util.get_repo_for_repository( trans.app, repository=None, repo_path=repo_dir, create=False )
uncompress_file = util.string_as_bool( kwd.get( 'uncompress_file', 'true' ) )
remove_repo_files_not_in_tar = util.string_as_bool( kwd.get( 'remove_repo_files_not_in_tar', 'true' ) )
uploaded_file = None
upload_point = commit_util.get_upload_point( repository, **kwd )
tip = repository.tip( trans.app )
file_data = kwd.get( 'file_data', '' )
url = kwd.get( 'url', '' )
# Part of the upload process is sending email notification to those that have registered to
# receive them. One scenario occurs when the first change set is produced for the repository.
# See the suc.handle_email_alerts() method for the definition of the scenarios.
new_repo_alert = repository.is_new( trans.app )
uploaded_directory = None
if kwd.get( 'upload_button', False ):
if file_data == '' and url == '':
message = 'No files were entered on the upload form.'
status = 'error'
uploaded_file = None
elif url and url.startswith( 'hg' ):
# Use mercurial clone to fetch repository, contents will then be copied over.
uploaded_directory = tempfile.mkdtemp()
repo_url = 'http%s' % url[ len( 'hg' ): ]
repo_url = repo_url.encode( 'ascii', 'replace' )
try:
commands.clone( hg_util.get_configured_ui(), repo_url, uploaded_directory )
except Exception, e:
message = 'Error uploading via mercurial clone: %s' % basic_util.to_html_string( str( e ) )
status = 'error'
basic_util.remove_dir( uploaded_directory )
uploaded_directory = None
elif url:
valid_url = True
try:
stream = urllib.urlopen( url )
except Exception, e:
valid_url = False
message = 'Error uploading file via http: %s' % str( e )
status = 'error'
uploaded_file = None
if valid_url:
fd, uploaded_file_name = tempfile.mkstemp()
uploaded_file = open( uploaded_file_name, 'wb' )
while 1:
chunk = stream.read( util.CHUNK_SIZE )
if not chunk:
break
uploaded_file.write( chunk )
uploaded_file.flush()
uploaded_file_filename = url.split( '/' )[ -1 ]
isempty = os.path.getsize( os.path.abspath( uploaded_file_name ) ) == 0
def view_or_manage_repository( self, trans, **kwd ):
repository = suc.get_repository_in_tool_shed( trans.app, kwd[ 'id' ] )
if trans.user_is_admin() or repository.user == trans.user:
return trans.response.send_redirect( web.url_for( controller='repository',
action='manage_repository',
**kwd ) )
else:
return trans.response.send_redirect( web.url_for( controller='repository',
action='view_repository',
**kwd ) )
def get_repo_info_dict( app, user, repository_id, changeset_revision ):
repository = suc.get_repository_in_tool_shed( app, repository_id )
repo = hg_util.get_repo_for_repository( app, repository=repository, repo_path=None, create=False )
repository_clone_url = common_util.generate_clone_url_for_repository_in_tool_shed( user, repository )
repository_metadata = suc.get_repository_metadata_by_changeset_revision( app,
repository_id,
changeset_revision )
if not repository_metadata:
# The received changeset_revision is no longer installable, so get the next changeset_revision
# in the repository's changelog. This generally occurs only with repositories of type
# repository_suite_definition or tool_dependency_definition.
next_downloadable_changeset_revision = \
suc.get_next_downloadable_changeset_revision( repository, repo, changeset_revision )
if next_downloadable_changeset_revision:
repository_metadata = suc.get_repository_metadata_by_changeset_revision( app,
repository_id,
next_downloadable_changeset_revision )
if repository_metadata:
# For now, we'll always assume that we'll get repository_metadata, but if we discover our assumption
# is not valid we'll have to enhance the callers to handle repository_metadata values of None in the
# returned repo_info_dict.
metadata = repository_metadata.metadata
if 'tools' in metadata:
includes_tools = True
else:
includes_tools = False
includes_tools_for_display_in_tool_panel = repository_metadata.includes_tools_for_display_in_tool_panel
repository_dependencies_dict = metadata.get( 'repository_dependencies', {} )
repository_dependencies = repository_dependencies_dict.get( 'repository_dependencies', [] )
has_repository_dependencies, has_repository_dependencies_only_if_compiling_contained_td = \
suc.get_repository_dependency_types( repository_dependencies )
if 'tool_dependencies' in metadata:
includes_tool_dependencies = True
else:
includes_tool_dependencies = False
else:
# Here's where we may have to handle enhancements to the callers. See above comment.
includes_tools = False
has_repository_dependencies = False
has_repository_dependencies_only_if_compiling_contained_td = False
includes_tool_dependencies = False
includes_tools_for_display_in_tool_panel = False
ctx = hg_util.get_changectx_for_changeset( repo, changeset_revision )
repo_info_dict = create_repo_info_dict( app=app,
repository_clone_url=repository_clone_url,
changeset_revision=changeset_revision,
ctx_rev=str( ctx.rev() ),
repository_owner=repository.user.username,
repository_name=repository.name,
repository=repository,
repository_metadata=repository_metadata,
tool_dependencies=None,
repository_dependencies=None )
return repo_info_dict, includes_tools, includes_tool_dependencies, includes_tools_for_display_in_tool_panel, \
has_repository_dependencies, has_repository_dependencies_only_if_compiling_contained_td
def load_tool_from_changeset_revision( self, repository_id, changeset_revision, tool_config_filename ):
"""
Return a loaded tool whose tool config file name (e.g., filtering.xml) is the value
of tool_config_filename. The value of changeset_revision is a valid (downloadable)
changeset revision. The tool config will be located in the repository manifest between
the received valid changeset revision and the first changeset revision in the repository,
searching backwards.
"""
original_tool_data_path = self.app.config.tool_data_path
repository = suc.get_repository_in_tool_shed( self.app, repository_id )
repo_files_dir = repository.repo_path( self.app )
repo = hg_util.get_repo_for_repository( self.app, repository=None, repo_path=repo_files_dir, create=False )
message = ''
tool = None
can_use_disk_file = False
tool_config_filepath = suc.get_absolute_path_to_file_in_repository( repo_files_dir, tool_config_filename )
work_dir = tempfile.mkdtemp( prefix="tmp-toolshed-ltfcr" )
can_use_disk_file = self.can_use_tool_config_disk_file( repository,
repo,
tool_config_filepath,
changeset_revision )
if can_use_disk_file:
self.app.config.tool_data_path = work_dir
tool, valid, message, sample_files = \
self.handle_sample_files_and_load_tool_from_disk( repo_files_dir,
repository_id,
tool_config_filepath,
work_dir )
if tool is not None:
invalid_files_and_errors_tups = \
self.check_tool_input_params( repo_files_dir,
tool_config_filename,
tool,
sample_files )
if invalid_files_and_errors_tups:
message2 = tool_util.generate_message_for_invalid_tools( self.app,
invalid_files_and_errors_tups,
repository,
metadata_dict=None,
as_html=True,
displaying_invalid_tool=True )
message = self.concat_messages( message, message2 )
else:
tool, message, sample_files = \
self.handle_sample_files_and_load_tool_from_tmp_config( repo,
repository_id,
changeset_revision,
tool_config_filename,
work_dir )
basic_util.remove_dir( work_dir )
self.app.config.tool_data_path = original_tool_data_path
# Reset the tool_data_tables by loading the empty tool_data_table_conf.xml file.
self.tdtm.reset_tool_data_tables()
return repository, tool, message
def undelete_repository(self, trans, **kwd):
message = escape(kwd.get('message', ''))
status = kwd.get('status', 'done')
id = kwd.get('id', None)
if id:
# Undeleting multiple items is currently not allowed (allow_multiple=False), so there will only be 1 id.
ids = util.listify(id)
count = 0
undeleted_repositories = ""
for repository_id in ids:
repository = suc.get_repository_in_tool_shed(
trans.app, repository_id)
if repository:
if repository.deleted:
# Inspect all repository_metadata records to determine those that are installable, and mark
# them accordingly.
for repository_metadata in repository.metadata_revisions:
metadata = repository_metadata.metadata
if metadata:
if metadata_util.is_downloadable(metadata):
repository_metadata.downloadable = True
trans.sa_session.add(repository_metadata)
# Mark the repository admin role as not deleted.
repository_admin_role = repository.admin_role
if repository_admin_role is not None:
repository_admin_role.deleted = False
trans.sa_session.add(repository_admin_role)
repository.deleted = False
trans.sa_session.add(repository)
trans.sa_session.flush()
if not repository.deprecated:
# Update the repository registry.
trans.app.repository_registry.add_entry(repository)
count += 1
undeleted_repositories += " %s" % repository.name
if count:
message = "Undeleted %d %s: %s" % (
count, inflector.cond_plural(
count, "repository"), undeleted_repositories)
else:
message = "No selected repositories were marked deleted, so they could not be undeleted."
else:
message = "No repository ids received for undeleting."
status = 'error'
trans.response.send_redirect(
web.url_for(controller='admin',
action='browse_repositories',
message=util.sanitize_text(message),
status='done'))
def undelete_repository(self, trans, **kwd):
message = escape(kwd.get("message", ""))
id = kwd.get("id", None)
if id:
# Undeleting multiple items is currently not allowed (allow_multiple=False), so there will only be 1 id.
ids = util.listify(id)
count = 0
undeleted_repositories = ""
for repository_id in ids:
repository = suc.get_repository_in_tool_shed(trans.app, repository_id)
if repository:
if repository.deleted:
# Inspect all repository_metadata records to determine those that are installable, and mark
# them accordingly.
for repository_metadata in repository.metadata_revisions:
metadata = repository_metadata.metadata
if metadata:
if metadata_util.is_downloadable(metadata):
repository_metadata.downloadable = True
trans.sa_session.add(repository_metadata)
# Mark the repository admin role as not deleted.
repository_admin_role = repository.admin_role
if repository_admin_role is not None:
repository_admin_role.deleted = False
trans.sa_session.add(repository_admin_role)
repository.deleted = False
trans.sa_session.add(repository)
trans.sa_session.flush()
if not repository.deprecated:
# Update the repository registry.
trans.app.repository_registry.add_entry(repository)
count += 1
undeleted_repositories += " %s" % repository.name
if count:
message = "Undeleted %d %s: %s" % (
count,
inflector.cond_plural(count, "repository"),
undeleted_repositories,
)
else:
message = "No selected repositories were marked deleted, so they could not be undeleted."
else:
message = "No repository ids received for undeleting."
trans.response.send_redirect(
web.url_for(
controller="admin", action="browse_repositories", message=util.sanitize_text(message), status="done"
)
)
def select_previous_review( self, trans, **kwd ):
# The value of the received id is the encoded repository id.
message = kwd.get( 'message', '' )
status = kwd.get( 'status', 'done' )
repository = suc.get_repository_in_tool_shed( trans, kwd[ 'id' ] )
changeset_revision = kwd.get( 'changeset_revision', None )
repo = hg.repository( suc.get_configured_ui(), repository.repo_path( trans.app ) )
previous_reviews_dict = review_util.get_previous_repository_reviews( trans, repository, changeset_revision )
rev, changeset_revision_label = suc.get_rev_label_from_changeset_revision( repo, changeset_revision )
return trans.fill_template( '/webapps/tool_shed/repository_review/select_previous_review.mako',
repository=repository,
changeset_revision=changeset_revision,
changeset_revision_label=changeset_revision_label,
previous_reviews_dict=previous_reviews_dict,
message=message,
status=status )
def manage_repository_reviews( self, trans, mine=False, **kwd ):
# The value of the received id is the encoded repository id.
message = escape( kwd.get( 'message', '' ) )
status = kwd.get( 'status', 'done' )
repository_id = kwd.get( 'id', None )
if repository_id:
repository = suc.get_repository_in_tool_shed( trans.app, repository_id )
repo = hg_util.get_repo_for_repository( trans.app, repository=repository, repo_path=None, create=False )
metadata_revision_hashes = [ metadata_revision.changeset_revision for metadata_revision in repository.metadata_revisions ]
reviewed_revision_hashes = [ review.changeset_revision for review in repository.reviews ]
reviews_dict = odict()
for changeset in hg_util.get_reversed_changelog_changesets( repo ):
ctx = repo.changectx( changeset )
changeset_revision = str( ctx )
if changeset_revision in metadata_revision_hashes or changeset_revision in reviewed_revision_hashes:
rev, changeset_revision_label = hg_util.get_rev_label_from_changeset_revision( repo, changeset_revision )
if changeset_revision in reviewed_revision_hashes:
# Find the review for this changeset_revision
repository_reviews = \
review_util.get_reviews_by_repository_id_changeset_revision( trans.app,
repository_id,
changeset_revision )
# Determine if the current user can add a review to this revision.
can_add_review = trans.user not in [ repository_review.user for repository_review in repository_reviews ]
repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, repository_id, changeset_revision )
if repository_metadata:
repository_metadata_reviews = util.listify( repository_metadata.reviews )
else:
repository_metadata_reviews = []
else:
repository_reviews = []
repository_metadata_reviews = []
can_add_review = True
installable = changeset_revision in metadata_revision_hashes
revision_dict = dict( changeset_revision_label=changeset_revision_label,
repository_reviews=repository_reviews,
repository_metadata_reviews=repository_metadata_reviews,
installable=installable,
can_add_review=can_add_review )
reviews_dict[ changeset_revision ] = revision_dict
return trans.fill_template( '/webapps/tool_shed/repository_review/reviews_of_repository.mako',
repository=repository,
reviews_dict=reviews_dict,
mine=mine,
message=message,
status=status )
def show(self, trans, id, **kwd):
"""
GET /api/repositories/{encoded_repository_id}
Returns information about a repository in the Tool Shed.
:param id: the encoded id of the Repository object
"""
# Example URL: http://localhost:9009/api/repositories/f9cad7b01a472135
repository = suc.get_repository_in_tool_shed(trans.app, id)
if repository is None:
log.debug("Unable to locate repository record for id %s." % (str(id)))
return {}
repository_dict = repository.to_dict(view="element", value_mapper=self.__get_value_mapper(trans))
repository_dict["url"] = web.url_for(
controller="repositories", action="show", id=trans.security.encode_id(repository.id)
)
return repository_dict
def delete_repository(self, trans, **kwd):
message = escape(kwd.get("message", ""))
status = kwd.get("status", "done")
id = kwd.get("id", None)
if id:
# Deleting multiple items is currently not allowed (allow_multiple=False), so there will only be 1 id.
ids = util.listify(id)
count = 0
deleted_repositories = ""
for repository_id in ids:
repository = suc.get_repository_in_tool_shed(trans.app, repository_id)
if repository:
if not repository.deleted:
# Mark all installable repository_metadata records as not installable.
for repository_metadata in repository.downloadable_revisions:
repository_metadata.downloadable = False
trans.sa_session.add(repository_metadata)
# Mark the repository admin role as deleted.
repository_admin_role = repository.admin_role
if repository_admin_role is not None:
repository_admin_role.deleted = True
trans.sa_session.add(repository_admin_role)
repository.deleted = True
trans.sa_session.add(repository)
trans.sa_session.flush()
# Update the repository registry.
trans.app.repository_registry.remove_entry(repository)
count += 1
deleted_repositories += " %s " % repository.name
if count:
message = "Deleted %d %s: %s" % (
count,
inflector.cond_plural(len(ids), "repository"),
escape(deleted_repositories),
)
else:
message = "All selected repositories were already marked deleted."
else:
message = "No repository ids received for deleting."
status = "error"
trans.response.send_redirect(
web.url_for(
controller="admin", action="browse_repositories", message=util.sanitize_text(message), status=status
)
)
def show( self, trans, id, **kwd ):
"""
GET /api/repositories/{encoded_repository_id}
Returns information about a repository in the Tool Shed.
:param id: the encoded id of the Repository object
"""
# Example URL: http://localhost:9009/api/repositories/f9cad7b01a472135
try:
repository = suc.get_repository_in_tool_shed( trans, id )
repository_dict = repository.get_api_value( view='element', value_mapper=default_repository_value_mapper( trans, repository ) )
repository_dict[ 'url' ] = web.url_for( controller='repositories',
action='show',
id=trans.security.encode_id( repository.id ) )
return repository_dict
except Exception, e:
message = "Error in the Tool Shed repositories API in show: %s" % str( e )
log.error( message, exc_info=True )
trans.response.status = 500
return message
def show(self, trans, id, **kwd):
"""
GET /api/repositories/{encoded_repository_id}
Returns information about a repository in the Tool Shed.
:param id: the encoded id of the Repository object
"""
# Example URL: http://localhost:9009/api/repositories/f9cad7b01a472135
repository = suc.get_repository_in_tool_shed(trans.app, id)
if repository is None:
log.debug("Unable to locate repository record for id %s." %
(str(id)))
return {}
repository_dict = repository.to_dict(
view='element', value_mapper=self.__get_value_mapper(trans))
repository_dict['url'] = web.url_for(controller='repositories',
action='show',
id=trans.security.encode_id(
repository.id))
return repository_dict
def manage_repository_reviews_of_revision( self, trans, **kwd ):
# The value of the received id is the encoded repository id.
message = kwd.get( 'message', '' )
status = kwd.get( 'status', 'done' )
repository_id = kwd.get( 'id', None )
changeset_revision = kwd.get( 'changeset_revision', None )
repository = suc.get_repository_in_tool_shed( trans, repository_id )
repo_dir = repository.repo_path( trans.app )
repo = hg.repository( suc.get_configured_ui(), repo_dir )
installable = changeset_revision in [ metadata_revision.changeset_revision for metadata_revision in repository.metadata_revisions ]
rev, changeset_revision_label = suc.get_rev_label_from_changeset_revision( repo, changeset_revision )
reviews = review_util.get_reviews_by_repository_id_changeset_revision( trans, repository_id, changeset_revision )
return trans.fill_template( '/webapps/tool_shed/repository_review/reviews_of_changeset_revision.mako',
repository=repository,
changeset_revision=changeset_revision,
changeset_revision_label=changeset_revision_label,
reviews=reviews,
installable=installable,
message=message,
status=status )
def undelete_repository( self, trans, **kwd ):
message = kwd.get( 'message', '' )
status = kwd.get( 'status', 'done' )
id = kwd.get( 'id', None )
if id:
# Undeleting multiple items is currently not allowed (allow_multiple=False), so there will only be 1 id.
ids = util.listify( id )
count = 0
undeleted_repositories = ""
for repository_id in ids:
repository = suc.get_repository_in_tool_shed( trans, repository_id )
if repository:
if repository.deleted:
# Inspect all repository_metadata records to determine those that are installable, and mark them accordingly.
for repository_metadata in repository.metadata_revisions:
metadata = repository_metadata.metadata
if metadata:
if metadata_util.is_downloadable( metadata ):
repository_metadata.downloadable = True
trans.sa_session.add( repository_metadata )
repository.deleted = False
trans.sa_session.add( repository )
trans.sa_session.flush()
count += 1
undeleted_repositories += " %s" % repository.name
if count:
message = "Undeleted %d %s: %s" % ( count, inflector.cond_plural( count, "repository" ), undeleted_repositories )
else:
message = "No selected repositories were marked deleted, so they could not be undeleted."
else:
message = "No repository ids received for undeleting."
status = 'error'
trans.response.send_redirect( web.url_for( controller='admin',
action='browse_repositories',
message=util.sanitize_text( message ),
status='done' ) )
def browse_repositories( self, trans, **kwd ):
# We add parameters to the keyword dict in this method in order to rename the param
# with an "f-" prefix, simulating filtering by clicking a search link. We have
# to take this approach because the "-" character is illegal in HTTP requests.
if 'operation' in kwd:
operation = kwd[ 'operation' ].lower()
if operation == "view_or_manage_repository":
return trans.response.send_redirect( web.url_for( controller='repository',
action='browse_repositories',
**kwd ) )
elif operation == "edit_repository":
return trans.response.send_redirect( web.url_for( controller='repository',
action='edit_repository',
**kwd ) )
elif operation == "repositories_by_user":
# Eliminate the current filters if any exist.
for k, v in kwd.items():
if k.startswith( 'f-' ):
del kwd[ k ]
if 'user_id' in kwd:
user = suc.get_user( trans.app, kwd[ 'user_id' ] )
kwd[ 'f-email' ] = user.email
del kwd[ 'user_id' ]
else:
# The received id is the repository id, so we need to get the id of the user
# that uploaded the repository.
repository_id = kwd.get( 'id', None )
repository = suc.get_repository_in_tool_shed( trans.app, repository_id )
kwd[ 'f-email' ] = repository.user.email
elif operation == "repositories_by_category":
# Eliminate the current filters if any exist.
for k, v in kwd.items():
if k.startswith( 'f-' ):
del kwd[ k ]
category_id = kwd.get( 'id', None )
category = suc.get_category( trans.app, category_id )
kwd[ 'f-Category.name' ] = category.name
elif operation == "receive email alerts":
if kwd[ 'id' ]:
kwd[ 'caller' ] = 'browse_repositories'
return trans.response.send_redirect( web.url_for( controller='repository',
action='set_email_alerts',
**kwd ) )
else:
del kwd[ 'operation' ]
elif operation == 'delete':
return self.delete_repository( trans, **kwd )
elif operation == "undelete":
return self.undelete_repository( trans, **kwd )
# The changeset_revision_select_field in the RepositoryGrid performs a refresh_on_change
# which sends in request parameters like changeset_revison_1, changeset_revision_2, etc. One
# of the many select fields on the grid performed the refresh_on_change, so we loop through
# all of the received values to see which value is not the repository tip. If we find it, we
# know the refresh_on_change occurred, and we have the necessary repository id and change set
# revision to pass on.
for k, v in kwd.items():
changeset_revision_str = 'changeset_revision_'
if k.startswith( changeset_revision_str ):
repository_id = trans.security.encode_id( int( k.lstrip( changeset_revision_str ) ) )
repository = suc.get_repository_in_tool_shed( tran.apps, repository_id )
if repository.tip( trans.app ) != v:
return trans.response.send_redirect( web.url_for( controller='repository',
action='browse_repositories',
operation='view_or_manage_repository',
id=trans.security.encode_id( repository.id ),
changeset_revision=v ) )
# Render the list view
return self.repository_grid( trans, **kwd )
def browse_repositories( self, trans, **kwd ):
# We add params to the keyword dict in this method in order to rename the param
# with an "f-" prefix, simulating filtering by clicking a search link. We have
# to take this approach because the "-" character is illegal in HTTP requests.
if 'operation' in kwd:
operation = kwd[ 'operation' ].lower()
if operation == "view_or_manage_repository":
return trans.response.send_redirect( web.url_for( controller='repository',
action='browse_repositories',
**kwd ) )
elif operation == "edit_repository":
return trans.response.send_redirect( web.url_for( controller='repository',
action='edit_repository',
**kwd ) )
elif operation == "repositories_by_user":
# Eliminate the current filters if any exist.
for k, v in kwd.items():
if k.startswith( 'f-' ):
del kwd[ k ]
if 'user_id' in kwd:
user = suc.get_user( trans, kwd[ 'user_id' ] )
kwd[ 'f-email' ] = user.email
del kwd[ 'user_id' ]
else:
# The received id is the repository id, so we need to get the id of the user
# that uploaded the repository.
repository_id = kwd.get( 'id', None )
repository = suc.get_repository_in_tool_shed( trans, repository_id )
kwd[ 'f-email' ] = repository.user.email
elif operation == "repositories_by_category":
# Eliminate the current filters if any exist.
for k, v in kwd.items():
if k.startswith( 'f-' ):
del kwd[ k ]
category_id = kwd.get( 'id', None )
category = suc.get_category( trans, category_id )
kwd[ 'f-Category.name' ] = category.name
elif operation == "receive email alerts":
if kwd[ 'id' ]:
kwd[ 'caller' ] = 'browse_repositories'
return trans.response.send_redirect( web.url_for( controller='repository',
action='set_email_alerts',
**kwd ) )
else:
del kwd[ 'operation' ]
elif operation == 'delete':
return self.delete_repository( trans, **kwd )
elif operation == "undelete":
return self.undelete_repository( trans, **kwd )
# The changeset_revision_select_field in the RepositoryGrid performs a refresh_on_change
# which sends in request parameters like changeset_revison_1, changeset_revision_2, etc. One
# of the many select fields on the grid performed the refresh_on_change, so we loop through
# all of the received values to see which value is not the repository tip. If we find it, we
# know the refresh_on_change occurred, and we have the necessary repository id and change set
# revision to pass on.
for k, v in kwd.items():
changeset_revision_str = 'changeset_revision_'
if k.startswith( changeset_revision_str ):
repository_id = trans.security.encode_id( int( k.lstrip( changeset_revision_str ) ) )
repository = suc.get_repository_in_tool_shed( trans, repository_id )
if repository.tip( trans.app ) != v:
return trans.response.send_redirect( web.url_for( controller='repository',
action='browse_repositories',
operation='view_or_manage_repository',
id=trans.security.encode_id( repository.id ),
changeset_revision=v ) )
# Render the list view
return self.repository_grid( trans, **kwd )
def upload( self, trans, **kwd ):
message = kwd.get( 'message', '' )
status = kwd.get( 'status', 'done' )
commit_message = kwd.get( 'commit_message', 'Uploaded' )
category_ids = util.listify( kwd.get( 'category_id', '' ) )
categories = suc.get_categories( trans )
repository_id = kwd.get( 'repository_id', '' )
repository = suc.get_repository_in_tool_shed( trans, repository_id )
repo_dir = repository.repo_path( trans.app )
repo = hg.repository( suc.get_configured_ui(), repo_dir )
uncompress_file = util.string_as_bool( kwd.get( 'uncompress_file', 'true' ) )
remove_repo_files_not_in_tar = util.string_as_bool( kwd.get( 'remove_repo_files_not_in_tar', 'true' ) )
uploaded_file = None
upload_point = commit_util.get_upload_point( repository, **kwd )
tip = repository.tip( trans.app )
file_data = kwd.get( 'file_data', '' )
url = kwd.get( 'url', '' )
# Part of the upload process is sending email notification to those that have registered to
# receive them. One scenario occurs when the first change set is produced for the repository.
# See the suc.handle_email_alerts() method for the definition of the scenarios.
new_repo_alert = repository.is_new( trans.app )
uploaded_directory = None
if kwd.get( 'upload_button', False ):
if file_data == '' and url == '':
message = 'No files were entered on the upload form.'
status = 'error'
uploaded_file = None
elif url and url.startswith( 'hg' ):
# Use mercurial clone to fetch repository, contents will then be copied over.
uploaded_directory = tempfile.mkdtemp()
repo_url = 'http%s' % url[ len( 'hg' ): ]
repo_url = repo_url.encode( 'ascii', 'replace' )
commands.clone( suc.get_configured_ui(), repo_url, uploaded_directory )
elif url:
valid_url = True
try:
stream = urllib.urlopen( url )
except Exception, e:
valid_url = False
message = 'Error uploading file via http: %s' % str( e )
status = 'error'
uploaded_file = None
if valid_url:
fd, uploaded_file_name = tempfile.mkstemp()
uploaded_file = open( uploaded_file_name, 'wb' )
while 1:
chunk = stream.read( util.CHUNK_SIZE )
if not chunk:
break
uploaded_file.write( chunk )
uploaded_file.flush()
uploaded_file_filename = url.split( '/' )[ -1 ]
isempty = os.path.getsize( os.path.abspath( uploaded_file_name ) ) == 0
elif file_data not in ( '', None ):
uploaded_file = file_data.file
uploaded_file_name = uploaded_file.name
uploaded_file_filename = os.path.split( file_data.filename )[ -1 ]
isempty = os.path.getsize( os.path.abspath( uploaded_file_name ) ) == 0
if uploaded_file or uploaded_directory:
ok = True
isgzip = False
isbz2 = False
if uploaded_file:
if uncompress_file:
isgzip = checkers.is_gzip( uploaded_file_name )
if not isgzip:
isbz2 = checkers.is_bz2( uploaded_file_name )
if isempty:
tar = None
istar = False
else:
# Determine what we have - a single file or an archive
try:
if ( isgzip or isbz2 ) and uncompress_file:
# Open for reading with transparent compression.
tar = tarfile.open( uploaded_file_name, 'r:*' )
else:
tar = tarfile.open( uploaded_file_name )
istar = True
except tarfile.ReadError, e:
tar = None
istar = False
else:
# Uploaded directory
istar = False
if istar:
ok, message, files_to_remove, content_alert_str, undesirable_dirs_removed, undesirable_files_removed = \
self.upload_tar( trans, repository, tar, uploaded_file, upload_point, remove_repo_files_not_in_tar, commit_message, new_repo_alert )
elif uploaded_directory:
ok, message, files_to_remove, content_alert_str, undesirable_dirs_removed, undesirable_files_removed = \
self.upload_directory( trans, repository, uploaded_directory, upload_point, remove_repo_files_not_in_tar, commit_message, new_repo_alert )
else:
if ( isgzip or isbz2 ) and uncompress_file:
uploaded_file_filename = commit_util.uncompress( repository, uploaded_file_name, uploaded_file_filename, isgzip=isgzip, isbz2=isbz2 )
if repository.type == rt_util.TOOL_DEPENDENCY_DEFINITION and uploaded_file_filename != suc.TOOL_DEPENDENCY_DEFINITION_FILENAME:
ok = False
message = 'Repositories of type <b>Tool dependency definition</b> can only contain a single file named <b>tool_dependencies.xml</b>.'
if ok:
if upload_point is not None:
full_path = os.path.abspath( os.path.join( repo_dir, upload_point, uploaded_file_filename ) )
else:
full_path = os.path.abspath( os.path.join( repo_dir, uploaded_file_filename ) )
# Move some version of the uploaded file to the load_point within the repository hierarchy.
if uploaded_file_filename in [ suc.REPOSITORY_DEPENDENCY_DEFINITION_FILENAME ]:
# Inspect the contents of the file to see if changeset_revision values are missing and if so, set them appropriately.
altered, root_elem, error_message = commit_util.handle_repository_dependencies_definition( trans,
uploaded_file_name,
unpopulate=False )
if error_message:
ok = False
message = error_message
status = 'error'
elif altered:
tmp_filename = xml_util.create_and_write_tmp_file( root_elem )
shutil.move( tmp_filename, full_path )
else:
shutil.move( uploaded_file_name, full_path )
elif uploaded_file_filename in [ suc.TOOL_DEPENDENCY_DEFINITION_FILENAME ]:
# Inspect the contents of the file to see if it defines a complex repository dependency definition whose changeset_revision values
# are missing and if so, set them appropriately.
altered, root_elem, error_message = commit_util.handle_tool_dependencies_definition( trans, uploaded_file_name )
if error_message:
ok = False
message = error_message
status = 'error'
if ok:
if altered:
tmp_filename = xml_util.create_and_write_tmp_file( root_elem )
shutil.move( tmp_filename, full_path )
else:
shutil.move( uploaded_file_name, full_path )
else:
shutil.move( uploaded_file_name, full_path )
if ok:
# See if any admin users have chosen to receive email alerts when a repository is updated. If so, check every uploaded file to ensure
# content is appropriate.
check_contents = commit_util.check_file_contents_for_email_alerts( trans )
if check_contents and os.path.isfile( full_path ):
content_alert_str = commit_util.check_file_content_for_html_and_images( full_path )
else:
content_alert_str = ''
commands.add( repo.ui, repo, full_path )
# Convert from unicode to prevent "TypeError: array item must be char"
full_path = full_path.encode( 'ascii', 'replace' )
commands.commit( repo.ui, repo, full_path, user=trans.user.username, message=commit_message )
if full_path.endswith( 'tool_data_table_conf.xml.sample' ):
# Handle the special case where a tool_data_table_conf.xml.sample file is being uploaded by parsing the file and adding new entries
# to the in-memory trans.app.tool_data_tables dictionary.
error, error_message = tool_util.handle_sample_tool_data_table_conf_file( trans.app, full_path )
if error:
message = '%s<br/>%s' % ( message, error_message )
# See if the content of the change set was valid.
admin_only = len( repository.downloadable_revisions ) != 1
suc.handle_email_alerts( trans,
repository,
content_alert_str=content_alert_str,
new_repo_alert=new_repo_alert,
admin_only=admin_only )
if ok:
# Update the repository files for browsing.
suc.update_repository( repo )
# Get the new repository tip.
if tip == repository.tip( trans.app ):
message = 'No changes to repository. '
status = 'warning'
else:
if ( isgzip or isbz2 ) and uncompress_file:
uncompress_str = ' uncompressed and '
else:
uncompress_str = ' '
if uploaded_directory:
source_type = "repository"
source = url
else:
source_type = "file"
source = uploaded_file_filename
message = "The %s <b>%s</b> has been successfully%suploaded to the repository. " % ( source_type, source, uncompress_str )
if istar and ( undesirable_dirs_removed or undesirable_files_removed ):
items_removed = undesirable_dirs_removed + undesirable_files_removed
message += " %d undesirable items (.hg .svn .git directories, .DS_Store, hgrc files, etc) were removed from the archive. " % items_removed
if istar and remove_repo_files_not_in_tar and files_to_remove:
if upload_point is not None:
message += " %d files were removed from the repository relative to the selected upload point '%s'. " % ( len( files_to_remove ), upload_point )
else:
message += " %d files were removed from the repository root. " % len( files_to_remove )
kwd[ 'message' ] = message
metadata_util.set_repository_metadata_due_to_new_tip( trans, repository, content_alert_str=content_alert_str, **kwd )
if repository.metadata_revisions:
# A repository's metadata revisions are order descending by update_time, so the zeroth revision will be the tip just after an upload.
metadata_dict = repository.metadata_revisions[0].metadata
else:
metadata_dict = {}
if str( repository.type ) != rt_util.TOOL_DEPENDENCY_DEFINITION:
change_repository_type_message = tool_dependency_util.generate_message_for_repository_type_change( trans, repository )
if change_repository_type_message:
message += change_repository_type_message
status = 'warning'
else:
# Provide a warning message if a tool_dependencies.xml file is provided, but tool dependencies weren't loaded due to a requirement tag mismatch
# or some other problem. Tool dependency definitions can define orphan tool dependencies (no relationship to any tools contained in the repository),
# so warning messages are important because orphans are always valid. The repository owner must be warned in case they did not intend to define an
# orphan dependency, but simply provided incorrect information (tool shed, name owner, changeset_revision) for the definition.
orphan_message = tool_dependency_util.generate_message_for_orphan_tool_dependencies( trans, repository, metadata_dict )
if orphan_message:
message += orphan_message
status = 'warning'
# Handle messaging for invalid tool dependencies.
invalid_tool_dependencies_message = tool_dependency_util.generate_message_for_invalid_tool_dependencies( metadata_dict )
if invalid_tool_dependencies_message:
message += invalid_tool_dependencies_message
status = 'error'
# Handle messaging for invalid repository dependencies.
invalid_repository_dependencies_message = repository_dependency_util.generate_message_for_invalid_repository_dependencies( metadata_dict )
if invalid_repository_dependencies_message:
message += invalid_repository_dependencies_message
status = 'error'
# Reset the tool_data_tables by loading the empty tool_data_table_conf.xml file.
tool_util.reset_tool_data_tables( trans.app )
if uploaded_directory:
suc.remove_dir( uploaded_directory )
trans.response.send_redirect( web.url_for( controller='repository',
action='browse_repository',
id=repository_id,
commit_message='Deleted selected files',
message=message,
status=status ) )
else:
if uploaded_directory:
suc.remove_dir( uploaded_directory )
status = 'error'
# Reset the tool_data_tables by loading the empty tool_data_table_conf.xml file.
tool_util.reset_tool_data_tables( trans.app )
def create_review( self, trans, **kwd ):
# The value of the received id is the encoded repository id.
message = escape( kwd.get( 'message', '' ) )
status = kwd.get( 'status', 'done' )
repository_id = kwd.get( 'id', None )
changeset_revision = kwd.get( 'changeset_revision', None )
previous_review_id = kwd.get( 'previous_review_id', None )
create_without_copying = 'create_without_copying' in kwd
if repository_id:
if changeset_revision:
# Make sure there is not already a review of the revision by the user.
repository = suc.get_repository_in_tool_shed( trans.app, repository_id )
if review_util.get_review_by_repository_id_changeset_revision_user_id( app=trans.app,
repository_id=repository_id,
changeset_revision=changeset_revision,
user_id=trans.security.encode_id( trans.user.id ) ):
message = "You have already created a review for revision <b>%s</b> of repository <b>%s</b>." % ( changeset_revision, escape( repository.name ) )
status = "error"
else:
# See if there are any reviews for previous changeset revisions that the user can copy.
if not create_without_copying and \
not previous_review_id and \
review_util.has_previous_repository_reviews( trans.app, repository, changeset_revision ):
return trans.response.send_redirect( web.url_for( controller='repository_review',
action='select_previous_review',
**kwd ) )
# A review can be initially performed only on an installable revision of a repository, so make sure we have metadata associated
# with the received changeset_revision.
repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app, repository_id, changeset_revision )
if repository_metadata:
metadata = repository_metadata.metadata
if metadata:
review = trans.app.model.RepositoryReview( repository_id=repository_metadata.repository_id,
changeset_revision=changeset_revision,
user_id=trans.user.id,
rating=None,
deleted=False )
trans.sa_session.add( review )
trans.sa_session.flush()
if previous_review_id:
review_to_copy = review_util.get_review( trans.app, previous_review_id )
self.copy_review( trans, review_to_copy, review )
review_id = trans.security.encode_id( review.id )
message = "Begin your review of revision <b>%s</b> of repository <b>%s</b>." \
% ( changeset_revision, repository.name )
status = 'done'
trans.response.send_redirect( web.url_for( controller='repository_review',
action='edit_review',
id=review_id,
message=message,
status=status ) )
else:
message = "A new review cannot be created for revision <b>%s</b> of repository <b>%s</b>. Select a valid revision and try again." \
% ( changeset_revision, escape( repository.name ) )
kwd[ 'message' ] = message
kwd[ 'status' ] = 'error'
else:
return trans.response.send_redirect( web.url_for( controller='repository_review',
action='manage_repository_reviews',
**kwd ) )
return trans.response.send_redirect( web.url_for( controller='repository_review',
action='view_or_manage_repository',
**kwd ) )
def upload(self, trans, **kwd):
message = kwd.get('message', '')
status = kwd.get('status', 'done')
commit_message = kwd.get('commit_message', 'Uploaded')
category_ids = util.listify(kwd.get('category_id', ''))
categories = suc.get_categories(trans)
repository_id = kwd.get('repository_id', '')
repository = suc.get_repository_in_tool_shed(trans, repository_id)
repo_dir = repository.repo_path(trans.app)
repo = hg.repository(suc.get_configured_ui(), repo_dir)
uncompress_file = util.string_as_bool(
kwd.get('uncompress_file', 'true'))
remove_repo_files_not_in_tar = util.string_as_bool(
kwd.get('remove_repo_files_not_in_tar', 'true'))
uploaded_file = None
upload_point = commit_util.get_upload_point(repository, **kwd)
tip = repository.tip(trans.app)
file_data = kwd.get('file_data', '')
url = kwd.get('url', '')
# Part of the upload process is sending email notification to those that have registered to
# receive them. One scenario occurs when the first change set is produced for the repository.
# See the suc.handle_email_alerts() method for the definition of the scenarios.
new_repo_alert = repository.is_new(trans.app)
uploaded_directory = None
if kwd.get('upload_button', False):
if file_data == '' and url == '':
message = 'No files were entered on the upload form.'
status = 'error'
uploaded_file = None
elif url and url.startswith('hg'):
# Use mercurial clone to fetch repository, contents will then be copied over.
uploaded_directory = tempfile.mkdtemp()
repo_url = 'http%s' % url[len('hg'):]
repo_url = repo_url.encode('ascii', 'replace')
commands.clone(suc.get_configured_ui(), repo_url,
uploaded_directory)
elif url:
valid_url = True
try:
stream = urllib.urlopen(url)
except Exception, e:
valid_url = False
message = 'Error uploading file via http: %s' % str(e)
status = 'error'
uploaded_file = None
if valid_url:
fd, uploaded_file_name = tempfile.mkstemp()
uploaded_file = open(uploaded_file_name, 'wb')
while 1:
chunk = stream.read(util.CHUNK_SIZE)
if not chunk:
break
uploaded_file.write(chunk)
uploaded_file.flush()
uploaded_file_filename = url.split('/')[-1]
isempty = os.path.getsize(
os.path.abspath(uploaded_file_name)) == 0
elif file_data not in ('', None):
uploaded_file = file_data.file
uploaded_file_name = uploaded_file.name
uploaded_file_filename = os.path.split(file_data.filename)[-1]
isempty = os.path.getsize(
os.path.abspath(uploaded_file_name)) == 0
if uploaded_file or uploaded_directory:
ok = True
isgzip = False
isbz2 = False
if uploaded_file:
if uncompress_file:
isgzip = checkers.is_gzip(uploaded_file_name)
if not isgzip:
isbz2 = checkers.is_bz2(uploaded_file_name)
if isempty:
tar = None
istar = False
else:
# Determine what we have - a single file or an archive
try:
if (isgzip or isbz2) and uncompress_file:
# Open for reading with transparent compression.
tar = tarfile.open(uploaded_file_name, 'r:*')
else:
tar = tarfile.open(uploaded_file_name)
istar = True
except tarfile.ReadError, e:
tar = None
istar = False
else:
# Uploaded directory
istar = False
if istar:
ok, message, files_to_remove, content_alert_str, undesirable_dirs_removed, undesirable_files_removed = \
self.upload_tar( trans, repository, tar, uploaded_file, upload_point, remove_repo_files_not_in_tar, commit_message, new_repo_alert )
elif uploaded_directory:
ok, message, files_to_remove, content_alert_str, undesirable_dirs_removed, undesirable_files_removed = \
self.upload_directory( trans, repository, uploaded_directory, upload_point, remove_repo_files_not_in_tar, commit_message, new_repo_alert )
else:
if (isgzip or isbz2) and uncompress_file:
uploaded_file_filename = commit_util.uncompress(
repository, uploaded_file_name,
uploaded_file_filename, isgzip, isbz2)
if repository.type == rt_util.TOOL_DEPENDENCY_DEFINITION and uploaded_file_filename != suc.TOOL_DEPENDENCY_DEFINITION_FILENAME:
ok = False
message = 'Repositories of type <b>Tool dependency definition</b> can only contain a single file named <b>tool_dependencies.xml</b>.'
if ok:
if upload_point is not None:
full_path = os.path.abspath(
os.path.join(repo_dir, upload_point,
uploaded_file_filename))
else:
full_path = os.path.abspath(
os.path.join(repo_dir, uploaded_file_filename))
# Move some version of the uploaded file to the load_point within the repository hierarchy.
if uploaded_file_filename in [
suc.REPOSITORY_DEPENDENCY_DEFINITION_FILENAME
]:
# Inspect the contents of the file to see if changeset_revision values are missing and if so, set them appropriately.
altered, root_elem = commit_util.handle_repository_dependencies_definition(
trans, uploaded_file_name)
if altered:
tmp_filename = xml_util.create_and_write_tmp_file(
root_elem)
shutil.move(tmp_filename, full_path)
else:
shutil.move(uploaded_file_name, full_path)
elif uploaded_file_filename in [
suc.TOOL_DEPENDENCY_DEFINITION_FILENAME
]:
# Inspect the contents of the file to see if it defines a complex repository dependency definition whose changeset_revision values
# are missing and if so, set them appropriately.
altered, root_elem = commit_util.handle_tool_dependencies_definition(
trans, uploaded_file_name)
if altered:
tmp_filename = xml_util.create_and_write_tmp_file(
root_elem)
shutil.move(tmp_filename, full_path)
else:
shutil.move(uploaded_file_name, full_path)
else:
shutil.move(uploaded_file_name, full_path)
# See if any admin users have chosen to receive email alerts when a repository is updated. If so, check every uploaded file to ensure
# content is appropriate.
check_contents = commit_util.check_file_contents_for_email_alerts(
trans)
if check_contents and os.path.isfile(full_path):
content_alert_str = commit_util.check_file_content_for_html_and_images(
full_path)
else:
content_alert_str = ''
commands.add(repo.ui, repo, full_path)
# Convert from unicode to prevent "TypeError: array item must be char"
full_path = full_path.encode('ascii', 'replace')
commands.commit(repo.ui,
repo,
full_path,
user=trans.user.username,
message=commit_message)
if full_path.endswith(
'tool_data_table_conf.xml.sample'):
# Handle the special case where a tool_data_table_conf.xml.sample file is being uploaded by parsing the file and adding new entries
# to the in-memory trans.app.tool_data_tables dictionary.
error, error_message = tool_util.handle_sample_tool_data_table_conf_file(
trans.app, full_path)
if error:
message = '%s<br/>%s' % (message,
error_message)
# See if the content of the change set was valid.
admin_only = len(
repository.downloadable_revisions) != 1
suc.handle_email_alerts(
trans,
repository,
content_alert_str=content_alert_str,
new_repo_alert=new_repo_alert,
admin_only=admin_only)
if ok:
# Update the repository files for browsing.
suc.update_repository(repo)
# Get the new repository tip.
if tip == repository.tip(trans.app):
message = 'No changes to repository. '
status = 'warning'
else:
if (isgzip or isbz2) and uncompress_file:
uncompress_str = ' uncompressed and '
else:
uncompress_str = ' '
if uploaded_directory:
source_type = "repository"
source = url
else:
source_type = "file"
source = uploaded_file_filename
message = "The %s <b>%s</b> has been successfully%suploaded to the repository. " % (
source_type, source, uncompress_str)
if istar and (undesirable_dirs_removed
or undesirable_files_removed):
items_removed = undesirable_dirs_removed + undesirable_files_removed
message += " %d undesirable items (.hg .svn .git directories, .DS_Store, hgrc files, etc) were removed from the archive. " % items_removed
if istar and remove_repo_files_not_in_tar and files_to_remove:
if upload_point is not None:
message += " %d files were removed from the repository relative to the selected upload point '%s'. " % (
len(files_to_remove), upload_point)
else:
message += " %d files were removed from the repository root. " % len(
files_to_remove)
kwd['message'] = message
metadata_util.set_repository_metadata_due_to_new_tip(
trans,
repository,
content_alert_str=content_alert_str,
**kwd)
if repository.metadata_revisions:
# A repository's metadata revisions are order descending by update_time, so the zeroth revision will be the tip just after an upload.
metadata_dict = repository.metadata_revisions[
0].metadata
else:
metadata_dict = {}
if str(repository.type
) != rt_util.TOOL_DEPENDENCY_DEFINITION:
# Provide a warning message if a tool_dependencies.xml file is provided, but tool dependencies weren't loaded due to a requirement tag mismatch
# or some other problem. Tool dependency definitions can define orphan tool dependencies (no relationship to any tools contained in the repository),
# so warning messages are important because orphans are always valid. The repository owner must be warned in case they did not intend to define an
# orphan dependency, but simply provided incorrect information (tool shed, name owner, changeset_revision) for the definition.
# Handle messaging for orphan tool dependencies.
orphan_message = tool_dependency_util.generate_message_for_orphan_tool_dependencies(
trans, repository, metadata_dict)
if orphan_message:
message += orphan_message
status = 'warning'
# Handle messaging for invalid tool dependencies.
invalid_tool_dependencies_message = tool_dependency_util.generate_message_for_invalid_tool_dependencies(
metadata_dict)
if invalid_tool_dependencies_message:
message += invalid_tool_dependencies_message
status = 'error'
# Handle messaging for invalid repository dependencies.
invalid_repository_dependencies_message = repository_dependency_util.generate_message_for_invalid_repository_dependencies(
metadata_dict)
if invalid_repository_dependencies_message:
message += invalid_repository_dependencies_message
status = 'error'
# Reset the tool_data_tables by loading the empty tool_data_table_conf.xml file.
tool_util.reset_tool_data_tables(trans.app)
trans.response.send_redirect(
web.url_for(controller='repository',
action='browse_repository',
id=repository_id,
commit_message='Deleted selected files',
message=message,
status=status))
else:
status = 'error'
# Reset the tool_data_tables by loading the empty tool_data_table_conf.xml file.
tool_util.reset_tool_data_tables(trans.app)
def create_changeset_revision( self, trans, id, payload, **kwd ):
"""
POST /api/repositories/{encoded_repository_id}/changeset_revision
Create a new tool shed repository commit - leaving PUT on parent
resource open for updating meta-attributes of the repository (and
Galaxy doesn't allow PUT multipart data anyway
https://trello.com/c/CQwmCeG6).
:param id: the encoded id of the Repository object
The following parameters may be included in the payload.
:param commit_message: hg commit message for update.
"""
# Example URL: http://localhost:9009/api/repositories/f9cad7b01a472135
rdah = attribute_handlers.RepositoryDependencyAttributeHandler( trans.app, unpopulate=False )
tdah = attribute_handlers.ToolDependencyAttributeHandler( trans.app, unpopulate=False )
repository = suc.get_repository_in_tool_shed( trans.app, id )
repo_dir = repository.repo_path( trans.app )
repo = hg_util.get_repo_for_repository( trans.app, repository=None, repo_path=repo_dir, create=False )
upload_point = commit_util.get_upload_point( repository, **kwd )
tip = repository.tip( trans.app )
file_data = payload.get('file')
# Code stolen from gx's upload_common.py
if isinstance( file_data, FieldStorage ):
assert not isinstance( file_data.file, StringIO.StringIO )
assert file_data.file.name != '<fdopen>'
local_filename = util.mkstemp_ln( file_data.file.name, 'upload_file_data_' )
file_data.file.close()
file_data = dict( filename=file_data.filename,
local_filename=local_filename )
elif type( file_data ) == dict and 'local_filename' not in file_data:
raise Exception( 'Uploaded file was encoded in a way not understood.' )
commit_message = kwd.get( 'commit_message', 'Uploaded' )
uploaded_file = open(file_data['local_filename'], 'rb')
uploaded_file_name = file_data['local_filename']
isgzip = False
isbz2 = False
isgzip = checkers.is_gzip( uploaded_file_name )
if not isgzip:
isbz2 = checkers.is_bz2( uploaded_file_name )
if ( isgzip or isbz2 ):
# Open for reading with transparent compression.
tar = tarfile.open( uploaded_file_name, 'r:*' )
else:
tar = tarfile.open( uploaded_file_name )
new_repo_alert = False
remove_repo_files_not_in_tar = True
ok, message, files_to_remove, content_alert_str, undesirable_dirs_removed, undesirable_files_removed = \
repository_content_util.upload_tar(
trans,
rdah,
tdah,
repository,
tar,
uploaded_file,
upload_point,
remove_repo_files_not_in_tar,
commit_message,
new_repo_alert
)
if ok:
# Update the repository files for browsing.
hg_util.update_repository( repo )
# Get the new repository tip.
if tip == repository.tip( trans.app ):
trans.response.status = 400
message = 'No changes to repository.'
ok = False
else:
rmm = repository_metadata_manager.RepositoryMetadataManager( app=trans.app,
user=trans.user,
repository=repository )
status, error_message = \
rmm.set_repository_metadata_due_to_new_tip( trans.request.host,
content_alert_str=content_alert_str,
**kwd )
if error_message:
ok = False
trans.response.status = 500
message = error_message
else:
trans.response.status = 500
if not ok:
return {
"err_msg": message,
"content_alert": content_alert_str
}
else:
return {
"message": message,
"content_alert": content_alert_str
}
repository,
None,
as_html=False )
else:
message = "Successfully reset metadata on repository %s owned by %s" % \
( str( repository.name ), str( repository.user.username ) )
except Exception, e:
message = "Error resetting metadata on repository %s owned by %s: %s" % \
( str( repository.name ), str( repository.user.username ), str( e ) )
status = '%s : %s' % ( str( repository.name ), message )
results[ 'repository_status' ].append( status )
return results
repository_id = payload.get( 'repository_id', None )
if repository_id is not None:
repository = suc.get_repository_in_tool_shed( trans.app, repository_id )
start_time = strftime( "%Y-%m-%d %H:%M:%S" )
log.debug( "%s...resetting metadata on repository %s" % ( start_time, str( repository.name ) ) )
results = handle_repository( trans, start_time, repository )
stop_time = strftime( "%Y-%m-%d %H:%M:%S" )
results[ 'stop_time' ] = stop_time
return json.dumps( results, sort_keys=True, indent=4 )
@web.expose_api_anonymous
def show( self, trans, id, **kwd ):
"""
GET /api/repositories/{encoded_repository_id}
Returns information about a repository in the Tool Shed.
:param id: the encoded id of the Repository object
"""
repository,
None,
as_html=False)
else:
message = "Successfully reset metadata on repository %s owned by %s" % \
( str( repository.name ), str( repository.user.username ) )
except Exception, e:
message = "Error resetting metadata on repository %s owned by %s: %s" % \
( str( repository.name ), str( repository.user.username ), str( e ) )
status = '%s : %s' % (str(repository.name), message)
results['repository_status'].append(status)
return results
repository_id = payload.get('repository_id', None)
if repository_id is not None:
repository = suc.get_repository_in_tool_shed(
trans.app, repository_id)
start_time = strftime("%Y-%m-%d %H:%M:%S")
log.debug("%s...resetting metadata on repository %s" %
(start_time, str(repository.name)))
results = handle_repository(trans, start_time, repository)
stop_time = strftime("%Y-%m-%d %H:%M:%S")
results['stop_time'] = stop_time
return json.dumps(results, sort_keys=True, indent=4)
@web.expose_api_anonymous
def show(self, trans, id, **kwd):
"""
GET /api/repositories/{encoded_repository_id}
Returns information about a repository in the Tool Shed.
:param id: the encoded id of the Repository object
def create_changeset_revision(self, trans, id, payload, **kwd):
"""
POST /api/repositories/{encoded_repository_id}/changeset_revision
Create a new tool shed repository commit - leaving PUT on parent
resource open for updating meta-attributes of the repository (and
Galaxy doesn't allow PUT multipart data anyway
https://trello.com/c/CQwmCeG6).
:param id: the encoded id of the Repository object
The following parameters may be included in the payload.
:param commit_message: hg commit message for update.
"""
# Example URL: http://localhost:9009/api/repositories/f9cad7b01a472135
rdah = attribute_handlers.RepositoryDependencyAttributeHandler(
trans.app, unpopulate=False)
tdah = attribute_handlers.ToolDependencyAttributeHandler(
trans.app, unpopulate=False)
repository = suc.get_repository_in_tool_shed(trans.app, id)
repo_dir = repository.repo_path(trans.app)
repo = hg_util.get_repo_for_repository(trans.app,
repository=None,
repo_path=repo_dir,
create=False)
upload_point = commit_util.get_upload_point(repository, **kwd)
tip = repository.tip(trans.app)
file_data = payload.get('file')
# Code stolen from gx's upload_common.py
if isinstance(file_data, FieldStorage):
assert not isinstance(file_data.file, StringIO.StringIO)
assert file_data.file.name != '<fdopen>'
local_filename = util.mkstemp_ln(file_data.file.name,
'upload_file_data_')
file_data.file.close()
file_data = dict(filename=file_data.filename,
local_filename=local_filename)
elif type(file_data) == dict and 'local_filename' not in file_data:
raise Exception(
'Uploaded file was encoded in a way not understood.')
commit_message = kwd.get('commit_message', 'Uploaded')
uploaded_file = open(file_data['local_filename'], 'rb')
uploaded_file_name = file_data['local_filename']
isgzip = False
isbz2 = False
isgzip = checkers.is_gzip(uploaded_file_name)
if not isgzip:
isbz2 = checkers.is_bz2(uploaded_file_name)
if (isgzip or isbz2):
# Open for reading with transparent compression.
tar = tarfile.open(uploaded_file_name, 'r:*')
else:
tar = tarfile.open(uploaded_file_name)
new_repo_alert = False
remove_repo_files_not_in_tar = True
ok, message, files_to_remove, content_alert_str, undesirable_dirs_removed, undesirable_files_removed = \
repository_content_util.upload_tar(
trans,
rdah,
tdah,
repository,
tar,
uploaded_file,
upload_point,
remove_repo_files_not_in_tar,
commit_message,
new_repo_alert
)
if ok:
# Update the repository files for browsing.
hg_util.update_repository(repo)
# Get the new repository tip.
if tip == repository.tip(trans.app):
trans.response.status = 400
message = 'No changes to repository.'
ok = False
else:
rmm = repository_metadata_manager.RepositoryMetadataManager(
app=trans.app, user=trans.user, repository=repository)
status, error_message = \
rmm.set_repository_metadata_due_to_new_tip( trans.request.host,
content_alert_str=content_alert_str,
**kwd )
if error_message:
ok = False
trans.response.status = 500
message = error_message
else:
trans.response.status = 500
if not ok:
return {"err_msg": message, "content_alert": content_alert_str}
else:
return {"message": message, "content_alert": content_alert_str}
def export_repository( trans, tool_shed_url, repository_id, repository_name, changeset_revision, file_type, export_repository_dependencies, api=False ):
repository = suc.get_repository_in_tool_shed( trans, repository_id )
repositories_archive_filename = generate_repository_archive_filename( tool_shed_url,
str( repository.name ),
str( repository.user.username ),
changeset_revision,
file_type,
export_repository_dependencies=export_repository_dependencies,
use_tmp_archive_dir=True )
if export_repository_dependencies:
repo_info_dicts = get_repo_info_dicts( trans, tool_shed_url, repository_id, changeset_revision )
repository_ids = get_repository_ids( trans, repo_info_dicts )
ordered_repository_ids, ordered_repositories, ordered_changeset_revisions = order_components_for_import( trans, repository_ids, repo_info_dicts )
else:
ordered_repository_ids = []
ordered_repositories = []
ordered_changeset_revisions = []
if repository:
repository_metadata = suc.get_current_repository_metadata_for_changeset_revision( trans, repository, changeset_revision )
if repository_metadata:
ordered_repository_ids = [ repository_id ]
ordered_repositories = [ repository ]
ordered_changeset_revisions = [ repository_metadata.changeset_revision ]
repositories_archive = None
error_messages = ''
lock = threading.Lock()
lock.acquire( True )
try:
repositories_archive = tarfile.open( repositories_archive_filename, "w:%s" % file_type )
exported_repository_registry = ExportedRepositoryRegistry()
for index, repository_id in enumerate( ordered_repository_ids ):
work_dir = tempfile.mkdtemp( prefix="tmp-toolshed-export-er" )
ordered_repository = ordered_repositories[ index ]
ordered_changeset_revision = ordered_changeset_revisions[ index ]
repository_archive, error_message = generate_repository_archive( trans,
work_dir,
tool_shed_url,
ordered_repository,
ordered_changeset_revision,
file_type )
if error_message:
error_messages = '%s %s' % ( error_messages, error_message )
else:
archive_name = str( os.path.basename( repository_archive.name ) )
repositories_archive.add( repository_archive.name, arcname=archive_name )
attributes, sub_elements = get_repository_attributes_and_sub_elements( ordered_repository, archive_name )
elem = xml_util.create_element( 'repository', attributes=attributes, sub_elements=sub_elements )
exported_repository_registry.exported_repository_elems.append( elem )
suc.remove_dir( work_dir )
# Keep information about the export in a file name export_info.xml in the archive.
sub_elements = generate_export_elem( tool_shed_url, repository, changeset_revision, export_repository_dependencies, api )
export_elem = xml_util.create_element( 'export_info', attributes=None, sub_elements=sub_elements )
tmp_export_info = xml_util.create_and_write_tmp_file( export_elem, use_indent=True )
repositories_archive.add( tmp_export_info, arcname='export_info.xml' )
# Write the manifest, which must preserve the order in which the repositories should be imported.
exported_repository_root = xml_util.create_element( 'repositories' )
for exported_repository_elem in exported_repository_registry.exported_repository_elems:
exported_repository_root.append( exported_repository_elem )
tmp_manifest = xml_util.create_and_write_tmp_file( exported_repository_root, use_indent=True )
repositories_archive.add( tmp_manifest, arcname='manifest.xml' )
except Exception, e:
log.exception( str( e ) )
