def handle_repository_dependency_elem( trans, elem, unpopulate=False ): # <repository name="molecule_datatypes" owner="test" changeset_revision="1a070566e9c6" /> error_message = '' name = elem.get( 'name' ) owner = elem.get( 'owner' ) # The name and owner attributes are always required, so if either are missing, return the error message. if not name or not owner: error_message = handle_missing_repository_attribute( elem ) return False, elem, error_message revised = False toolshed = elem.get( 'toolshed' ) changeset_revision = elem.get( 'changeset_revision' ) if unpopulate: # We're exporting the repository, so eliminate all toolshed and changeset_revision attributes from the <repository> tag. if toolshed or changeset_revision: attributes = odict() attributes[ 'name' ] = name attributes[ 'owner' ] = owner attributes[ 'prior_installation_required' ] = elem.get( 'prior_installation_required', 'False' ) elem = xml_util.create_element( 'repository', attributes=attributes, sub_elements=None ) revised = True return revised, elem, error_message # From here on we're populating the toolshed and changeset_revisions if necessary. if not toolshed: # Default the setting to the current tool shed. toolshed = str( url_for( '/', qualified=True ) ).rstrip( '/' ) elem.attrib[ 'toolshed' ] = toolshed revised = True if not changeset_revision: # Populate the changeset_revision attribute with the latest installable metadata revision for the defined repository. # We use the latest installable revision instead of the latest metadata revision to ensure that the contents of the # revision are valid. repository = suc.get_repository_by_name_and_owner( trans.app, name, owner ) if repository: repo_dir = repository.repo_path( trans.app ) repo = hg.repository( suc.get_configured_ui(), repo_dir ) lastest_installable_changeset_revision = suc.get_latest_downloadable_changeset_revision( trans, repository, repo ) if lastest_installable_changeset_revision != suc.INITIAL_CHANGELOG_HASH: elem.attrib[ 'changeset_revision' ] = lastest_installable_changeset_revision revised = True else: error_message = 'Invalid latest installable changeset_revision %s ' % str( lastest_installable_changeset_revision ) error_message += 'retrieved for repository %s owned by %s. ' % ( str( name ), str( owner ) ) else: error_message = 'Unable to locate repository with name %s and owner %s. ' % ( str( name ), str( owner ) ) return revised, elem, error_message
def handle_elem( self, elem ): """Populate or unpopulate the changeset_revision and toolshed attributes of repository tags.""" # <repository name="molecule_datatypes" owner="test" changeset_revision="1a070566e9c6" /> # <repository changeset_revision="xxx" name="package_xorg_macros_1_17_1" owner="test" toolshed="yyy"> # <package name="xorg_macros" version="1.17.1" /> # </repository> error_message = '' name = elem.get( 'name' ) owner = elem.get( 'owner' ) # The name and owner attributes are always required, so if either are missing, return the error message. if not name or not owner: error_message = self.check_tag_attributes( elem ) return False, elem, error_message altered = False toolshed = elem.get( 'toolshed' ) changeset_revision = elem.get( 'changeset_revision' ) # Over a short period of time a bug existed which caused the prior_installation_required attribute # to be set to False and included in the <repository> tag when a repository was exported along with # its dependencies. The following will eliminate this problematic attribute upon import. prior_installation_required = elem.get( 'prior_installation_required' ) if prior_installation_required is not None and not asbool( prior_installation_required ): del elem.attrib[ 'prior_installation_required' ] sub_elems = [ child_elem for child_elem in list( elem ) ] if len( sub_elems ) > 0: # At this point, a <repository> tag will point only to a package. # <package name="xorg_macros" version="1.17.1" /> # Coerce the list to an odict(). sub_elements = odict() packages = [] for sub_elem in sub_elems: sub_elem_type = sub_elem.tag sub_elem_name = sub_elem.get( 'name' ) sub_elem_version = sub_elem.get( 'version' ) if sub_elem_type and sub_elem_name and sub_elem_version: packages.append( ( sub_elem_name, sub_elem_version ) ) sub_elements[ 'packages' ] = packages else: # Set to None. sub_elements = None if self.unpopulate: # We're exporting the repository, so eliminate all toolshed and changeset_revision attributes # from the <repository> tag. if toolshed or changeset_revision: attributes = odict() attributes[ 'name' ] = name attributes[ 'owner' ] = owner prior_installation_required = elem.get( 'prior_installation_required' ) if asbool( prior_installation_required ): attributes[ 'prior_installation_required' ] = 'True' new_elem = xml_util.create_element( 'repository', attributes=attributes, sub_elements=sub_elements ) altered = True return altered, new_elem, error_message # From here on we're populating the toolshed and changeset_revision attributes if necessary. if not toolshed: # Default the setting to the current tool shed. toolshed = str( url_for( '/', qualified=True ) ).rstrip( '/' ) elem.attrib[ 'toolshed' ] = toolshed altered = True if not changeset_revision: # Populate the changeset_revision attribute with the latest installable metadata revision for # the defined repository. We use the latest installable revision instead of the latest metadata # revision to ensure that the contents of the revision are valid. repository = suc.get_repository_by_name_and_owner( self.app, name, owner ) if repository: repo = hg_util.get_repo_for_repository( self.app, repository=repository, repo_path=None, create=False ) lastest_installable_changeset_revision = \ suc.get_latest_downloadable_changeset_revision( self.app, repository, repo ) if lastest_installable_changeset_revision != hg_util.INITIAL_CHANGELOG_HASH: elem.attrib[ 'changeset_revision' ] = lastest_installable_changeset_revision altered = True else: error_message = 'Invalid latest installable changeset_revision %s ' % \ str( lastest_installable_changeset_revision ) error_message += 'retrieved for repository %s owned by %s. ' % ( str( name ), str( owner ) ) else: error_message = 'Unable to locate repository with name %s and owner %s. ' % ( str( name ), str( owner ) ) return altered, elem, error_message
def handle_elem(self, elem): """Populate or unpopulate the changeset_revision and toolshed attributes of repository tags.""" # <repository name="molecule_datatypes" owner="test" changeset_revision="1a070566e9c6" /> # <repository changeset_revision="xxx" name="package_xorg_macros_1_17_1" owner="test" toolshed="yyy"> # <package name="xorg_macros" version="1.17.1" /> # </repository> error_message = '' name = elem.get('name') owner = elem.get('owner') # The name and owner attributes are always required, so if either are missing, return the error message. if not name or not owner: error_message = self.check_tag_attributes(elem) return False, elem, error_message altered = False toolshed = elem.get('toolshed') changeset_revision = elem.get('changeset_revision') # Over a short period of time a bug existed which caused the prior_installation_required attribute # to be set to False and included in the <repository> tag when a repository was exported along with # its dependencies. The following will eliminate this problematic attribute upon import. prior_installation_required = elem.get('prior_installation_required') if prior_installation_required is not None and not asbool(prior_installation_required): del elem.attrib['prior_installation_required'] sub_elems = [child_elem for child_elem in list(elem)] if len(sub_elems) > 0: # At this point, a <repository> tag will point only to a package. # <package name="xorg_macros" version="1.17.1" /> # Coerce the list to an odict(). sub_elements = odict() packages = [] for sub_elem in sub_elems: sub_elem_type = sub_elem.tag sub_elem_name = sub_elem.get('name') sub_elem_version = sub_elem.get('version') if sub_elem_type and sub_elem_name and sub_elem_version: packages.append((sub_elem_name, sub_elem_version)) sub_elements['packages'] = packages else: # Set to None. sub_elements = None if self.unpopulate: # We're exporting the repository, so eliminate all toolshed and changeset_revision attributes # from the <repository> tag. if toolshed or changeset_revision: attributes = odict() attributes['name'] = name attributes['owner'] = owner prior_installation_required = elem.get('prior_installation_required') if asbool(prior_installation_required): attributes['prior_installation_required'] = 'True' new_elem = xml_util.create_element('repository', attributes=attributes, sub_elements=sub_elements) altered = True return altered, new_elem, error_message # From here on we're populating the toolshed and changeset_revision attributes if necessary. if not toolshed: # Default the setting to the current tool shed. toolshed = str(url_for('/', qualified=True)).rstrip('/') elem.attrib['toolshed'] = toolshed altered = True if not changeset_revision: # Populate the changeset_revision attribute with the latest installable metadata revision for # the defined repository. We use the latest installable revision instead of the latest metadata # revision to ensure that the contents of the revision are valid. repository = repository_util.get_repository_by_name_and_owner(self.app, name, owner) if repository: lastest_installable_changeset_revision = \ metadata_util.get_latest_downloadable_changeset_revision(self.app, repository) if lastest_installable_changeset_revision != hg_util.INITIAL_CHANGELOG_HASH: elem.attrib['changeset_revision'] = lastest_installable_changeset_revision altered = True else: error_message = 'Invalid latest installable changeset_revision %s ' % \ str(lastest_installable_changeset_revision) error_message += 'retrieved for repository %s owned by %s. ' % (str(name), str(owner)) else: error_message = 'Unable to locate repository with name %s and owner %s. ' % (str(name), str(owner)) return altered, elem, error_message
def export_repository( trans, tool_shed_url, repository_id, repository_name, changeset_revision, file_type, export_repository_dependencies, api=False ): repository = suc.get_repository_in_tool_shed( trans, repository_id ) repositories_archive_filename = generate_repository_archive_filename( tool_shed_url, str( repository.name ), str( repository.user.username ), changeset_revision, file_type, export_repository_dependencies=export_repository_dependencies, use_tmp_archive_dir=True ) if export_repository_dependencies: repo_info_dicts = get_repo_info_dicts( trans, tool_shed_url, repository_id, changeset_revision ) repository_ids = get_repository_ids( trans, repo_info_dicts ) ordered_repository_ids, ordered_repositories, ordered_changeset_revisions = order_components_for_import( trans, repository_ids, repo_info_dicts ) else: ordered_repository_ids = [] ordered_repositories = [] ordered_changeset_revisions = [] if repository: repository_metadata = suc.get_current_repository_metadata_for_changeset_revision( trans, repository, changeset_revision ) if repository_metadata: ordered_repository_ids = [ repository_id ] ordered_repositories = [ repository ] ordered_changeset_revisions = [ repository_metadata.changeset_revision ] repositories_archive = None error_messages = '' lock = threading.Lock() lock.acquire( True ) try: repositories_archive = tarfile.open( repositories_archive_filename, "w:%s" % file_type ) exported_repository_registry = ExportedRepositoryRegistry() for index, repository_id in enumerate( ordered_repository_ids ): work_dir = tempfile.mkdtemp( prefix="tmp-toolshed-export-er" ) ordered_repository = ordered_repositories[ index ] ordered_changeset_revision = ordered_changeset_revisions[ index ] repository_archive, error_message = generate_repository_archive( trans, work_dir, tool_shed_url, ordered_repository, ordered_changeset_revision, file_type ) if error_message: error_messages = '%s %s' % ( error_messages, error_message ) else: archive_name = str( os.path.basename( repository_archive.name ) ) repositories_archive.add( repository_archive.name, arcname=archive_name ) attributes, sub_elements = get_repository_attributes_and_sub_elements( ordered_repository, archive_name ) elem = xml_util.create_element( 'repository', attributes=attributes, sub_elements=sub_elements ) exported_repository_registry.exported_repository_elems.append( elem ) suc.remove_dir( work_dir ) # Keep information about the export in a file name export_info.xml in the archive. sub_elements = generate_export_elem( tool_shed_url, repository, changeset_revision, export_repository_dependencies, api ) export_elem = xml_util.create_element( 'export_info', attributes=None, sub_elements=sub_elements ) tmp_export_info = xml_util.create_and_write_tmp_file( export_elem, use_indent=True ) repositories_archive.add( tmp_export_info, arcname='export_info.xml' ) # Write the manifest, which must preserve the order in which the repositories should be imported. exported_repository_root = xml_util.create_element( 'repositories' ) for exported_repository_elem in exported_repository_registry.exported_repository_elems: exported_repository_root.append( exported_repository_elem ) tmp_manifest = xml_util.create_and_write_tmp_file( exported_repository_root, use_indent=True ) repositories_archive.add( tmp_manifest, arcname='manifest.xml' ) except Exception, e: log.exception( str( e ) )