def upload_tar(trans, rdah, tdah, repository, tar, uploaded_file, upload_point,
               remove_repo_files_not_in_tar, commit_message, new_repo_alert):
    # Upload a tar archive of files.
    repo_dir = repository.repo_path(trans.app)
    hg_util.get_repo_for_repository(trans.app,
                                    repository=None,
                                    repo_path=repo_dir,
                                    create=False)
    undesirable_dirs_removed = 0
    undesirable_files_removed = 0
    check_results = commit_util.check_archive(repository, tar)
    if check_results.invalid:
        tar.close()
        uploaded_file.close()
        message = '%s Invalid paths were: %s' % (' '.join(
            check_results.errors), ', '.join(check_results.invalid))
        return False, message, [], '', undesirable_dirs_removed, undesirable_files_removed
    else:
        if upload_point is not None:
            full_path = os.path.abspath(os.path.join(repo_dir, upload_point))
        else:
            full_path = os.path.abspath(repo_dir)
        undesirable_files_removed = len(check_results.undesirable_files)
        undesirable_dirs_removed = len(check_results.undesirable_dirs)
        filenames_in_archive = [ti.name for ti in check_results.valid]
        # Extract the uploaded tar to the load_point within the repository hierarchy.
        tar.extractall(path=full_path, members=check_results.valid)
        tar.close()
        uploaded_file.close()
        for filename in filenames_in_archive:
            uploaded_file_name = os.path.join(full_path, filename)
            if os.path.split(uploaded_file_name)[
                    -1] == rt_util.REPOSITORY_DEPENDENCY_DEFINITION_FILENAME:
                # Inspect the contents of the file to see if toolshed or changeset_revision attributes
                # are missing and if so, set them appropriately.
                altered, root_elem, error_message = rdah.handle_tag_attributes(
                    uploaded_file_name)
                if error_message:
                    return False, error_message, [], '', [], []
                elif altered:
                    tmp_filename = xml_util.create_and_write_tmp_file(
                        root_elem)
                    shutil.move(tmp_filename, uploaded_file_name)
            elif os.path.split(uploaded_file_name)[
                    -1] == rt_util.TOOL_DEPENDENCY_DEFINITION_FILENAME:
                # Inspect the contents of the file to see if toolshed or changeset_revision
                # attributes are missing and if so, set them appropriately.
                altered, root_elem, error_message = tdah.handle_tag_attributes(
                    uploaded_file_name)
                if error_message:
                    return False, error_message, [], '', [], []
                if altered:
                    tmp_filename = xml_util.create_and_write_tmp_file(
                        root_elem)
                    shutil.move(tmp_filename, uploaded_file_name)
        return commit_util.handle_directory_changes(
            trans.app, trans.request.host, trans.user.username, repository,
            full_path, filenames_in_archive, remove_repo_files_not_in_tar,
            new_repo_alert, commit_message, undesirable_dirs_removed,
            undesirable_files_removed)
def upload_tar( trans, rdah, tdah, repository, tar, uploaded_file, upload_point, remove_repo_files_not_in_tar,
                commit_message, new_repo_alert ):
    # Upload a tar archive of files.
    repo_dir = repository.repo_path( trans.app )
    hg_util.get_repo_for_repository( trans.app, repository=None, repo_path=repo_dir, create=False )
    undesirable_dirs_removed = 0
    undesirable_files_removed = 0
    check_results = commit_util.check_archive( repository, tar )
    if check_results.invalid:
        tar.close()
        uploaded_file.close()
        message = '%s Invalid paths were: %s' % (
            ' '.join( check_results.errors ), ', '.join( check_results.invalid ) )
        return False, message, [], '', undesirable_dirs_removed, undesirable_files_removed
    else:
        if upload_point is not None:
            full_path = os.path.abspath( os.path.join( repo_dir, upload_point ) )
        else:
            full_path = os.path.abspath( repo_dir )
        undesirable_files_removed = len( check_results.undesirable_files )
        undesirable_dirs_removed = len( check_results.undesirable_dirs )
        filenames_in_archive = [ ti.name for ti in check_results.valid ]
        # Extract the uploaded tar to the load_point within the repository hierarchy.
        tar.extractall( path=full_path, members=check_results.valid )
        tar.close()
        uploaded_file.close()
        for filename in filenames_in_archive:
            uploaded_file_name = os.path.join( full_path, filename )
            if os.path.split( uploaded_file_name )[ -1 ] == rt_util.REPOSITORY_DEPENDENCY_DEFINITION_FILENAME:
                # Inspect the contents of the file to see if toolshed or changeset_revision attributes
                # are missing and if so, set them appropriately.
                altered, root_elem, error_message = rdah.handle_tag_attributes( uploaded_file_name )
                if error_message:
                    return False, error_message, [], '', [], []
                elif altered:
                    tmp_filename = xml_util.create_and_write_tmp_file( root_elem )
                    shutil.move( tmp_filename, uploaded_file_name )
            elif os.path.split( uploaded_file_name )[ -1 ] == rt_util.TOOL_DEPENDENCY_DEFINITION_FILENAME:
                # Inspect the contents of the file to see if toolshed or changeset_revision
                # attributes are missing and if so, set them appropriately.
                altered, root_elem, error_message = tdah.handle_tag_attributes( uploaded_file_name )
                if error_message:
                    return False, error_message, [], '', [], []
                if altered:
                    tmp_filename = xml_util.create_and_write_tmp_file( root_elem )
                    shutil.move( tmp_filename, uploaded_file_name )
        return commit_util.handle_directory_changes( trans.app,
                                                     trans.request.host,
                                                     trans.user.username,
                                                     repository,
                                                     full_path,
                                                     filenames_in_archive,
                                                     remove_repo_files_not_in_tar,
                                                     new_repo_alert,
                                                     commit_message,
                                                     undesirable_dirs_removed,
                                                     undesirable_files_removed )
Esempio n. 3
0
 def upload_tar( self, trans, repository, tar, uploaded_file, upload_point, remove_repo_files_not_in_tar, commit_message, new_repo_alert ):
     # Upload a tar archive of files.
     repo_dir = repository.repo_path( trans.app )
     repo = hg.repository( suc.get_configured_ui(), repo_dir )
     undesirable_dirs_removed = 0
     undesirable_files_removed = 0
     ok, message = commit_util.check_archive( repository, tar )
     if not ok:
         tar.close()
         uploaded_file.close()
         return ok, message, [], '', undesirable_dirs_removed, undesirable_files_removed
     else:
         if upload_point is not None:
             full_path = os.path.abspath( os.path.join( repo_dir, upload_point ) )
         else:
             full_path = os.path.abspath( repo_dir )
         filenames_in_archive = []
         for tarinfo_obj in tar.getmembers():
             ok = os.path.basename( tarinfo_obj.name ) not in commit_util.UNDESIRABLE_FILES
             if ok:
                 for file_path_item in tarinfo_obj.name.split( '/' ):
                     if file_path_item in commit_util.UNDESIRABLE_DIRS:
                         undesirable_dirs_removed += 1
                         ok = False
                         break
             else:
                 undesirable_files_removed += 1
             if ok:
                 filenames_in_archive.append( tarinfo_obj.name )
         # Extract the uploaded tar to the load_point within the repository hierarchy.
         tar.extractall( path=full_path )
         tar.close()
         uploaded_file.close()
         for filename in filenames_in_archive:
             uploaded_file_name = os.path.join( full_path, filename )
             if os.path.split( uploaded_file_name )[ -1 ] == suc.REPOSITORY_DEPENDENCY_DEFINITION_FILENAME:
                 # Inspect the contents of the file to see if changeset_revision values are missing and if so, set them appropriately.
                 altered, root_elem = commit_util.handle_repository_dependencies_definition( trans, uploaded_file_name )
                 if altered:
                     tmp_filename = xml_util.create_and_write_tmp_file( root_elem )
                     shutil.move( tmp_filename, uploaded_file_name )
             elif os.path.split( uploaded_file_name )[ -1 ] == suc.TOOL_DEPENDENCY_DEFINITION_FILENAME:
                 # Inspect the contents of the file to see if changeset_revision values are missing and if so, set them appropriately.
                 altered, root_elem = commit_util.handle_tool_dependencies_definition( trans, uploaded_file_name )
                 if altered:
                     tmp_filename = xml_util.create_and_write_tmp_file( root_elem )
                     shutil.move( tmp_filename, uploaded_file_name )
         return commit_util.handle_directory_changes( trans,
                                                      repository,
                                                      full_path,
                                                      filenames_in_archive,
                                                      remove_repo_files_not_in_tar,
                                                      new_repo_alert,
                                                      commit_message,
                                                      undesirable_dirs_removed,
                                                      undesirable_files_removed )
def upload_tar( trans, rdah, tdah, repository, tar, uploaded_file, upload_point, remove_repo_files_not_in_tar,
                commit_message, new_repo_alert ):
    # Upload a tar archive of files.
    repo_dir = repository.repo_path( trans.app )
    hg_util.get_repo_for_repository( trans.app, repository=None, repo_path=repo_dir, create=False )
    undesirable_dirs_removed = 0
    undesirable_files_removed = 0
    ok, message = commit_util.check_archive( repository, tar )
    if not ok:
        tar.close()
        uploaded_file.close()
        return ok, message, [], '', undesirable_dirs_removed, undesirable_files_removed
    else:
        if upload_point is not None:
            full_path = os.path.abspath( os.path.join( repo_dir, upload_point ) )
        else:
            full_path = os.path.abspath( repo_dir )
        filenames_in_archive = []
        for tarinfo_obj in tar.getmembers():
            ok = os.path.basename( tarinfo_obj.name ) not in commit_util.UNDESIRABLE_FILES
            if ok:
                for file_path_item in tarinfo_obj.name.split( '/' ):
                    if file_path_item in commit_util.UNDESIRABLE_DIRS:
                        undesirable_dirs_removed += 1
                        ok = False
                        break
            else:
                undesirable_files_removed += 1
            if ok:
                filenames_in_archive.append( tarinfo_obj.name )
        # Extract the uploaded tar to the load_point within the repository hierarchy.
        tar.extractall( path=full_path )
        tar.close()
        uploaded_file.close()
        for filename in filenames_in_archive:
            uploaded_file_name = os.path.join( full_path, filename )
            if os.path.split( uploaded_file_name )[ -1 ] == rt_util.REPOSITORY_DEPENDENCY_DEFINITION_FILENAME:
                # Inspect the contents of the file to see if toolshed or changeset_revision attributes
                # are missing and if so, set them appropriately.
                altered, root_elem, error_message = rdah.handle_tag_attributes( uploaded_file_name )
                if error_message:
                    return False, error_message, [], '', [], []
                elif altered:
                    tmp_filename = xml_util.create_and_write_tmp_file( root_elem )
                    shutil.move( tmp_filename, uploaded_file_name )
            elif os.path.split( uploaded_file_name )[ -1 ] == rt_util.TOOL_DEPENDENCY_DEFINITION_FILENAME:
                # Inspect the contents of the file to see if toolshed or changeset_revision
                # attributes are missing and if so, set them appropriately.
                altered, root_elem, error_message = tdah.handle_tag_attributes( uploaded_file_name )
                if error_message:
                    return False, error_message, [], '', [], []
                if altered:
                    tmp_filename = xml_util.create_and_write_tmp_file( root_elem )
                    shutil.move( tmp_filename, uploaded_file_name )
        return commit_util.handle_directory_changes( trans.app,
                                                     trans.request.host,
                                                     trans.user.username,
                                                     repository,
                                                     full_path,
                                                     filenames_in_archive,
                                                     remove_repo_files_not_in_tar,
                                                     new_repo_alert,
                                                     commit_message,
                                                     undesirable_dirs_removed,
                                                     undesirable_files_removed )
Esempio n. 5
0
def import_repository_archive( trans, repository, repository_archive_dict ):
    """Import a repository archive contained within a repository capsule."""
    archive_file_name = repository_archive_dict.get( 'archive_file_name', None )
    capsule_file_name = repository_archive_dict[ 'capsule_file_name' ]
    encoded_file_path = repository_archive_dict[ 'encoded_file_path' ]
    file_path = encoding_util.tool_shed_decode( encoded_file_path )
    results_dict = dict( ok=True, error_message='' )
    archive_file_path = os.path.join( file_path, archive_file_name )
    archive = tarfile.open( archive_file_path, 'r:*' )
    repo_dir = repository.repo_path( trans.app )
    repo = hg.repository( suc.get_configured_ui(), repo_dir )
    undesirable_dirs_removed = 0
    undesirable_files_removed = 0
    ok, error_message = commit_util.check_archive( repository, archive )
    if ok:
        full_path = os.path.abspath( repo_dir )
        filenames_in_archive = []
        for tarinfo_obj in archive.getmembers():
            # Check files and directories in the archive.
            ok = os.path.basename( tarinfo_obj.name ) not in commit_util.UNDESIRABLE_FILES
            if ok:
                for file_path_item in tarinfo_obj.name.split( '/' ):
                    if file_path_item in commit_util.UNDESIRABLE_DIRS:
                        undesirable_dirs_removed += 1
                        error_message = 'Import failed: invalid file path <b>%s</b> in archive <b>%s</b>' % \
                            ( str( file_path_item ), str( archive_file_name ) )
                        results_dict[ 'ok' ] = False
                        results_dict[ 'error_message' ] += error_message
                        return results_dict
                filenames_in_archive.append( tarinfo_obj.name )
            else:
                undesirable_files_removed += 1
        # Extract the uploaded archive to the repository root.
        archive.extractall( path=full_path )
        archive.close()
        for filename in filenames_in_archive:
            uploaded_file_name = os.path.join( full_path, filename )
            if os.path.split( uploaded_file_name )[ -1 ] == suc.REPOSITORY_DEPENDENCY_DEFINITION_FILENAME:
                # Inspect the contents of the file to see if changeset_revision values are missing and if so, set them appropriately.
                altered, root_elem, error_message = commit_util.handle_repository_dependencies_definition( trans,
                                                                                                           uploaded_file_name,
                                                                                                           unpopulate=False )
                if error_message:
                    results_dict[ 'ok' ] = False
                    results_dict[ 'error_message' ] += error_message
                if altered:
                    tmp_filename = xml_util.create_and_write_tmp_file( root_elem )
                    shutil.move( tmp_filename, uploaded_file_name )
            elif os.path.split( uploaded_file_name )[ -1 ] == suc.TOOL_DEPENDENCY_DEFINITION_FILENAME:
                # Inspect the contents of the file to see if changeset_revision values are missing and if so, set them appropriately.
                altered, root_elem, error_message = commit_util.handle_tool_dependencies_definition( trans, uploaded_file_name )
                if error_message:
                    results_dict[ 'ok' ] = False
                    results_dict[ 'error_message' ] += error_message
                if altered:
                    tmp_filename = xml_util.create_and_write_tmp_file( root_elem )
                    shutil.move( tmp_filename, uploaded_file_name )
        commit_message = 'Imported from capsule %s' % str( capsule_file_name )
        # Send email notification to those that have registered to receive alerts for new repositories in this Tool Shed.
        new_repo_alert = True
        # Since the repository is new, the following must be False.
        remove_repo_files_not_in_tar = False
        ok, error_message, files_to_remove, content_alert_str, undesirable_dirs_removed, undesirable_files_removed = \
            commit_util.handle_directory_changes( trans,
                                                  repository,
                                                  full_path,
                                                  filenames_in_archive,
                                                  remove_repo_files_not_in_tar,
                                                  new_repo_alert,
                                                  commit_message,
                                                  undesirable_dirs_removed,
                                                  undesirable_files_removed )
        try:
            metadata_util.set_repository_metadata_due_to_new_tip( trans, repository, content_alert_str=content_alert_str )
        except Exception, e:
            log.debug( "Error setting metadata on repository %s created from imported archive %s: %s" % \
                ( str( repository.name ), str( archive_file_name ), str( e ) ) )
        results_dict[ 'ok' ] = ok
        results_dict[ 'error_message' ] += error_message
Esempio n. 6
0
 def upload_tar(self, trans, repository, tar, uploaded_file, upload_point,
                remove_repo_files_not_in_tar, commit_message,
                new_repo_alert):
     # Upload a tar archive of files.
     repo_dir = repository.repo_path(trans.app)
     repo = hg.repository(suc.get_configured_ui(), repo_dir)
     undesirable_dirs_removed = 0
     undesirable_files_removed = 0
     ok, message = commit_util.check_archive(repository, tar)
     if not ok:
         tar.close()
         uploaded_file.close()
         return ok, message, [], '', undesirable_dirs_removed, undesirable_files_removed
     else:
         if upload_point is not None:
             full_path = os.path.abspath(
                 os.path.join(repo_dir, upload_point))
         else:
             full_path = os.path.abspath(repo_dir)
         filenames_in_archive = []
         for tarinfo_obj in tar.getmembers():
             ok = os.path.basename(
                 tarinfo_obj.name) not in commit_util.UNDESIRABLE_FILES
             if ok:
                 for file_path_item in tarinfo_obj.name.split('/'):
                     if file_path_item in commit_util.UNDESIRABLE_DIRS:
                         undesirable_dirs_removed += 1
                         ok = False
                         break
             else:
                 undesirable_files_removed += 1
             if ok:
                 filenames_in_archive.append(tarinfo_obj.name)
         # Extract the uploaded tar to the load_point within the repository hierarchy.
         tar.extractall(path=full_path)
         tar.close()
         uploaded_file.close()
         for filename in filenames_in_archive:
             uploaded_file_name = os.path.join(full_path, filename)
             if os.path.split(uploaded_file_name)[
                     -1] == suc.REPOSITORY_DEPENDENCY_DEFINITION_FILENAME:
                 # Inspect the contents of the file to see if changeset_revision values are missing and if so, set them appropriately.
                 altered, root_elem = commit_util.handle_repository_dependencies_definition(
                     trans, uploaded_file_name)
                 if altered:
                     tmp_filename = xml_util.create_and_write_tmp_file(
                         root_elem)
                     shutil.move(tmp_filename, uploaded_file_name)
             elif os.path.split(uploaded_file_name)[
                     -1] == suc.TOOL_DEPENDENCY_DEFINITION_FILENAME:
                 # Inspect the contents of the file to see if changeset_revision values are missing and if so, set them appropriately.
                 altered, root_elem = commit_util.handle_tool_dependencies_definition(
                     trans, uploaded_file_name)
                 if altered:
                     tmp_filename = xml_util.create_and_write_tmp_file(
                         root_elem)
                     shutil.move(tmp_filename, uploaded_file_name)
         return commit_util.handle_directory_changes(
             trans, repository, full_path, filenames_in_archive,
             remove_repo_files_not_in_tar, new_repo_alert, commit_message,
             undesirable_dirs_removed, undesirable_files_removed)