def fetchArtifacts(remoteRepoUrl, localRepoDir, artifactList, classifiers, excludedTypes, checksumMode): """Create a Maven repository based on a remote repository url and a list of artifacts""" logging.info('Retrieving artifacts from repository: %s', remoteRepoUrl) if not os.path.exists(localRepoDir): os.makedirs(localRepoDir) parsedUrl = urlparse.urlparse(remoteRepoUrl) protocol = parsedUrl[0] repoPath = parsedUrl[2] if protocol == 'http' or protocol == 'https': # Create thread pool pool = ThreadPool(maven_repo_util.MAX_THREADS) errors = Queue.Queue() mkdirLock = threading.Lock() for artifact in artifactList: if artifact.artifactType in excludedTypes: logging.info( "Skipping download of %s:%s:%s:%s because of excluded type", artifact.groupId, artifact.artifactId, artifact.artifactType, artifact.version) continue if artifact.isSnapshot(): maven_repo_util.updateSnapshotVersionSuffix( artifact, remoteRepoUrl) pool.apply_async(downloadArtifacts, [ remoteRepoUrl, localRepoDir, artifact, classifiers, checksumMode, mkdirLock, errors ]) # Close pool and wait till all workers are finnished pool.close() pool.join() # If one of the workers threw error, exit with 1 if not errors.empty(): sys.exit(1) elif protocol == 'file': repoPath = remoteRepoUrl.replace('file://', '') for artifact in artifactList: if artifact.artifactType in excludedTypes: logging.info( "Skipping copy of %s:%s:%s:%s because of excluded type", artifact.groupId, artifact.artifactId, artifact.artifactType, artifact.version) continue if artifact.isSnapshot(): maven_repo_util.updateSnapshotVersionSuffix( artifact, remoteRepoUrl) copyArtifact(repoPath, localRepoDir, artifact, classifiers, checksumMode) else: logging.error('Unknown protocol: %s', protocol)
def fetchArtifacts(remoteRepoUrl, localRepoDir, artifactList, classifiers, excludedTypes, checksumMode): """Create a Maven repository based on a remote repository url and a list of artifacts""" logging.info('Retrieving artifacts from repository: %s', remoteRepoUrl) if not os.path.exists(localRepoDir): os.makedirs(localRepoDir) parsedUrl = urlparse.urlparse(remoteRepoUrl) protocol = parsedUrl[0] repoPath = parsedUrl[2] if protocol == 'http' or protocol == 'https': # Create thread pool pool = ThreadPool(maven_repo_util.MAX_THREADS) errors = Queue.Queue() mkdirLock = threading.Lock() for artifact in artifactList: if artifact.artifactType in excludedTypes: logging.info("Skipping download of %s:%s:%s:%s because of excluded type", artifact.groupId, artifact.artifactId, artifact.artifactType, artifact.version) continue if artifact.isSnapshot(): maven_repo_util.updateSnapshotVersionSuffix(artifact, remoteRepoUrl) pool.apply_async( downloadArtifacts, [remoteRepoUrl, localRepoDir, artifact, classifiers, checksumMode, mkdirLock, errors] ) # Close pool and wait till all workers are finnished pool.close() pool.join() # If one of the workers threw error, exit with 1 if not errors.empty(): sys.exit(1) elif protocol == 'file': repoPath = remoteRepoUrl.replace('file://', '') for artifact in artifactList: if artifact.artifactType in excludedTypes: logging.info("Skipping copy of %s:%s:%s:%s because of excluded type", artifact.groupId, artifact.artifactId, artifact.artifactType, artifact.version) continue if artifact.isSnapshot(): maven_repo_util.updateSnapshotVersionSuffix(artifact, remoteRepoUrl) copyArtifact(repoPath, localRepoDir, artifact, classifiers, checksumMode) else: logging.error('Unknown protocol: %s', protocol)
def fetchArtifactList(remoteRepoUrl, localRepoDir, artifactList, checksumMode): """Create a Maven repository based on a remote repository url and a list of artifacts""" logging.info('Retrieving artifacts from repository: %s', remoteRepoUrl) if not os.path.exists(localRepoDir): os.makedirs(localRepoDir) parsedUrl = urlparse.urlparse(remoteRepoUrl) protocol = parsedUrl[0] repoPath = parsedUrl[2] if protocol == 'http' or protocol == 'https': # Create thread pool pool = ThreadPool(maven_repo_util.MAX_THREADS) errors = Queue() mkdirLock = Lock() filesetLock = Lock() fileset = set([]) for artifact in artifactList: if artifact.isSnapshot(): maven_repo_util.updateSnapshotVersionSuffix( artifact, remoteRepoUrl) pool.apply_async(downloadArtifacts, [ remoteRepoUrl, localRepoDir, artifact, checksumMode, mkdirLock, filesetLock, fileset, errors ]) # Close pool and wait till all workers are finished pool.close() pool.join() # If one of the workers threw an error, log it if not errors.empty(): logging.error( "During fetching files from repository %s %i error(s) occurred.", remoteRepoUrl, errors.qsize()) elif protocol == 'file': repoPath = remoteRepoUrl.replace('file://', '') for artifact in artifactList: if artifact.isSnapshot(): maven_repo_util.updateSnapshotVersionSuffix( artifact, remoteRepoUrl) copyArtifact(repoPath, localRepoDir, artifact, checksumMode) else: logging.error('Unknown protocol: %s', protocol)
def fetchArtifactList(remoteRepoUrl, localRepoDir, artifactList, checksumMode, threadnum): """Create a Maven repository based on a remote repository url and a list of artifacts""" logging.info('Retrieving artifacts from repository: %s', remoteRepoUrl) if not os.path.exists(localRepoDir): os.makedirs(localRepoDir) parsedUrl = urlparse.urlparse(remoteRepoUrl) protocol = parsedUrl[0] repoPath = parsedUrl[2] if protocol == 'http' or protocol == 'https': # Create thread pool pool = ThreadPool(threadnum) errors = Queue() mkdirLock = Lock() filesetLock = Lock() fileset = set([]) for artifact in artifactList: if artifact.isSnapshot(): maven_repo_util.updateSnapshotVersionSuffix(artifact, remoteRepoUrl) pool.apply_async( downloadArtifacts, [remoteRepoUrl, localRepoDir, artifact, checksumMode, mkdirLock, filesetLock, fileset, errors] ) # Close pool and wait till all workers are finished pool.close() pool.join() # If one of the workers threw an error, log it if not errors.empty(): logging.error("During fetching files from repository %s %i error(s) occurred.", remoteRepoUrl, errors.qsize()) elif protocol == 'file': repoPath = remoteRepoUrl.replace('file://', '') for artifact in artifactList: if artifact.isSnapshot(): maven_repo_util.updateSnapshotVersionSuffix(artifact, remoteRepoUrl) copyArtifact(repoPath, localRepoDir, artifact, checksumMode) else: logging.error('Unknown protocol: %s', protocol)