def fetchArtifacts(remoteRepoUrl, localRepoDir, artifactList, classifiers,
excludedTypes, checksumMode):
"""Create a Maven repository based on a remote repository url and a list of artifacts"""
logging.info('Retrieving artifacts from repository: %s', remoteRepoUrl)
if not os.path.exists(localRepoDir):
os.makedirs(localRepoDir)
parsedUrl = urlparse.urlparse(remoteRepoUrl)
protocol = parsedUrl[0]
repoPath = parsedUrl[2]
if protocol == 'http' or protocol == 'https':
# Create thread pool
pool = ThreadPool(maven_repo_util.MAX_THREADS)
errors = Queue.Queue()
mkdirLock = threading.Lock()
for artifact in artifactList:
if artifact.artifactType in excludedTypes:
logging.info(
"Skipping download of %s:%s:%s:%s because of excluded type",
artifact.groupId, artifact.artifactId,
artifact.artifactType, artifact.version)
continue
if artifact.isSnapshot():
maven_repo_util.updateSnapshotVersionSuffix(
artifact, remoteRepoUrl)
pool.apply_async(downloadArtifacts, [
remoteRepoUrl, localRepoDir, artifact, classifiers,
checksumMode, mkdirLock, errors
])
# Close pool and wait till all workers are finnished
pool.close()
pool.join()
# If one of the workers threw error, exit with 1
if not errors.empty():
sys.exit(1)
elif protocol == 'file':
repoPath = remoteRepoUrl.replace('file://', '')
for artifact in artifactList:
if artifact.artifactType in excludedTypes:
logging.info(
"Skipping copy of %s:%s:%s:%s because of excluded type",
artifact.groupId, artifact.artifactId,
artifact.artifactType, artifact.version)
continue
if artifact.isSnapshot():
maven_repo_util.updateSnapshotVersionSuffix(
artifact, remoteRepoUrl)
copyArtifact(repoPath, localRepoDir, artifact, classifiers,
checksumMode)
else:
logging.error('Unknown protocol: %s', protocol)
def fetchArtifacts(remoteRepoUrl, localRepoDir, artifactList, classifiers, excludedTypes, checksumMode):
"""Create a Maven repository based on a remote repository url and a list of artifacts"""
logging.info('Retrieving artifacts from repository: %s', remoteRepoUrl)
if not os.path.exists(localRepoDir):
os.makedirs(localRepoDir)
parsedUrl = urlparse.urlparse(remoteRepoUrl)
protocol = parsedUrl[0]
repoPath = parsedUrl[2]
if protocol == 'http' or protocol == 'https':
# Create thread pool
pool = ThreadPool(maven_repo_util.MAX_THREADS)
errors = Queue.Queue()
mkdirLock = threading.Lock()
for artifact in artifactList:
if artifact.artifactType in excludedTypes:
logging.info("Skipping download of %s:%s:%s:%s because of excluded type", artifact.groupId,
artifact.artifactId, artifact.artifactType, artifact.version)
continue
if artifact.isSnapshot():
maven_repo_util.updateSnapshotVersionSuffix(artifact, remoteRepoUrl)
pool.apply_async(
downloadArtifacts,
[remoteRepoUrl, localRepoDir, artifact, classifiers, checksumMode, mkdirLock, errors]
)
# Close pool and wait till all workers are finnished
pool.close()
pool.join()
# If one of the workers threw error, exit with 1
if not errors.empty():
sys.exit(1)
elif protocol == 'file':
repoPath = remoteRepoUrl.replace('file://', '')
for artifact in artifactList:
if artifact.artifactType in excludedTypes:
logging.info("Skipping copy of %s:%s:%s:%s because of excluded type", artifact.groupId,
artifact.artifactId, artifact.artifactType, artifact.version)
continue
if artifact.isSnapshot():
maven_repo_util.updateSnapshotVersionSuffix(artifact, remoteRepoUrl)
copyArtifact(repoPath, localRepoDir, artifact, classifiers, checksumMode)
else:
logging.error('Unknown protocol: %s', protocol)
def fetchArtifactList(remoteRepoUrl, localRepoDir, artifactList, checksumMode):
"""Create a Maven repository based on a remote repository url and a list of artifacts"""
logging.info('Retrieving artifacts from repository: %s', remoteRepoUrl)
if not os.path.exists(localRepoDir):
os.makedirs(localRepoDir)
parsedUrl = urlparse.urlparse(remoteRepoUrl)
protocol = parsedUrl[0]
repoPath = parsedUrl[2]
if protocol == 'http' or protocol == 'https':
# Create thread pool
pool = ThreadPool(maven_repo_util.MAX_THREADS)
errors = Queue()
mkdirLock = Lock()
filesetLock = Lock()
fileset = set([])
for artifact in artifactList:
if artifact.isSnapshot():
maven_repo_util.updateSnapshotVersionSuffix(
artifact, remoteRepoUrl)
pool.apply_async(downloadArtifacts, [
remoteRepoUrl, localRepoDir, artifact, checksumMode, mkdirLock,
filesetLock, fileset, errors
])
# Close pool and wait till all workers are finished
pool.close()
pool.join()
# If one of the workers threw an error, log it
if not errors.empty():
logging.error(
"During fetching files from repository %s %i error(s) occurred.",
remoteRepoUrl, errors.qsize())
elif protocol == 'file':
repoPath = remoteRepoUrl.replace('file://', '')
for artifact in artifactList:
if artifact.isSnapshot():
maven_repo_util.updateSnapshotVersionSuffix(
artifact, remoteRepoUrl)
copyArtifact(repoPath, localRepoDir, artifact, checksumMode)
else:
logging.error('Unknown protocol: %s', protocol)
def fetchArtifactList(remoteRepoUrl, localRepoDir, artifactList, checksumMode, threadnum):
"""Create a Maven repository based on a remote repository url and a list of artifacts"""
logging.info('Retrieving artifacts from repository: %s', remoteRepoUrl)
if not os.path.exists(localRepoDir):
os.makedirs(localRepoDir)
parsedUrl = urlparse.urlparse(remoteRepoUrl)
protocol = parsedUrl[0]
repoPath = parsedUrl[2]
if protocol == 'http' or protocol == 'https':
# Create thread pool
pool = ThreadPool(threadnum)
errors = Queue()
mkdirLock = Lock()
filesetLock = Lock()
fileset = set([])
for artifact in artifactList:
if artifact.isSnapshot():
maven_repo_util.updateSnapshotVersionSuffix(artifact, remoteRepoUrl)
pool.apply_async(
downloadArtifacts,
[remoteRepoUrl, localRepoDir, artifact, checksumMode, mkdirLock, filesetLock, fileset, errors]
)
# Close pool and wait till all workers are finished
pool.close()
pool.join()
# If one of the workers threw an error, log it
if not errors.empty():
logging.error("During fetching files from repository %s %i error(s) occurred.", remoteRepoUrl,
errors.qsize())
elif protocol == 'file':
repoPath = remoteRepoUrl.replace('file://', '')
for artifact in artifactList:
if artifact.isSnapshot():
maven_repo_util.updateSnapshotVersionSuffix(artifact, remoteRepoUrl)
copyArtifact(repoPath, localRepoDir, artifact, checksumMode)
else:
logging.error('Unknown protocol: %s', protocol)
