def process_all_packages(pkgmgr, client_dir, upload_paths, remove=False):
"""Process a full upload of packages as a directory upload."""
test_dir = os.path.join(client_dir, "tests")
site_test_dir = os.path.join(client_dir, "site_tests")
dep_dir = os.path.join(client_dir, "deps")
prof_dir = os.path.join(client_dir, "profilers")
# Directory where all are kept
temp_dir = tempfile.mkdtemp()
try:
packages.check_diskspace(temp_dir)
except packages.RepoDiskFull:
print ("Temp destination for packages is full %s, aborting upload"
% temp_dir)
os.rmdir(temp_dir)
sys.exit(1)
# process tests
tests_list = get_subdir_list('tests', client_dir)
tests = ','.join(tests_list)
# process site_tests
site_tests_list = get_subdir_list('site_tests', client_dir)
site_tests = ','.join(site_tests_list)
# process deps
deps_list = get_subdir_list('deps', client_dir)
deps = ','.join(deps_list)
# process profilers
profilers_list = get_subdir_list('profilers', client_dir)
profilers = ','.join(profilers_list)
# Update md5sum
if not remove:
tar_packages(pkgmgr, 'profiler', profilers, prof_dir, temp_dir)
tar_packages(pkgmgr, 'dep', deps, dep_dir, temp_dir)
tar_packages(pkgmgr, 'test', site_tests, client_dir, temp_dir)
tar_packages(pkgmgr, 'test', tests, client_dir, temp_dir)
tar_packages(pkgmgr, 'client', 'autotest', client_dir, temp_dir)
cwd = os.getcwd()
os.chdir(temp_dir)
client_utils.system('md5sum * > packages.checksum')
os.chdir(cwd)
for path in upload_paths:
print "Uploading to: " + path
pkgmgr.upload_pkg(temp_dir, path)
client_utils.run('rm -rf ' + temp_dir)
else:
for repo_url in upload_paths:
process_packages(pkgmgr, 'test', tests, client_dir, repo_url,
remove=remove)
process_packages(pkgmgr, 'test', site_tests, client_dir, repo_url,
remove=remove)
process_packages(pkgmgr, 'client', 'autotest', client_dir, repo_url,
remove=remove)
process_packages(pkgmgr, 'dep', deps, dep_dir, repo_url,
remove=remove)
process_packages(pkgmgr, 'profiler', profilers, prof_dir, repo_url,
remove=remove)
def process_all_packages(pkgmgr, client_dir, action):
"""Process a full upload of packages as a directory upload."""
dep_dir = os.path.join(client_dir, "deps")
prof_dir = os.path.join(client_dir, "profilers")
# Directory where all are kept
temp_dir = tempfile.mkdtemp()
try:
packages.check_diskspace(temp_dir)
except error.RepoDiskFullError, e:
print("Temp destination for packages is full %s, aborting upload: %s" %
(temp_dir, e))
os.rmdir(temp_dir)
sys.exit(1)
def process_packages(pkgmgr, pkg_type, pkg_names, src_dir, repo_url,
remove=False):
exclude_string = ' .'
names = [p.strip() for p in pkg_names.split(',')]
for name in names:
print "Processing %s ... " % name
if pkg_type=='client':
pkg_dir = src_dir
exclude_string = get_exclude_string(pkg_dir)
elif pkg_type=='test':
# if the package is a test then look whether it is in client/tests
# or client/site_tests
pkg_dir = os.path.join(get_test_dir(name, src_dir), name)
else:
# for the profilers and deps
pkg_dir = os.path.join(src_dir, name)
pkg_name = pkgmgr.get_tarball_name(name, pkg_type)
if not remove:
# Tar the source and upload
try:
temp_dir = tempfile.mkdtemp()
try:
packages.check_diskspace(temp_dir)
except packages.RepoDiskFull:
msg = ("Temporary directory for packages does not have "
"enough space available")
raise packages.RepoDiskFull(msg)
tarball_path = pkgmgr.tar_package(pkg_name, pkg_dir,
temp_dir, exclude_string)
pkgmgr.upload_pkg(tarball_path, repo_url, update_checksum=True)
finally:
# remove the temporary directory
shutil.rmtree(temp_dir)
else:
pkgmgr.remove_pkg(pkg_name, repo_url, remove_checksum=True)
print "Done."
def process_packages(pkgmgr,
pkg_type,
pkg_names,
src_dir,
action,
dest_dir=None):
"""Method to upload or remove package depending on the flag passed to it.
If tar_only is set to True, this routine is solely used to generate a
tarball and compute the md5sum from that tarball.
If the tar_only flag is True, then the remove flag is ignored.
"""
exclude_string = ' .'
names = [p.strip() for p in pkg_names.split(',')]
for name in names:
print "process_packages: Processing %s ... " % name
if pkg_type == 'client':
pkg_dir = src_dir
exclude_string = get_exclude_string(pkg_dir)
elif pkg_type == 'test':
# if the package is a test then look whether it is in client/tests
# or client/site_tests
pkg_dir = os.path.join(get_test_dir(name, src_dir), name)
else:
# for the profilers and deps
pkg_dir = os.path.join(src_dir, name)
pkg_name = pkgmgr.get_tarball_name(name, pkg_type)
if action == ACTION_TAR_ONLY:
# We don't want any pre-existing tarballs and checksums to
# be repackaged, so we should purge these.
exclude_string_tar = (
(' --exclude="**%s" --exclude="**%s.checksum" ' %
(pkg_name, pkg_name)) + exclude_string)
build_dir = get_build_dir(name, dest_dir, pkg_type)
try:
packages.check_diskspace(build_dir)
except error.RepoDiskFullError as e:
msg = ("Work_dir directory for packages %s does not have "
"enough space available: %s" % (build_dir, e))
raise error.RepoDiskFullError(msg)
tarball_path = pkgmgr.tar_package(pkg_name, pkg_dir, build_dir,
exclude_string_tar)
# Create the md5 hash too.
md5sum = pkgmgr.compute_checksum(tarball_path)
md5sum_filepath = os.path.join(build_dir, pkg_name + '.checksum')
with open(md5sum_filepath, "w") as f:
f.write(md5sum)
elif action == ACTION_UPLOAD:
# Tar the source and upload
temp_dir = tempfile.mkdtemp()
try:
try:
packages.check_diskspace(temp_dir)
except error.RepoDiskFullError, e:
msg = ("Temporary directory for packages %s does not have "
"enough space available: %s" % (temp_dir, e))
raise error.RepoDiskFullError(msg)
# Check if tarball already exists. If it does, and the checksum
# is the same as what is in the checksum dictionary, then don't
# create a tarball again.
tarball_path = os.path.join(pkg_dir, pkg_name)
if os.path.exists(tarball_path) and pkgmgr.compare_checksum(
tarball_path):
print("process_packages: Tarball %s already exists" %
tarball_path)
else:
tarball_path = pkgmgr.tar_package(pkg_name, pkg_dir,
temp_dir, exclude_string)
pkgmgr.upload_pkg(tarball_path, update_checksum=True)
finally:
