def upload(metadata):
"""
Uploads a given set of packages
"""
product, release, version_string, revision_date, package_name = metadata
settings = SourceCollector.json_loads('{0}/{1}'.format(os.path.dirname(os.path.realpath(__file__)), 'settings.json'))
working_directory = settings['base_path'].format(product)
package_path = SourceCollector.package_path.format(working_directory)
redhat_folder = '{0}/redhat'.format(package_path)
package_info = settings['repositories']['packages']['redhat']
for destination in package_info:
server = destination['ip']
user = destination['user']
base_path = destination['base_path']
packages = [p for p in os.listdir(redhat_folder) if p.endswith('.rpm')]
for package in packages:
package_source_path = os.path.join(redhat_folder, package)
command = 'scp {0} {1}@{2}:{3}/pool/{4}'.format(package_source_path, user, server, base_path, release)
print('Uploading package {0}'.format(package))
SourceCollector.run(command,
working_directory=redhat_folder)
if len(packages) > 0:
# Cleanup existing files
command = 'ssh {0}@{1} {2}/cleanup_repo.py {2}/pool/{3}/'.format(user, server, base_path, release)
print(SourceCollector.run(command,
working_directory=redhat_folder))
command = 'ssh {0}@{1} createrepo --update {2}/dists/{3}'.format(user, server, base_path, release)
SourceCollector.run(command,
working_directory=redhat_folder)
def package(metadata):
"""
Packages a given package.
"""
product, release, version_string, revision_date, package_name = metadata
settings = SourceCollector.json_loads('{0}/{1}'.format(os.path.dirname(os.path.realpath(__file__)), 'settings.json'))
working_directory = settings['base_path'].format(product)
repo_path_code = SourceCollector.repo_path_code.format(working_directory)
package_path = SourceCollector.package_path.format(working_directory)
# Prepare
# /<pp>/debian
debian_folder = '{0}/debian'.format(package_path)
if os.path.exists(debian_folder):
shutil.rmtree(debian_folder)
# /<rp>/packaging/debian -> /<pp>/debian
shutil.copytree('{0}/packaging/debian'.format(repo_path_code), debian_folder)
# Rename tgz
# /<pp>/<packagename>_1.2.3.tar.gz -> /<pp>/debian/<packagename>_1.2.3.orig.tar.gz
shutil.copyfile('{0}/{1}_{2}.tar.gz'.format(package_path, package_name, version_string),
'{0}/{1}_{2}.orig.tar.gz'.format(debian_folder, package_name, version_string))
# /<pp>/debian/<packagename>-1.2.3/...
SourceCollector.run(command='tar -xzf {0}_{1}.orig.tar.gz'.format(package_name, version_string),
working_directory=debian_folder)
# Move the debian package metadata into the extracted source
# /<pp>/debian/debian -> /<pp>/debian/<packagename>-1.2.3/
SourceCollector.run(command='mv {0}/debian {0}/{1}-{2}/'.format(debian_folder, package_name, version_string),
working_directory=package_path)
# Build changelog entry
with open('{0}/{1}-{2}/debian/changelog'.format(debian_folder, package_name, version_string), 'w') as changelog_file:
changelog_file.write("""{0} ({1}-1) {2}; urgency=low
* For changes, see individual changelogs
-- Packaging System <*****@*****.**> {3}
""".format(package_name, version_string, release, revision_date.strftime('%a, %d %b %Y %H:%M:%S +0000')))
# Some more tweaks
SourceCollector.run(command='chmod 770 {0}/{1}-{2}/debian/rules'.format(debian_folder, package_name, version_string),
working_directory=package_path)
SourceCollector.run(command="sed -i -e 's/__NEW_VERSION__/{0}/' *.*".format(version_string),
working_directory='{0}/{1}-{2}/debian'.format(debian_folder, package_name, version_string))
# Build the package
SourceCollector.run(command='dpkg-buildpackage',
working_directory='{0}/{1}-{2}'.format(debian_folder, package_name, version_string))
def upload(metadata):
"""
Uploads a given set of packages
"""
product, release, version_string, revision_date, package_name = metadata
settings = SourceCollector.json_loads('{0}/{1}'.format(os.path.dirname(os.path.realpath(__file__)), 'settings.json'))
working_directory = settings['base_path'].format(product)
package_path = SourceCollector.package_path.format(working_directory)
package_info = settings['repositories']['packages']['debian']
for destination in package_info:
server = destination['ip']
user = destination['user']
base_path = destination['base_path']
upload_path = os.path.join(base_path, release)
print("Uploading {0} {1}".format(package_name, version_string))
debs_path = os.path.join(package_path, 'debian')
deb_packages = [filename for filename in os.listdir(debs_path) if filename.endswith('.deb')]
create_releasename_command = "ssh {0}@{1} mkdir -p {2}".format(user, server, upload_path)
SourceCollector.run(command=create_releasename_command,
working_directory=debs_path)
for deb_package in deb_packages:
source_path = os.path.join(debs_path, deb_package)
destination_path = os.path.join(upload_path, deb_package)
check_package_command = "ssh {0}@{1} ls {2}".format(user, server, upload_path)
existing_packages = SourceCollector.run(command=check_package_command,
working_directory=debs_path).split()
upload = deb_package not in existing_packages
if upload is False:
print("Package already uploaded, done...")
else:
scp_command = "scp {0} {1}@{2}:{3}".format(source_path, user, server, destination_path)
SourceCollector.run(command=scp_command,
working_directory=debs_path)
remote_command = "ssh {0}@{1} reprepro -Vb {2}/debian includedeb {3} {4}".format(user, server, base_path, release, destination_path)
SourceCollector.run(command=remote_command,
working_directory=debs_path)
def package(metadata):
"""
Packages a given package.
"""
product, release, version_string, revision_date, package_name = metadata
settings = SourceCollector.json_loads('{0}/{1}'.format(os.path.dirname(os.path.realpath(__file__)), 'settings.json'))
working_directory = settings['base_path'].format(product)
repo_path_code = SourceCollector.repo_path_code.format(working_directory)
package_path = SourceCollector.package_path.format(working_directory)
# Prepare
redhat_folder = '{0}/redhat'.format(package_path)
if os.path.exists(redhat_folder):
shutil.rmtree(redhat_folder)
os.mkdir(redhat_folder)
# extract tar.gz to redhat_folder
shutil.copyfile('{0}/{1}_{2}.tar.gz'.format(package_path, package_name, version_string),
'{0}/{1}_{2}.orig.tar.gz'.format(redhat_folder, package_name, version_string))
# /<pp>/debian/<packagename>-1.2.3/...
SourceCollector.run(command='tar -xzf {0}_{1}.orig.tar.gz'.format(package_name, version_string),
working_directory=redhat_folder)
code_source_path = '{0}/{1}-{2}'.format(redhat_folder, package_name, version_string)
# copy packaging
source_packaging_path = os.path.join(repo_path_code, 'packaging')
dest_packaging_path = os.path.join(code_source_path, 'packaging')
if os.path.exists(source_packaging_path):
shutil.copytree(source_packaging_path, dest_packaging_path)
# load config
config_dir = '{0}/packaging/redhat/cfgs'.format(repo_path_code)
packages = os.listdir(config_dir)
for package in packages:
package_filename = '{0}/{1}'.format(config_dir, package)
package_cfg = RawConfigParser()
package_cfg.read(package_filename)
package_name = package_cfg.get('main', 'name')
dirs = package_cfg.get('main', 'dirs')
files = package_cfg.get('main', 'files')
depends_packages = package_cfg.get('main', 'depends').replace('$Version', version_string.replace('-', '_'))
depends = ""
if depends_packages != '':
depends = []
for depends_package in depends_packages.split(','):
depends.append('-d "{0}"'.format(depends_package.strip()))
depends = ' '.join(depends)
package_root_path = os.path.join(package_path, package_name)
if os.path.exists(package_root_path):
shutil.rmtree(package_root_path)
os.mkdir(package_root_path)
for dir_ in dirs.split(','):
dir_ = dir_.strip()
if dir_ != "''":
source_dir, dest_location = dir_.split('=')
# source_dir = dir to copy - from repo root
# dest_location = dir under which to copy the source_dir
source_full_path = os.path.join(code_source_path, source_dir.strip())
dest_full_path = os.path.join(package_root_path, dest_location.strip())
shutil.copytree(source_full_path, dest_full_path)
for file_ in files.split(','):
file_ = file_.strip()
if file_ != "''" and file_ != '':
source_file, dest_location = file_.split('=')
source_full_path = os.path.join(code_source_path, source_file.strip())
dest_full_path = os.path.join(package_root_path, dest_location.strip())
if not os.path.exists(dest_full_path):
os.makedirs(dest_full_path)
shutil.copy(source_full_path, dest_full_path)
before_install, after_install = ' ', ' '
script_root = '{0}/packaging/redhat/scripts'.format(code_source_path)
before_install_script = '{0}.before-install.sh'.format(package_name)
before_install_script_path = os.path.join(script_root, before_install_script)
if os.path.exists(before_install_script_path):
before_install = ' --before-install {0} '.format(before_install_script_path)
after_install_script = '{0}.after-install.sh'.format(package_name)
after_install_script_path = os.path.join(script_root, after_install_script)
if os.path.exists(after_install_script_path):
after_install = ' --after-install {0} '.format(after_install_script_path)
SourceCollector.run(command="sed -i -e 's/$Version/{0}/g' {1}".format(version_string,
after_install_script_path),
working_directory='{0}'.format(script_root))
params = {'version': version_string,
'package_name': package_cfg.get('main', 'name'),
'summary': package_cfg.get('main', 'summary'),
'license': package_cfg.get('main', 'license'),
'URL': package_cfg.get('main', 'URL'),
'source': package_cfg.get('main', 'source'),
'arch': package_cfg.get('main', 'arch'),
'description': package_cfg.get('main', 'description'),
'maintainer': package_cfg.get('main', 'maintainer'),
'depends': depends,
'package_root': package_root_path,
'before_install': before_install,
'after_install': after_install,
}
command = """fpm -s dir -t rpm -n {package_name} -v {version} --description "{description}" --maintainer "{maintainer}" --license "{license}" --url {URL} -a {arch} --vendor "Open vStorage" {depends}{before_install}{after_install} --prefix=/ -C {package_root}""".format(**params)
SourceCollector.run(command,
working_directory=redhat_folder)
print(os.listdir(redhat_folder))
if __name__ == '__main__':
parser = OptionParser(description='Open vStorage repo maintenance')
parser.add_option('-f', '--from-release', dest='from_release')
parser.add_option('-t', '--to-release', dest='to_release')
parser.add_option('-s', '--skip', dest='skip')
parser.add_option('-d',
'--dry-run',
dest='dry_run',
action='store_true',
default=False)
options, args = parser.parse_args()
dry_run = options.dry_run
skips = tuple(options.skip.split(',')) if options.skip is not None else ()
settings = SourceCollector.json_loads('{0}/{1}'.format(
os.path.dirname(os.path.realpath(__file__)), 'settings.json'))
package_info = settings['repositories']['packages'].get('debian', [])
for destination in package_info:
server = destination['ip']
user = destination['user']
base_path = destination['base_path']
print 'Processing {0}@{1}'.format(user, server)
print ' Reading releases'
source_package_map = {}
destination_package_map = {}
for release, package_map in {
options.from_release: source_package_map,
