def upload(source_metadata):
"""
Uploads a given set of packages
"""
filename = "{0}/../settings.cfg".format(os.path.dirname(os.path.abspath(__file__)))
settings = RawConfigParser()
settings.read(filename)
package_name = settings.get("packaging", "package_name")
package_path = SourceCollector.package_path.format(settings.get("packaging", "working_dir"), package_name)
version_string = source_metadata[1]
new_package = version_string not in SourceCollector.run(
command="ssh [email protected] \"grep '{0}_{1}-1_amd64' /data/www/apt/*/Packages\" || true".format(
package_name, version_string
),
working_directory=package_path,
)
print "Uploading {0} package: {1}".format(
"new" if new_package else "existing", "{0}_{1}-1_amd64".format(package_name, version_string)
)
SourceCollector.run(
command="dput -c {0}/debian/dput.cfg ovs-apt {0}/debian/{1}_{2}-1_amd64.changes".format(
package_path, package_name, version_string
),
working_directory=package_path,
)
SourceCollector.run(
command='ssh [email protected] "mini-dinstall -b{0}"'.format(
"" if new_package else " --no-db"
),
working_directory=package_path,
)
def upload(source_metadata):
"""
Uploads a given set of packages
"""
_ = source_metadata
root_path = ROOT_PATH
filename = '{0}/../settings.cfg'.format(root_path)
settings = RawConfigParser()
settings.read(filename)
package_name = settings.get('packaging', 'package_name')
package_path = SourceCollector.package_path.format(settings.get('packaging', 'working_dir'), package_name)
redhat_folder = '{0}/redhat'.format(package_path)
destination_folder = '/usr/share/repo/CentOS/7/x86_64/'
destination_server = '172.20.3.17'
user = '******'
packages = os.listdir(redhat_folder)
for package in packages:
package_source_path = os.path.join(redhat_folder, package)
command = 'scp {0} {1}@{2}:{3}'.format(package_source_path, user, destination_server, destination_folder)
print('Uploading package {0}'.format(package))
SourceCollector.run(command,
working_directory=redhat_folder)
if len(packages) > 0:
command = 'ssh {0}@{1} createrepo --update {2}'.format(user, destination_server, destination_folder)
SourceCollector.run(command,
working_directory=redhat_folder)
def upload(source_metadata):
"""
Uploads a given set of packages
"""
filename = '{0}/../settings.cfg'.format(
os.path.dirname(os.path.abspath(__file__)))
settings = RawConfigParser()
settings.read(filename)
package_name = settings.get('packaging', 'package_name')
package_path = SourceCollector.package_path.format(
settings.get('packaging', 'working_dir'), package_name)
version_string = source_metadata[1]
new_package = version_string not in SourceCollector.run(
command=
'ssh [email protected] "grep \'{0}_{1}-1_amd64\' /data/www/apt/*/Packages" || true'
.format(package_name, version_string),
working_directory=package_path)
print 'Uploading {0} package: {1}'.format(
'new' if new_package else 'existing',
'{0}_{1}-1_amd64'.format(package_name, version_string))
SourceCollector.run(
command=
'dput -c {0}/debian/dput.cfg ovs-apt {0}/debian/{1}_{2}-1_amd64.changes'
.format(package_path, package_name, version_string),
working_directory=package_path)
SourceCollector.run(
command=
'ssh [email protected] "mini-dinstall -b{0}"'.
format('' if new_package else ' --no-db'),
working_directory=package_path)
def package(source_metadata):
"""
Packages a given package.
"""
distribution, version_string, revision_date = source_metadata
filename = '{0}/../settings.cfg'.format(os.path.dirname(os.path.abspath(__file__)))
settings = RawConfigParser()
settings.read(filename)
package_name = settings.get('packaging', 'package_name')
repo_path_code = SourceCollector.repo_path_code.format(settings.get('packaging', 'working_dir'), package_name)
package_path = SourceCollector.package_path.format(settings.get('packaging', 'working_dir'), package_name)
# Prepare
# /<pp>/debian
debian_folder = '{0}/debian'.format(package_path)
if os.path.exists(debian_folder):
shutil.rmtree(debian_folder)
# /<rp>/packaging/debian -> /<pp>/debian
shutil.copytree('{0}/packaging/debian'.format(repo_path_code), debian_folder)
# Rename tgz
# /<pp>/<packagename>_1.2.3.tar.gz -> /<pp>/debian/<packagename>_1.2.3.orig.tar.gz
shutil.copyfile('{0}/{1}_{2}.tar.gz'.format(package_path, package_name, version_string),
'{0}/{1}_{2}.orig.tar.gz'.format(debian_folder, package_name, version_string))
# /<pp>/debian/<packagename>-1.2.3/...
SourceCollector.run(command='tar -xzf {0}_{1}.orig.tar.gz'.format(package_name, version_string),
working_directory=debian_folder)
# Move the debian package metadata into the extracted source
# /<pp>/debian/debian -> /<pp>/debian/<packagename>-1.2.3/
SourceCollector.run(command='mv {0}/debian {0}/{1}-{2}/'.format(debian_folder, package_name, version_string),
working_directory=package_path)
# Build changelog entry
with open('{0}/{1}-{2}/debian/changelog'.format(debian_folder, package_name, version_string), 'w') as changelog_file:
changelog_file.write("""{0} ({1}-1) {2}; urgency=low
* For changes, see individual changelogs
-- Packaging System <*****@*****.**> {3}
""".format(package_name, version_string, distribution, revision_date.strftime('%a, %d %b %Y %H:%M:%S +0000')))
# Some more tweaks
SourceCollector.run(command='chmod 770 {0}/{1}-{2}/debian/rules'.format(debian_folder, package_name, version_string),
working_directory=package_path)
SourceCollector.run(command="sed -i -e 's/__NEW_VERSION__/{0}/' *.*".format(version_string),
working_directory='{0}/{1}-{2}/debian'.format(debian_folder, package_name, version_string))
# Build the package
SourceCollector.run(command='dpkg-buildpackage',
working_directory='{0}/{1}-{2}'.format(debian_folder, package_name, version_string))
def upload(source_metadata):
"""
Uploads a given set of packages
"""
root_path = ROOT_PATH
filename = '{0}/../settings.cfg'.format(root_path)
settings = RawConfigParser()
settings.read(filename)
# Get release name from the repo code settings.cfg not master settings.cfg
repo_path_code = SourceCollector.repo_path_code.format(settings.get('packaging', 'working_dir'), settings.get('packaging', 'package_name'))
filename = '{0}/packaging/settings.cfg'.format(repo_path_code)
code_settings = RawConfigParser()
code_settings.read(filename)
releasename = code_settings.get('version', 'releasename').lower()
target, version_string, _ = source_metadata
package_name = settings.get('packaging', 'package_name')
package_path = SourceCollector.package_path.format(settings.get('packaging', 'working_dir'), package_name)
redhat_folder = '{0}/redhat'.format(package_path)
destination_folder = '/usr/share/repo/CentOS/7/x86_64/'
destination_server = '172.20.3.17'
user = '******'
packages = os.listdir(redhat_folder)
for package in packages:
package_source_path = os.path.join(redhat_folder, package)
command = 'scp {0} {1}@{2}:{3}/pool/{4}/{5}'.format(package_source_path, user, destination_server, destination_folder, target, releasename)
print('Uploading package {0}'.format(package))
SourceCollector.run(command,
working_directory=redhat_folder)
if len(packages) > 0:
# Cleanup existing files
command = 'ssh {0}@{1} {2}/cleanup_repo.py {2}/pool/{3}/{4}'.format(user, destination_server, destination_folder, target, releasename)
print(SourceCollector.run(command,
working_directory=redhat_folder))
command = 'ssh {0}@{1} createrepo --update {2}/dists/{3}-{4}'.format(user, destination_server, destination_folder, releasename, target)
SourceCollector.run(command,
working_directory=redhat_folder)
def upload(source_metadata):
"""
Uploads a given set of packages
"""
filename = '{0}/../settings.cfg'.format(os.path.dirname(os.path.abspath(__file__)))
settings = RawConfigParser()
settings.read(filename)
package_name = settings.get('packaging', 'package_name')
package_path = SourceCollector.package_path.format(settings.get('packaging', 'working_dir'), package_name)
version_string = source_metadata[1]
new_package = version_string not in SourceCollector.run(command='ssh [email protected] "grep \'{0}_{1}-1_amd64\' /data/www/apt/*/Packages" || true'.format(package_name,
version_string),
working_directory=package_path)
print 'Uploading {0} package: {1}'.format('new' if new_package else 'existing', '{0}_{1}-1_amd64'.format(package_name, version_string))
SourceCollector.run(command='dput -c {0}/debian/dput.cfg ovs-apt {0}/debian/{1}_{2}-1_amd64.changes'.format(package_path, package_name, version_string),
working_directory=package_path)
SourceCollector.run(command='ssh [email protected] "mini-dinstall -b{0}"'.format('' if new_package else ' --no-db'),
working_directory=package_path)
def upload(metadata):
"""
Uploads a given set of packages
"""
product, release, version_string, revision_date, package_name = metadata
settings = SourceCollector.json_loads('{0}/{1}'.format(os.path.dirname(os.path.realpath(__file__)), 'settings.json'))
working_directory = settings['base_path'].format(product)
package_path = SourceCollector.package_path.format(working_directory)
redhat_folder = '{0}/redhat'.format(package_path)
package_info = settings['repositories']['packages']['redhat']
for destination in package_info:
server = destination['ip']
user = destination['user']
base_path = destination['base_path']
packages = [p for p in os.listdir(redhat_folder) if p.endswith('.rpm')]
for package in packages:
package_source_path = os.path.join(redhat_folder, package)
command = 'scp {0} {1}@{2}:{3}/pool/{4}'.format(package_source_path, user, server, base_path, release)
print('Uploading package {0}'.format(package))
SourceCollector.run(command,
working_directory=redhat_folder)
if len(packages) > 0:
# Cleanup existing files
command = 'ssh {0}@{1} {2}/cleanup_repo.py {2}/pool/{3}/'.format(user, server, base_path, release)
print(SourceCollector.run(command,
working_directory=redhat_folder))
command = 'ssh {0}@{1} createrepo --update {2}/dists/{3}'.format(user, server, base_path, release)
SourceCollector.run(command,
working_directory=redhat_folder)
def upload(source_metadata):
"""
Uploads a given set of packages
"""
filename = '{0}/../settings.cfg'.format(os.path.dirname(os.path.abspath(__file__)))
settings = RawConfigParser()
settings.read(filename)
package_name = settings.get('packaging', 'package_name')
package_path = SourceCollector.package_path.format(settings.get('packaging', 'working_dir'), package_name)
# Get release name from the repo code settings.cfg not master settings.cfg
repo_path_code = SourceCollector.repo_path_code.format(settings.get('packaging', 'working_dir'), settings.get('packaging', 'package_name'))
filename = '{0}/packaging/settings.cfg'.format(repo_path_code)
code_settings = RawConfigParser()
code_settings.read(filename)
releasename = code_settings.get('version', 'releasename').lower()
target, version_string, _ = source_metadata
user = "******"
server = "172.20.3.16"
repo_root_path = "/usr/share/repo"
upload_path = os.path.join(repo_root_path, target, releasename)
print("Uploading {0} {1}".format(package_name, version_string))
debs_path = os.path.join(package_path, 'debian')
deb_packages = [filename for filename in os.listdir(debs_path) if filename.endswith('.deb')]
create_releasename_command = "ssh {0}@{1} mkdir -p {2}".format(user, server, upload_path)
SourceCollector.run(command=create_releasename_command,
working_directory=debs_path)
for deb_package in deb_packages:
source_path = os.path.join(debs_path, deb_package)
destination_path = os.path.join(upload_path, deb_package)
check_package_command = "ssh {0}@{1} ls {2}".format(user, server, upload_path)
existing_packages = SourceCollector.run(command=check_package_command,
working_directory=debs_path).split()
upload = True
for package in existing_packages:
if package == deb_package:
print("Package already uploaded, done...")
upload = False
continue
if upload:
scp_command = "scp {0} {1}@{2}:{3}".format(source_path, user, server, destination_path)
SourceCollector.run(command=scp_command,
working_directory=debs_path)
remote_command = "ssh {0}@{1} reprepro -Vb {2}/debian includedeb {3}-{4} {5}".format(user, server, repo_root_path, releasename, target, destination_path)
SourceCollector.run(command=remote_command,
working_directory=debs_path)
def upload(metadata):
"""
Uploads a given set of packages
"""
product, release, version_string, revision_date, package_name = metadata
settings = SourceCollector.json_loads('{0}/{1}'.format(os.path.dirname(os.path.realpath(__file__)), 'settings.json'))
working_directory = settings['base_path'].format(product)
package_path = SourceCollector.package_path.format(working_directory)
package_info = settings['repositories']['packages']['debian']
for destination in package_info:
server = destination['ip']
user = destination['user']
base_path = destination['base_path']
upload_path = os.path.join(base_path, release)
print("Uploading {0} {1}".format(package_name, version_string))
debs_path = os.path.join(package_path, 'debian')
deb_packages = [filename for filename in os.listdir(debs_path) if filename.endswith('.deb')]
create_releasename_command = "ssh {0}@{1} mkdir -p {2}".format(user, server, upload_path)
SourceCollector.run(command=create_releasename_command,
working_directory=debs_path)
for deb_package in deb_packages:
source_path = os.path.join(debs_path, deb_package)
destination_path = os.path.join(upload_path, deb_package)
check_package_command = "ssh {0}@{1} ls {2}".format(user, server, upload_path)
existing_packages = SourceCollector.run(command=check_package_command,
working_directory=debs_path).split()
upload = deb_package not in existing_packages
if upload is False:
print("Package already uploaded, done...")
else:
scp_command = "scp {0} {1}@{2}:{3}".format(source_path, user, server, destination_path)
SourceCollector.run(command=scp_command,
working_directory=debs_path)
remote_command = "ssh {0}@{1} reprepro -Vb {2}/debian includedeb {3} {4}".format(user, server, base_path, release, destination_path)
SourceCollector.run(command=remote_command,
working_directory=debs_path)
action='store_false',
default=True)
parser.add_option('--pip',
dest='is_pip',
action='store_true',
default=False)
# Currently used as a workarond. The jenkins user does not have py2deb as a command wheras root does
parser.add_option('--py2deb-path', dest='py2deb_path', default='py2deb')
options, args = parser.parse_args()
print 'Received arguments: {0}'.format(options)
# 1. Collect sources
source_collector = SourceCollector(product=options.product,
release=options.release,
revision=options.revision,
artifact_only=options.artifact_only,
dry_run=options.dry_run,
is_pip=options.is_pip,
py2deb_path=options.py2deb_path)
# Setting it to artifact only also means no uploading
if options.artifact_only is True:
options.no_upload = True
settings = source_collector.settings
metadata = source_collector.collect()
print 'Package metadata: {0}'.format(metadata)
if metadata is not None:
add_package = options.release != 'hotfix'
# 2. Build & Upload packages
packagers = []
if options.is_pip is True and options.product in settings['pip'][
from debian import DebianPackager
if __name__ == '__main__':
parser = OptionParser(description='Open vStorage packager')
parser.add_option('-d', '--target', dest='target', default='unstable')
parser.add_option('-r', '--revision', dest='revision', default=None)
parser.add_option('-s', '--suffix', dest='suffix', default=None)
options, args = parser.parse_args()
target = options.target
if target.startswith('release'):
if not target.startswith('release,'):
raise RuntimeError(
"In case a release target is specified, it should be of the format: 'release,<release branch>'"
)
else:
target = tuple(target.split(','))
if target.startswith('experimental,'):
target = tuple(target.split(','))
# 1. Collect sources
source_metadata = SourceCollector.collect(target=target,
revision=options.revision,
suffix=options.suffix)
if source_metadata is not None:
# 2. Build & Upload packages
# - Debian
DebianPackager.package(source_metadata)
DebianPackager.upload(source_metadata)
from redhat import RPMPackager
if __name__ == '__main__':
parser = OptionParser(description='Open vStorage packager')
parser.add_option('-d', '--target', dest='target', default='unstable')
parser.add_option('-r', '--revision', dest='revision', default=None)
parser.add_option('-s', '--suffix', dest='suffix', default=None)
options, args = parser.parse_args()
target = options.target
if target.startswith('release'):
if not target.startswith('release,'):
raise RuntimeError("In case a release target is specified, it should be of the format: 'release,<release branch>'")
else:
target = tuple(target.split(','))
elif target.startswith('experimental,'):
target = tuple(target.split(','))
# 1. Collect sources
source_metadata = SourceCollector.collect(target=target, revision=options.revision, suffix=options.suffix)
if source_metadata is not None:
# 2. Build & Upload packages
# - Debian
DebianPackager.package(source_metadata)
DebianPackager.upload(source_metadata)
# - RPM
RPMPackager.package(source_metadata)
RPMPackager.upload(source_metadata)
def package(source_metadata, skip_sign=False, sign_key=None, source_only=False):
"""
Packages a given package.
"""
distribution, version_string, revision_date = source_metadata
build_opts = []
if skip_sign:
build_opts.append("-uc -us")
elif sign_key:
# TODO: if GPG key is password-protected, signing will fail because
# check_output is not interactive
build_opts.append('-k"%s"' % sign_key)
if source_only:
build_opts.append("-S")
filename = "{0}/../settings.cfg".format(os.path.dirname(os.path.abspath(__file__)))
settings = RawConfigParser()
settings.read(filename)
package_name = settings.get("packaging", "package_name")
repo_path_code = SourceCollector.repo_path_code.format(settings.get("packaging", "working_dir"), package_name)
package_path = SourceCollector.package_path.format(settings.get("packaging", "working_dir"), package_name)
# Prepare
# /<pp>/debian
debian_folder = "{0}/debian".format(package_path)
if os.path.exists(debian_folder):
shutil.rmtree(debian_folder)
# /<rp>/packaging/debian -> /<pp>/debian
shutil.copytree("{0}/packaging/debian".format(repo_path_code), debian_folder)
# Rename tgz
# /<pp>/<packagename>_1.2.3.tar.gz -> /<pp>/debian/<packagename>_1.2.3.orig.tar.gz
shutil.copyfile(
"{0}/{1}_{2}.tar.gz".format(package_path, package_name, version_string),
"{0}/{1}_{2}.orig.tar.gz".format(debian_folder, package_name, version_string),
)
# /<pp>/debian/<packagename>-1.2.3/...
SourceCollector.run(
command="tar -xzf {0}_{1}.orig.tar.gz".format(package_name, version_string), working_directory=debian_folder
)
# Move the debian package metadata into the extracted source
# /<pp>/debian/debian -> /<pp>/debian/<packagename>-1.2.3/
SourceCollector.run(
command="mv {0}/debian {0}/{1}-{2}/".format(debian_folder, package_name, version_string),
working_directory=package_path,
)
# Build changelog entry
with open(
"{0}/{1}-{2}/debian/changelog".format(debian_folder, package_name, version_string), "w"
) as changelog_file:
changelog_file.write(
"""{0} ({1}-1) {2}; urgency=low
* For changes, see individual changelogs
-- Packaging System <*****@*****.**> {3}
""".format(
package_name, version_string, distribution, revision_date.strftime("%a, %d %b %Y %H:%M:%S +0000")
)
)
# Some more tweaks
SourceCollector.run(
command="chmod 770 {0}/{1}-{2}/debian/rules".format(debian_folder, package_name, version_string),
working_directory=package_path,
)
SourceCollector.run(
command="sed -i -e 's/__NEW_VERSION__/{0}/' *.*".format(version_string),
working_directory="{0}/{1}-{2}/debian".format(debian_folder, package_name, version_string),
)
# Build the package
SourceCollector.run(
command="dpkg-buildpackage %s" % " ".join(build_opts),
working_directory="{0}/{1}-{2}".format(debian_folder, package_name, version_string),
)
def package(source_metadata):
"""
Packages a given package.
"""
distribution, version_string, revision_date = source_metadata
root_path = ROOT_PATH
filename = '{0}/../settings.cfg'.format(root_path)
settings = RawConfigParser()
settings.read(filename)
package_name = settings.get('packaging', 'package_name')
repo_path_code = SourceCollector.repo_path_code.format(settings.get('packaging', 'working_dir'), package_name)
package_path = SourceCollector.package_path.format(settings.get('packaging', 'working_dir'), package_name)
# Prepare
redhat_folder = '{0}/redhat'.format(package_path)
if os.path.exists(redhat_folder):
shutil.rmtree(redhat_folder)
os.mkdir(redhat_folder)
# load config
config_dir = '{0}/../redhat/cfgs'.format(root_path)
packages = os.listdir(config_dir)
for package in packages:
package_filename = '{0}/{1}'.format(config_dir, package)
package_cfg = RawConfigParser()
package_cfg.read(package_filename)
package_name = package_cfg.get('main', 'name')
dirs = package_cfg.get('main', 'dirs')
files = package_cfg.get('main', 'files')
depends_packages = package_cfg.get('main', 'depends').replace('$Version', version_string.replace('-', '_'))
depends = ""
if depends_packages != '':
depends = []
for depends_package in depends_packages.split(','):
depends.append('-d "{}"'.format(depends_package.strip()))
depends = ' '.join(depends)
package_root_path = os.path.join(package_path, package_name)
if os.path.exists(package_root_path):
shutil.rmtree(package_root_path)
os.mkdir(package_root_path)
for dir_ in dirs.split(','):
dir_ = dir_.strip()
if dir_ != "''":
source_dir, dest_location = dir_.split('=')
# source_dir = dir to copy - from repo root
# dest_location = dir under which to copy the source_dir
source_full_path = os.path.join(repo_path_code, source_dir.strip())
dest_full_path = os.path.join(package_root_path, dest_location.strip())
shutil.copytree(source_full_path, dest_full_path)
for file_ in files.split(','):
file_ = file_.strip()
if file_ != "''" and file_ != '':
source_file, dest_location = file_.split('=')
source_full_path = os.path.join(repo_path_code, source_file.strip())
dest_full_path = os.path.join(package_root_path, dest_location.strip())
if not os.path.exists(dest_full_path):
os.makedirs(dest_full_path)
shutil.copy(source_full_path, dest_full_path)
before_install, after_install = ' ', ' '
script_root = '{0}/../redhat/scripts'.format(root_path)
before_install_script = '{0}.before-install.sh'.format(package_name)
before_install_script_path = os.path.join(script_root, before_install_script)
if os.path.exists(before_install_script_path):
before_install = ' --before-install {0} '.format(before_install_script_path)
after_install_script = '{0}.after-install.sh'.format(package_name)
after_install_script_path = os.path.join(script_root, after_install_script)
if os.path.exists(after_install_script_path):
after_install = ' --after-install {0} '.format(after_install_script_path)
SourceCollector.run(command="sed -i -e 's/$Version/{0}/g' {1}".format(version_string,
after_install_script_path),
working_directory='{0}'.format(script_root))
params = {'version': version_string,
'package_name': package_cfg.get('main', 'name'),
'summary': package_cfg.get('main', 'summary'),
'license': package_cfg.get('main', 'license'),
'URL': package_cfg.get('main', 'URL'),
'source': package_cfg.get('main', 'source'),
'arch': package_cfg.get('main', 'arch'),
'description': package_cfg.get('main', 'description'),
'maintainer': package_cfg.get('main', 'maintainer'),
'depends': depends,
'package_root': package_root_path,
'before_install': before_install,
'after_install': after_install,
}
command = """fpm -s dir -t rpm -n {package_name} -v {version} --description "{description}" --maintainer "{maintainer}" --license "{license}" --url {URL} -a {arch} --vendor "Open vStorage" {depends}{before_install}{after_install} --prefix=/ -C {package_root}""".format(**params)
SourceCollector.run(command,
working_directory=redhat_folder)
print(os.listdir(redhat_folder))
from optparse import OptionParser
from sourcecollector import SourceCollector
from debian import DebianPackager
from redhat import RPMPackager
if __name__ == '__main__':
parser = OptionParser(description='Open vStorage packager')
parser.add_option('-p', '--product', dest='product')
parser.add_option('-r', '--release', dest='release', default=None)
parser.add_option('-e', '--revision', dest='revision', default=None)
parser.add_option('-s', '--suffix', dest='suffix', default=None)
parser.add_option('--no-rpm', dest='rpm', action='store_false', default=True)
parser.add_option('--no-deb', dest='deb', action='store_false', default=True)
options, args = parser.parse_args()
# 1. Collect sources
metadata = SourceCollector.collect(product=options.product,
release=options.release,
revision=options.revision,
suffix=options.suffix)
if metadata is not None:
# 2. Build & Upload packages
if options.deb is True:
DebianPackager.package(metadata)
DebianPackager.upload(metadata)
if options.rpm is True:
RPMPackager.package(metadata)
RPMPackager.upload(metadata)
def package(source_metadata,
skip_sign=False,
sign_key=None,
source_only=False):
"""
Packages a given package.
"""
distribution, version_string, revision_date = source_metadata
build_opts = []
if skip_sign:
build_opts.append('-uc -us')
elif sign_key:
# TODO: if GPG key is password-protected, signing will fail because
# check_output is not interactive
build_opts.append('-k"%s"' % sign_key)
if source_only:
build_opts.append('-S')
filename = '{0}/../settings.cfg'.format(
os.path.dirname(os.path.abspath(__file__)))
settings = RawConfigParser()
settings.read(filename)
package_name = settings.get('packaging', 'package_name')
repo_path_code = SourceCollector.repo_path_code.format(
settings.get('packaging', 'working_dir'), package_name)
package_path = SourceCollector.package_path.format(
settings.get('packaging', 'working_dir'), package_name)
# Prepare
# /<pp>/debian
debian_folder = '{0}/debian'.format(package_path)
if os.path.exists(debian_folder):
shutil.rmtree(debian_folder)
# /<rp>/packaging/debian -> /<pp>/debian
shutil.copytree('{0}/packaging/debian'.format(repo_path_code),
debian_folder)
# Rename tgz
# /<pp>/<packagename>_1.2.3.tar.gz -> /<pp>/debian/<packagename>_1.2.3.orig.tar.gz
shutil.copyfile(
'{0}/{1}_{2}.tar.gz'.format(package_path, package_name,
version_string),
'{0}/{1}_{2}.orig.tar.gz'.format(debian_folder, package_name,
version_string))
# /<pp>/debian/<packagename>-1.2.3/...
SourceCollector.run(command='tar -xzf {0}_{1}.orig.tar.gz'.format(
package_name, version_string),
working_directory=debian_folder)
# Move the debian package metadata into the extracted source
# /<pp>/debian/debian -> /<pp>/debian/<packagename>-1.2.3/
SourceCollector.run(command='mv {0}/debian {0}/{1}-{2}/'.format(
debian_folder, package_name, version_string),
working_directory=package_path)
# Build changelog entry
with open(
'{0}/{1}-{2}/debian/changelog'.format(debian_folder,
package_name,
version_string),
'w') as changelog_file:
changelog_file.write("""{0} ({1}-1) {2}; urgency=low
* For changes, see individual changelogs
-- Packaging System <*****@*****.**> {3}
""".format(package_name, version_string, distribution,
revision_date.strftime('%a, %d %b %Y %H:%M:%S +0000')))
# Some more tweaks
SourceCollector.run(
command='chmod 770 {0}/{1}-{2}/debian/rules'.format(
debian_folder, package_name, version_string),
working_directory=package_path)
SourceCollector.run(
command="sed -i -e 's/__NEW_VERSION__/{0}/' *.*".format(
version_string),
working_directory='{0}/{1}-{2}/debian'.format(
debian_folder, package_name, version_string))
# Build the package
SourceCollector.run(
command='dpkg-buildpackage %s' % ' '.join(build_opts),
working_directory='{0}/{1}-{2}'.format(debian_folder, package_name,
version_string))
def package(metadata):
"""
Packages a given package.
"""
product, release, version_string, revision_date, package_name = metadata
settings = SourceCollector.json_loads('{0}/{1}'.format(os.path.dirname(os.path.realpath(__file__)), 'settings.json'))
working_directory = settings['base_path'].format(product)
repo_path_code = SourceCollector.repo_path_code.format(working_directory)
package_path = SourceCollector.package_path.format(working_directory)
# Prepare
redhat_folder = '{0}/redhat'.format(package_path)
if os.path.exists(redhat_folder):
shutil.rmtree(redhat_folder)
os.mkdir(redhat_folder)
# extract tar.gz to redhat_folder
shutil.copyfile('{0}/{1}_{2}.tar.gz'.format(package_path, package_name, version_string),
'{0}/{1}_{2}.orig.tar.gz'.format(redhat_folder, package_name, version_string))
# /<pp>/debian/<packagename>-1.2.3/...
SourceCollector.run(command='tar -xzf {0}_{1}.orig.tar.gz'.format(package_name, version_string),
working_directory=redhat_folder)
code_source_path = '{0}/{1}-{2}'.format(redhat_folder, package_name, version_string)
# copy packaging
source_packaging_path = os.path.join(repo_path_code, 'packaging')
dest_packaging_path = os.path.join(code_source_path, 'packaging')
if os.path.exists(source_packaging_path):
shutil.copytree(source_packaging_path, dest_packaging_path)
# load config
config_dir = '{0}/packaging/redhat/cfgs'.format(repo_path_code)
packages = os.listdir(config_dir)
for package in packages:
package_filename = '{0}/{1}'.format(config_dir, package)
package_cfg = RawConfigParser()
package_cfg.read(package_filename)
package_name = package_cfg.get('main', 'name')
dirs = package_cfg.get('main', 'dirs')
files = package_cfg.get('main', 'files')
depends_packages = package_cfg.get('main', 'depends').replace('$Version', version_string.replace('-', '_'))
depends = ""
if depends_packages != '':
depends = []
for depends_package in depends_packages.split(','):
depends.append('-d "{0}"'.format(depends_package.strip()))
depends = ' '.join(depends)
package_root_path = os.path.join(package_path, package_name)
if os.path.exists(package_root_path):
shutil.rmtree(package_root_path)
os.mkdir(package_root_path)
for dir_ in dirs.split(','):
dir_ = dir_.strip()
if dir_ != "''":
source_dir, dest_location = dir_.split('=')
# source_dir = dir to copy - from repo root
# dest_location = dir under which to copy the source_dir
source_full_path = os.path.join(code_source_path, source_dir.strip())
dest_full_path = os.path.join(package_root_path, dest_location.strip())
shutil.copytree(source_full_path, dest_full_path)
for file_ in files.split(','):
file_ = file_.strip()
if file_ != "''" and file_ != '':
source_file, dest_location = file_.split('=')
source_full_path = os.path.join(code_source_path, source_file.strip())
dest_full_path = os.path.join(package_root_path, dest_location.strip())
if not os.path.exists(dest_full_path):
os.makedirs(dest_full_path)
shutil.copy(source_full_path, dest_full_path)
before_install, after_install = ' ', ' '
script_root = '{0}/packaging/redhat/scripts'.format(code_source_path)
before_install_script = '{0}.before-install.sh'.format(package_name)
before_install_script_path = os.path.join(script_root, before_install_script)
if os.path.exists(before_install_script_path):
before_install = ' --before-install {0} '.format(before_install_script_path)
after_install_script = '{0}.after-install.sh'.format(package_name)
after_install_script_path = os.path.join(script_root, after_install_script)
if os.path.exists(after_install_script_path):
after_install = ' --after-install {0} '.format(after_install_script_path)
SourceCollector.run(command="sed -i -e 's/$Version/{0}/g' {1}".format(version_string,
after_install_script_path),
working_directory='{0}'.format(script_root))
params = {'version': version_string,
'package_name': package_cfg.get('main', 'name'),
'summary': package_cfg.get('main', 'summary'),
'license': package_cfg.get('main', 'license'),
'URL': package_cfg.get('main', 'URL'),
'source': package_cfg.get('main', 'source'),
'arch': package_cfg.get('main', 'arch'),
'description': package_cfg.get('main', 'description'),
'maintainer': package_cfg.get('main', 'maintainer'),
'depends': depends,
'package_root': package_root_path,
'before_install': before_install,
'after_install': after_install,
}
command = """fpm -s dir -t rpm -n {package_name} -v {version} --description "{description}" --maintainer "{maintainer}" --license "{license}" --url {URL} -a {arch} --vendor "Open vStorage" {depends}{before_install}{after_install} --prefix=/ -C {package_root}""".format(**params)
SourceCollector.run(command,
working_directory=redhat_folder)
print(os.listdir(redhat_folder))
if __name__ == '__main__':
parser = OptionParser(description='Open vStorage repo maintenance')
parser.add_option('-f', '--from-release', dest='from_release')
parser.add_option('-t', '--to-release', dest='to_release')
parser.add_option('-s', '--skip', dest='skip')
parser.add_option('-d',
'--dry-run',
dest='dry_run',
action='store_true',
default=False)
options, args = parser.parse_args()
dry_run = options.dry_run
skips = tuple(options.skip.split(',')) if options.skip is not None else ()
settings = SourceCollector.json_loads('{0}/{1}'.format(
os.path.dirname(os.path.realpath(__file__)), 'settings.json'))
package_info = settings['repositories']['packages'].get('debian', [])
for destination in package_info:
server = destination['ip']
user = destination['user']
base_path = destination['base_path']
print 'Processing {0}@{1}'.format(user, server)
print ' Reading releases'
source_package_map = {}
destination_package_map = {}
for release, package_map in {
options.from_release: source_package_map,
