def extract_changelog(changes_file, component, pool_dir):
"""
Extract the changelog according to the changes_file
If handling a _source.changes, extract the real changelog
If handling binary .changes just creat a link for each of the binaries
"""
global config, options
extract_dir = '/tmp/changelog_extract'
control_file = DebianControlFile(changes_file)
name = control_file['Source']
name_version = name + '_' + control_file.version()
if name.startswith('lib'):
prefix = name[:4]
else:
prefix = name[0]
pool_dir = join(options.output_dir, 'pool', \
component, prefix, name)
dirname = os.path.dirname(changes_file)
if changes_file.endswith('_source.changes'): # Really extract
if os.path.isdir(extract_dir):
shutil.rmtree(extract_dir)
for file in control_file.files_list():
if file.name.endswith('.dsc'):
(rc, output) = commands.getstatusoutput('dpkg-source -x %s %s' % \
(join(pool_dir, file.name), extract_dir))
if rc <> 0 or not os.path.isdir(extract_dir):
Log.print_(output)
Log.print_(
"Unable to extract source to retrieve changelog")
else:
extacted_changelog = os.path.join(extract_dir, 'debian',
'changelog')
if not exists(extacted_changelog):
Log.print_("Unable to find changelog on source")
if not os.path.exists(pool_dir):
os.makedirs(pool_dir, 0755)
print pool_dir
changelog_fn = join(
pool_dir,
os.path.basename(changes_file).rsplit('.', 1)[0] +
'.changelog')
shutil.copy(extacted_changelog, changelog_fn)
if os.path.isdir(extract_dir):
shutil.rmtree(extract_dir)
else: # binary build .changes, create a link to the corresponding source
files = control_file.files_list()
for file in files:
if file.name.endswith('.deb'):
try:
os.symlink(name_version+"_source.changelog", \
join(pool_dir, file.name.rsplit('.', 1)[0]+'.changelog'))
except OSError: # Already exists ?
pass
def extract_changelog(changes_file, component, pool_dir):
"""
Extract the changelog according to the changes_file
If handling a _source.changes, extract the real changelog
If handling binary .changes just creat a link for each of the binaries
"""
global config, options
extract_dir = "/tmp/changelog_extract"
control_file = DebianControlFile(changes_file)
name = control_file["Source"]
name_version = name + "_" + control_file.version()
if name.startswith("lib"):
prefix = name[:4]
else:
prefix = name[0]
pool_dir = join(options.output_dir, "pool", component, prefix, name)
dirname = os.path.dirname(changes_file)
if changes_file.endswith("_source.changes"): # Really extract
if os.path.isdir(extract_dir):
shutil.rmtree(extract_dir)
for file in control_file.files_list():
if file.name.endswith(".dsc"):
(rc, output) = commands.getstatusoutput(
"dpkg-source -x %s %s" % (join(pool_dir, file.name), extract_dir)
)
if rc <> 0 or not os.path.isdir(extract_dir):
Log.print_(output)
Log.print_("Unable to extract source to retrieve changelog")
else:
extacted_changelog = os.path.join(extract_dir, "debian", "changelog")
if not exists(extacted_changelog):
Log.print_("Unable to find changelog on source")
if not os.path.exists(pool_dir):
os.makedirs(pool_dir, 0755)
print pool_dir
changelog_fn = join(pool_dir, os.path.basename(changes_file).rsplit(".", 1)[0] + ".changelog")
shutil.copy(extacted_changelog, changelog_fn)
if os.path.isdir(extract_dir):
shutil.rmtree(extract_dir)
else: # binary build .changes, create a link to the corresponding source
files = control_file.files_list()
for file in files:
if file.name.endswith(".deb"):
try:
os.symlink(
name_version + "_source.changelog", join(pool_dir, file.name.rsplit(".", 1)[0] + ".changelog")
)
except OSError: # Already exists ?
pass
release = commands.getoutput('lsb_release -cs')
run_dir = os.getcwd()
if not os.path.isdir('/tmp/build_test'):
os.mkdir('/tmp/build_test')
os.chdir('/tmp/build_test')
dsc_file = os.path.basename(dsc_url)
# Get and extract, apply the current release, generate a _source changes
run_or_exit('dget -ux '+dsc_url)
debian_control = DebianControlFile(dsc_file)
upstream_version = debian_control.upstream_version()
package_name = debian_control['Source']
os.chdir(join('/tmp/build_test', '%s-%s' % (package_name, upstream_version)))
run_or_exit('dch -D %s Testing package' % release)
run_or_exit('debuild -S -sa -uc')
changes_control = DebianControlFile('/tmp/build_test/%s_%s_source.changes' % (package_name, debian_control.version()))
if not os.path.exists('/tmp/build_incoming/'+release+'/apps'):
os.makedirs('/tmp/build_incoming/'+release+'/apps')
changes_control.move('/tmp/build_incoming/'+release+'/apps', '/tmp/build_test/')
os.chdir(run_dir)
# Test the process
run_or_exit('build/process_incoming.py --skip-gpg-check -i /tmp/build_incoming -o /tmp/pre_build')
run_or_exit('build/process_pre_build.py --skip-gpg-check -i /tmp/pre_build -o /tmp/post_build')
if not os.environ.get('REPREPRO_BASE_DIR'):
print "Skipping reprepro test, REPREPRO_BASE_DIR not defined"
else:
run_or_exit('build/process_post_build.py -i /tmp/post_build')
def check_changes(release, component, filename):
"""
Check a _.changes file and include it into the repository
"""
global config, options
source_dir = "%s/%s/%s" % (options.input_dir, release, component)
changes_file = "%s/%s" % (source_dir, filename)
if not os.path.exists(changes_file):
return 1
Log.print_("Including %s/%s/%s" % (release, component, filename))
# Remove previous failed status
if exists("%s.failed" % changes_file):
os.unlink("%s.failed" % changes_file)
control_file = DebianControlFile(changes_file)
name = control_file["Source"]
version = control_file.version()
name_version = "%s_%s" % (control_file["Source"], control_file.version())
report_title = "Include on testing for %s/%s/%s FAILED\n" % (release, component, name_version)
report_msg = "File: %s/%s/%s\n" % (release, component, filename)
report_msg += "-----------------\n"
# target_mails.append(control_file['Changed-By'])
report_msg += "Signed By: %s\n\n" % control_file["Changed-By"]
# Get list of files described on the changes
report_msg += "List of files:\n"
report_msg += "--------------\n"
file_list = control_file.files_list()
for file_info in file_list:
report_msg += "%s (%s) MD5: %s \n" % (file_info.name, file_info.size, file_info.md5sum)
if name.startswith("lib"):
prefix = name[:4]
else:
prefix = name[0]
pool_dir = join(options.output_dir, "pool", component, prefix, name)
file_on_dest = "%s/%s" % (pool_dir, filename)
# Remove all packages related to source package
if filename.endswith("_source.changes"):
os.system("reprepro removesrc %s-getdeb-testing %s %s" % (release, name, control_file["Version"]))
# Delete orphaned changelogs
changelogs = glob.glob("%s/*.changelog" % (pool_dir))
for changelog in changelogs:
deb = changelog.rsplit(".", 1)[0] + ".deb"
if not os.path.exists(deb):
Log.print_("Removing changelog: %s" % (changelog))
os.unlink(changelog)
# Include the package (use standard as "normal" does not exist due to Debian policy)
# (LP: #735381, #735428)
# Now use "optional" as priority because it fits even better to the Debian policy
# (LP: #1085565)
command = "reprepro -P optional --ignore=wrongdistribution -C %s include %s-getdeb-testing %s" % (
component,
release,
changes_file,
)
(rc, output) = commands.getstatusoutput(command)
print output
report_msg += output
if rc == 0:
extract_changelog(changes_file, component, pool_dir)
status = "SUCCESSFUL"
control_file.remove()
else:
status = "FAILED"
shutil.move(changes_file, "%s.failed" % changes_file)
report_title = "Included on testing %s/%s/%s %s\n" % (release, component, name_version, status)
Log.print_(report_title)
return rc
def check_source_changes(release, component, filename):
"""
Check a _source.changes file and proceed as described in the script
action flow .
"""
global options
global config
Log.print_("Checking %s/%s/%s" % (release, component, filename))
source_dir = "%s/%s/%s" \
% (options.input_dir, release, component)
full_pre_build_dir = "%s/%s/%s" \
% (options.output_dir, release, component)
changes_file = "%s/%s" % (source_dir, filename)
if not os.path.exists(full_pre_build_dir):
os.makedirs(full_pre_build_dir, 0755)
control_file = DebianControlFile("%s/%s" % (source_dir, filename))
if not options.skip_gpg:
gpg_sign_author = control_file.verify_gpg(os.environ['HOME'] \
+'/debfactory/keyrings/uploaders.gpg ', Log.verbose)
if not gpg_sign_author:
Log.print_("ERROR: Unable to verify GPG key for %s" % changes_file)
return
else:
gpg_sign_author = control_file['Changed-By']
if not gpg_sign_author:
Log.print_("ERROR: Changed-By was not found in %s" % changes_file)
return
name_version = "%s_%s" % (control_file['Source'] \
, control_file.version())
package_release = control_file['Distribution']
if package_release != release:
report_title = "Upload for %s/%s/%s FAILED\n" \
% (release, component, name_version)
report_msg = u"The release %s on debian/changelog does noth match"\
" the target %s\n" % (package_release, release)
Log.print_(report_msg)
Log.print_(report_title)
send_mail(config['sender_email'], gpg_sign_author, report_title, report_msg)
return
report_title = "Upload for %s/%s/%s FAILED\n" \
% (release, component, name_version)
report_msg = u"File: %s/%s/%s\n" % (release, component, filename)
report_msg += '-----------------\n'
report_msg = u"Signed By: %s\n\n" % gpg_sign_author
orig_file_extensions = [ "gz", "bz2", "lzma", "xz" ]
found_orig_file = False
# Check if orig_file is available
for orig_file_extension in orig_file_extensions:
orig_file = "%s_%s.orig.tar.%s" % (control_file['Source'], \
control_file.upstream_version(), orig_file_extension)
if not orig_file:
Log.print_("FIXME: This should never happen")
# FIXME: This should never happen but we should send a message
# anyway
return
if os.path.exists("%s/%s" % (source_dir, orig_file)):
found_orig_file = True
else:
pre_build_orig = "%s/%s" % (full_pre_build_dir, orig_file)
if os.path.exists(pre_build_orig):
found_orig_file = True
Log.print_('No orig.tar.%s, using %s ' % (orig_file_extension, pre_build_orig))
if not found_orig_file:
print "report_msg:\n", report_msg
report_msg += u"ERROR: Missing orig.tar.[gz,bz2,lzma,xz] for %s\n" \
% (changes_file)
Log.print_(report_msg)
send_mail(config['sender_email'], gpg_sign_author, report_title, report_msg)
return
# Get list of files described on the changes
report_msg += u"List of files:\n"
report_msg += u"--------------\n"
file_list = control_file.files_list()
for file_info in file_list:
report_msg += u"%s (%s) MD5: %s \n" \
% (file_info.name, file_info.size, file_info.md5sum)
try:
if not options.check_only:
control_file.move(full_pre_build_dir)
except DebianControlFile.MD5Error, e:
report_msg = u"MD5 mismatch: Expected %s, got %s, file: %s\n" \
% (e.expected_md5, e.found_md5, e.name)
if not os.path.isdir('/tmp/build_test'):
os.mkdir('/tmp/build_test')
os.chdir('/tmp/build_test')
dsc_file = os.path.basename(dsc_url)
# Get and extract, apply the current release, generate a _source changes
run_or_exit('dget -ux ' + dsc_url)
debian_control = DebianControlFile(dsc_file)
upstream_version = debian_control.upstream_version()
package_name = debian_control['Source']
os.chdir(join('/tmp/build_test', '%s-%s' % (package_name, upstream_version)))
run_or_exit('dch -D %s Testing package' % release)
run_or_exit('debuild -S -sa -uc')
changes_control = DebianControlFile('/tmp/build_test/%s_%s_source.changes' %
(package_name, debian_control.version()))
if not os.path.exists('/tmp/build_incoming/' + release + '/apps'):
os.makedirs('/tmp/build_incoming/' + release + '/apps')
changes_control.move('/tmp/build_incoming/' + release + '/apps',
'/tmp/build_test/')
os.chdir(run_dir)
# Test the process
run_or_exit(
'build/process_incoming.py --skip-gpg-check -i /tmp/build_incoming -o /tmp/pre_build'
)
run_or_exit(
'build/process_pre_build.py --skip-gpg-check -i /tmp/pre_build -o /tmp/post_build'
)
if not os.environ.get('REPREPRO_BASE_DIR'):
print "Skipping reprepro test, REPREPRO_BASE_DIR not defined"
def check_changes(release, component, filename):
"""
Check a _.changes file and include it into the repository
"""
global config, options
source_dir = "%s/%s/%s" \
% (options.input_dir, release, component)
changes_file = "%s/%s" % (source_dir, filename)
if not os.path.exists(changes_file):
return 1
Log.print_("Including %s/%s/%s" % (release, component, filename))
# Remove previous failed status
if exists('%s.failed' % changes_file):
os.unlink('%s.failed' % changes_file)
control_file = DebianControlFile(changes_file)
name = control_file['Source']
version = control_file.version()
name_version = "%s_%s" % (control_file['Source'] \
, control_file.version())
report_title = "Include on testing for %s/%s/%s FAILED\n" \
% (release, component, name_version)
report_msg = "File: %s/%s/%s\n" % (release, component, filename)
report_msg += '-----------------\n'
#target_mails.append(control_file['Changed-By'])
report_msg += "Signed By: %s\n\n" % control_file['Changed-By']
# Get list of files described on the changes
report_msg += "List of files:\n"
report_msg += "--------------\n"
file_list = control_file.files_list()
for file_info in file_list:
report_msg += "%s (%s) MD5: %s \n" \
% (file_info.name, file_info.size, file_info.md5sum)
if name.startswith('lib'):
prefix = name[:4]
else:
prefix = name[0]
pool_dir = join(options.output_dir, 'pool', \
component, prefix, name)
file_on_dest = "%s/%s" % (pool_dir, filename)
# Remove all packages related to source package
if (filename.endswith("_source.changes")):
os.system("reprepro removesrc %s-getdeb-testing %s %s" %
(release, name, control_file['Version']))
# Delete orphaned changelogs
changelogs = glob.glob("%s/*.changelog" % (pool_dir))
for changelog in changelogs:
deb = changelog.rsplit('.', 1)[0] + '.deb'
if not os.path.exists(deb):
Log.print_("Removing changelog: %s" % (changelog))
os.unlink(changelog)
# Include the package (use standard as "normal" does not exist due to Debian policy)
# (LP: #735381, #735428)
# Now use "optional" as priority because it fits even better to the Debian policy
# (LP: #1085565)
command = "reprepro -P optional --ignore=wrongdistribution -C %s include %s-getdeb-testing %s" \
% (component, release, changes_file)
(rc, output) = commands.getstatusoutput(command)
print output
report_msg += output
if rc == 0:
extract_changelog(changes_file, component, pool_dir)
status = "SUCCESSFUL"
control_file.remove()
else:
status = "FAILED"
shutil.move(changes_file, "%s.failed" % changes_file)
report_title = "Included on testing %s/%s/%s %s\n" \
% (release, component, name_version, status)
Log.print_(report_title)
return rc
def check_source_changes(release, component, filename):
"""
Check a _source.changes file and proceed as described in the script
action flow .
"""
global options
global config
Log.print_("Checking %s/%s/%s" % (release, component, filename))
source_dir = "%s/%s/%s" \
% (options.input_dir, release, component)
full_pre_build_dir = "%s/%s/%s" \
% (options.output_dir, release, component)
changes_file = "%s/%s" % (source_dir, filename)
if not os.path.exists(full_pre_build_dir):
os.makedirs(full_pre_build_dir, 0755)
control_file = DebianControlFile("%s/%s" % (source_dir, filename))
if not options.skip_gpg:
gpg_sign_author = control_file.verify_gpg(os.environ['HOME'] \
+'/debfactory/keyrings/uploaders.gpg ', Log.verbose)
if not gpg_sign_author:
Log.print_("ERROR: Unable to verify GPG key for %s" % changes_file)
return
else:
gpg_sign_author = control_file['Changed-By']
if not gpg_sign_author:
Log.print_("ERROR: Changed-By was not found in %s" % changes_file)
return
name_version = "%s_%s" % (control_file['Source'] \
, control_file.version())
package_release = control_file['Distribution']
if package_release != release:
report_title = "Upload for %s/%s/%s FAILED\n" \
% (release, component, name_version)
report_msg = u"The release %s on debian/changelog does noth match"\
" the target %s\n" % (package_release, release)
Log.print_(report_msg)
Log.print_(report_title)
send_mail(config['sender_email'], gpg_sign_author, report_title,
report_msg)
return
report_title = "Upload for %s/%s/%s FAILED\n" \
% (release, component, name_version)
report_msg = u"File: %s/%s/%s\n" % (release, component, filename)
report_msg += '-----------------\n'
report_msg = u"Signed By: %s\n\n" % gpg_sign_author
orig_file_extensions = ["gz", "bz2", "lzma", "xz"]
found_orig_file = False
# Check if orig_file is available
for orig_file_extension in orig_file_extensions:
orig_file = "%s_%s.orig.tar.%s" % (control_file['Source'], \
control_file.upstream_version(), orig_file_extension)
if not orig_file:
Log.print_("FIXME: This should never happen")
# FIXME: This should never happen but we should send a message
# anyway
return
if os.path.exists("%s/%s" % (source_dir, orig_file)):
found_orig_file = True
else:
pre_build_orig = "%s/%s" % (full_pre_build_dir, orig_file)
if os.path.exists(pre_build_orig):
found_orig_file = True
Log.print_('No orig.tar.%s, using %s ' %
(orig_file_extension, pre_build_orig))
if not found_orig_file:
print "report_msg:\n", report_msg
report_msg += u"ERROR: Missing orig.tar.[gz,bz2,lzma,xz] for %s\n" \
% (changes_file)
Log.print_(report_msg)
send_mail(config['sender_email'], gpg_sign_author, report_title,
report_msg)
return
# Get list of files described on the changes
report_msg += u"List of files:\n"
report_msg += u"--------------\n"
file_list = control_file.files_list()
for file_info in file_list:
report_msg += u"%s (%s) MD5: %s \n" \
% (file_info.name, file_info.size, file_info.md5sum)
try:
if not options.check_only:
control_file.move(full_pre_build_dir)
except DebianControlFile.MD5Error, e:
report_msg = u"MD5 mismatch: Expected %s, got %s, file: %s\n" \
% (e.expected_md5, e.found_md5, e.name)