def __init__(self, filename=None, string=None):
"""
Object constructor. The object allows the user to specify **either**:
#. a path to a *changes* file to parse
#. a string with the *changes* file contents.
::
a = Changes(filename='/tmp/packagename_version.changes')
b = Changes(string='Source: packagename\\nMaintainer: ...')
``filename``
Path to *changes* file to parse.
``string``
*changes* file in a string to parse.
"""
if (filename and string) or (not filename and not string):
raise TypeError
if filename:
self._absfile = os.path.abspath(filename)
self._directory = os.path.dirname(self._absfile)
self._data = deb822.Changes(open(filename, encoding='utf-8'))
self.basename = os.path.basename(filename)
else:
self._data = deb822.Changes(string)
if len(self._data) == 0:
raise ChangesFileException('Changes file could not be parsed.')
def forge_changes_file(fname, dist, **kwargs):
dsc = deb822.Dsc(open(fname, 'r'))
changes = deb822.Changes()
changes['Format'] = '1.8'
changes['Date'] = email.utils.formatdate(time.mktime(
dt.datetime.utcnow().timetuple()),
usegmt=True)
for key in [
'Source', 'Version', 'Maintainer', 'Checksums-Sha1',
'Checksums-Sha256', 'Files'
]:
if not dsc.has_key(key):
raise MissingChangesFieldException(key)
changes[key] = dsc[key]
for algo, key, h, s, f in file_info(fname):
if algo == 'md5':
algo = 'md5sum'
entry = deb822.Deb822Dict()
entry[algo] = h
entry['size'] = s
entry['name'] = f
changes[key].append(entry)
for entry in changes['Files']:
entry['section'] = 'not-implemented'
entry['priority'] = 'not-implemented'
changes['Distribution'] = dist
changes['Urgency'] = 'low'
changes['Changed-By'] = 'Archive Rebuilder <*****@*****.**>'
changes['Architecture'] = 'source'
changes['Binary'] = 'not implemented either'
changes['Description'] = """This feature is not implemented.
This is a pretty damn hard to deal with right now. I might write this
later."""
changes['Changes'] = """
{source} ({version}) {dist}; urgency={urgency}
.
* This is a fake ChangeLog entry used by ricky to force a rebuild
on debuild.me.""".format(
source=changes['Source'],
version=changes['Version'],
urgency=changes['Urgency'],
dist=dist,
)
for k, v in kwargs.items():
changes[k] = v
return changes
def test_unicode_values(self):
"""Deb822 objects should contain only unicode values
(Technically, they are allowed to contain any type of object, but when
parsed from files, and when only string-type objects are added, the
resulting object should have only unicode values.)
"""
objects = []
objects.append(deb822.Deb822(UNPARSED_PACKAGE))
objects.append(deb822.Deb822(CHANGES_FILE))
with open_utf8('test_Packages') as f:
objects.extend(deb822.Deb822.iter_paragraphs(f))
with open_utf8('test_Packages') as f:
objects.extend(deb822.Packages.iter_paragraphs(f))
with open_utf8('test_Sources') as f:
objects.extend(deb822.Deb822.iter_paragraphs(f))
with open('test_Sources.iso8859-1', 'rb') as f:
objects.extend(
deb822.Deb822.iter_paragraphs(f, encoding="iso8859-1"))
for d in objects:
for value in d.values():
self.assertTrue(isinstance(value, six.text_type))
# The same should be true for Sources and Changes except for their
# _multivalued fields
multi = []
multi.append(deb822.Changes(CHANGES_FILE))
multi.append(
deb822.Changes(SIGNED_CHECKSUM_CHANGES_FILE %
CHECKSUM_CHANGES_FILE))
with open_utf8('test_Sources') as f:
multi.extend(deb822.Sources.iter_paragraphs(f))
for d in multi:
for key, value in d.items():
if key.lower() not in d.__class__._multivalued_fields:
self.assertTrue(isinstance(value, six.text_type))
def uploadpackage(changes, delete_existing=False):
"""Upload a package to the repository, using the changes file."""
if delete_existing:
data = open(changes).read()
source_package = deb822.Changes(data)
# Delete .upload file so dupload always refreshes the files.
upload = upload_filepath(source_package)
local("rm -f {0}".format(upload))
local("dupload {0}".format(changes))
if delete_existing:
# Remove previous package from repository.
distribution = "{0}-testing".format(source_package["Distribution"])
execute(repo.rm_packages, distribution, source_package["Source"])
# Import new packages into repository.
run("import-new-debs.sh")
def get_upload_data(changesfn):
achanges = deb822.Changes(file(changesfn))
changesname = os.path.basename(changesfn)
delay = os.path.basename(os.path.dirname(changesfn))
m = re.match(r'([0-9]+)-day', delay)
if m:
delaydays = int(m.group(1))
remainingtime = (delaydays > 0) * max(
0, 24 * 60 * 60 + os.stat(changesfn).st_mtime - time.time())
delay = "%d days %02d:%02d" % (max(
delaydays - 1, 0), int(
remainingtime / 3600), int(remainingtime / 60) % 60)
else:
delaydays = 0
remainingtime = 0
uploader = achanges.get('changed-by')
uploader = re.sub(r'^\s*(\S.*)\s+<.*>', r'\1', uploader)
with utils.open_file(changesfn) as f:
fingerprint = SignedFile(f.read(),
keyrings=get_active_keyring_paths(),
require_signature=False).fingerprint
if Cnf.has_key("Show-Deferred::LinkPath"):
isnew = 0
suites = get_suites_source_in(achanges['source'])
if 'unstable' not in suites and 'experimental' not in suites:
isnew = 1
for b in achanges['binary'].split():
suites = get_suites_binary_in(b)
if 'unstable' not in suites and 'experimental' not in suites:
isnew = 1
if not isnew:
# we don't link .changes because we don't want other people to
# upload it with the existing signature.
for afn in map(lambda x: x['name'], achanges['files']):
lfn = os.path.join(Cnf["Show-Deferred::LinkPath"], afn)
qfn = os.path.join(os.path.dirname(changesfn), afn)
if os.path.islink(lfn):
os.unlink(lfn)
if os.path.exists(qfn):
os.symlink(qfn, lfn)
os.chmod(qfn, 0o644)
return (max(delaydays - 1, 0) * 24 * 60 * 60 + remainingtime, changesname,
delay, uploader, achanges.get('closes', '').split(), fingerprint,
achanges, delaydays)
def search(self, needle, flags_search):
#flags_search = ['equals','contains','description','depends',]
xxx = os.path.join(self.folder, self.search_path)
list_distributions = os.listdir(xxx)
global_result = []
for release in list_distributions:
if os.path.exists(os.path.join(xxx, release, 'Release')):
index_file = deb822.Changes(
codecs.open(os.path.join(xxx, release, 'Release'),
encoding='utf-8'))
# Extract index files from Release file.
list_files = [
w.strip().split(" ")[2]
for w in index_file['md5sum'].split("\n")
if w.endswith('Packages')
]
for x in list_files:
with codecs.open(os.path.join(xxx, release, x),
encoding='utf-8') as fd:
for package in deb822.Deb822.iter_paragraphs(fd):
found = False
if 'source' in flags_search and \
( ( 'Source' in package.keys() and needle == package['Source'] ) or \
( 'Source' not in package.keys() and needle == package['Package'] ) ):
found = True
if ( 'equals' in flags_search and needle == package['Package'] ) or \
('contains' in flags_search and needle in package['Package'] ) or \
('description' in flags_search and needle in package['Description']):
found = True
if found:
global_result.append({
'package': dict(package),
'release': release,
'index': x
})
return global_result
def test_bug487902_multivalued_checksums(self):
"""New multivalued field Checksums was not handled correctly, see #487902."""
changesobj = deb822.Changes(CHECKSUM_CHANGES_FILE.splitlines())
self.assertEqual(CHECKSUM_CHANGES_FILE, changesobj.dump())
def test_bug457929_multivalued_dump_works(self):
"""dump() was not working in multivalued classes, see #457929."""
changesobj = deb822.Changes(CHANGES_FILE.splitlines())
self.assertEqual(CHANGES_FILE, changesobj.dump())
def run_queue(self, debug):
# check for *.changes in incoming directory
for changes_file in glob.glob(
os.path.join(self.incoming_path, "*.changes")):
# check gpg key
gpg_res = functions.command_result("gpgv %(changes)s" %
{"changes": changes_file},
output=False)
if gpg_res == 0:
# debug
functions.debug_message(
debug, "I: incoming - found %s" % changes_file)
# load .changes file
deb_changes = deb822.Changes(file(changes_file))
# check sha1 checksum
sha1_checksum = True
source_files = deb_changes['Checksums-Sha1']
for source_file in source_files:
source_file_path = os.path.join(self.incoming_path,
source_file['name'])
if source_file['size'] != str(
os.stat(source_file_path).st_size):
print "E: %s has not a valid size!" % source_file_path
sha1_checksum = False
elif source_file['sha1'] != functions.sha1file(
source_file_path):
print "E: %s has not a valid SHA1 checksum!" % source_file_path
sha1_checksum = False
if sha1_checksum:
# calculate the next build id
build_id = str(
len(glob.glob(os.path.join(self.builds_path, "*"))) +
1)
# build paths
build_dir = os.path.join(self.builds_path, build_id)
build_dir_source = os.path.join(build_dir, "source")
# move files to new source directory
os.makedirs(build_dir_source)
os.system("chmod 777 %s" % build_dir)
os.rename(
changes_file,
os.path.join(build_dir_source,
os.path.basename(changes_file)))
for source_file in source_files:
source_file_path = os.path.join(
self.incoming_path, source_file['name'])
os.rename(
source_file_path,
os.path.join(build_dir_source,
os.path.basename(
source_file['name'])))
# save package info
package_info = {}
package_info['build_id'] = build_id
package_info['timestamp'] = time.strftime(
"%Y/%m/%d %H:%M:%S")
package_info['package'] = deb_changes['Source']
package_info['version'] = deb_changes['Version']
package_info['maintainer'] = deb_changes['Maintainer']
package_info['changed_by'] = deb_changes['Changed-By']
package_info['source_dir'] = build_dir_source
package_info['dists'] = self.config.as_list('dists')
package_info['archs'] = self.config.as_list('archs')
package_info_filename = os.path.join(
build_dir, "build.json")
functions.json_save(package_info, package_info_filename)
# prepare single build requests
for dist in self.config.as_list('dists'):
for arch in self.config.as_list('archs'):
queue = {}
queue['build_id'] = build_id
queue['package'] = deb_changes['Source']
queue['version'] = deb_changes['Version']
queue['maintainer'] = deb_changes['Maintainer']
queue['changed_by'] = deb_changes['Changed-By']
queue['source_dir'] = build_dir_source
queue['dist'] = dist
queue['arch'] = arch
queue_filename = os.path.join(self.queue_path, \
"%(id)s_%(package)s_%(version)s_%(dist)s_%(arch)s.json" % \
{
"id": build_id,
"package": queue['package'],
"version": queue['version'],
"dist": dist,
"arch": arch,
})
functions.json_save(queue, queue_filename)
sendmail = os.popen("sendmail -t", "w")
sendmail.write("From: %s\n" %
"AutoMate <*****@*****.**>")
sendmail.write("To: %s\n" % deb_changes['Changed-By'])
if deb_changes['Maintainer'] != deb_changes['Changed-By']:
sendmail.write("Cc: %s\n" % deb_changes['Maintainer'])
sendmail.write("Subject: %(changes)s ACCEPTED into %(instance)s\n" % \
{
"changes": os.path.basename(changes_file),
"instance": self.instance
})
sendmail.write("\n")
sendmail.write("Accepted:\n")
for source_file in source_files:
sendmail.write("%s\n" % source_file['name'])
sendmail.write("\n")
sendmail.write("%s\n" % deb_changes['Description'])
sendmail.write("%s\n" % deb_changes['Changes'])
sendmail.write("\n")
sendmail.write("Thank you for your contribution to %s.\n" %
self.instance)
sendmail_result = sendmail.close()
else:
print "E: %s has not a valid GPG signature!" % os.path.basename(
changes_file)
def _get_changes_obj(self):
val = dict(deb822.Changes(open(self.get_changes_file(), 'r')))
val = self._obj_strip(val)
return val
def __init__(self, path):
self.path = path
self.filename = osp.basename(path)
self.changes = deb822.Changes(open(path))
self.dirname = osp.dirname(path)
async def build(self, build_lease, statesdb):
package = build_lease.package
architecture = build_lease.architecture
version = build_lease.version
binnmu_version = build_lease.binnmu_version
binnmu_changelog = build_lease.binnmu_changelog
with self.mkbuilddir(package, architecture, version) as builddir:
logging.info(
f'Starting build of {package}:{architecture} version {build_lease.versionstr()}'
)
proc = await asyncio.create_subprocess_exec(
'sbuild',
f'--chroot-mode={conf.sbuild_chroot_mode}',
'-c',
f'chroot:{conf.sbuild_chroot_name}',
'-d',
'unstable',
'--no-arch-any' if architecture == 'all' else '--no-arch-all',
*((f'--binNMU={binnmu_version}',
f'--make-binNMU={binnmu_changelog}')
if binnmu_version is not None else ()),
'-m',
conf.maintainer,
'--keyid',
conf.sbuild_key_id,
f'{package}_{version}',
cwd=builddir,
stdin=asyncio.subprocess.DEVNULL,
stdout=asyncio.subprocess.
DEVNULL, # sbuild creates log file itself, no need to save redundant stdout
stderr=asyncio.subprocess.DEVNULL)
await proc.wait()
# it's not technically necessary to hold the locks past this point - but
# if we do, that means that if the incoming_loop was waiting for the lock
# (which should happen on any build taking more than 5 minutes) then the
# next iteration will pick up the new artifacts we will place there
try:
logfile = next(p for p in builddir.glob('*.build')
if not p.is_symlink())
except StopIteration:
logging.warning('sbuild failed to create log file')
return
loginfo = self.scan_log(logfile)
logging.info(
f'Build of {package}:{architecture} version {build_lease.versionstr()} completed with status {loginfo["Status"]}'
)
logfile.rename(conf.rebuild_logs_dir / logfile.name)
await statesdb.register_log(loginfo)
if loginfo['Status'] == 'successful':
try:
changesfile = next(builddir.glob('*.changes'))
except StopIteration:
logging.warning('sbuild failed to create changes file')
return
incomingdir = conf.rebuild_repo_incoming_dir
with open(changesfile) as fh:
changes = deb822.Changes(fh)
for fname in (entry['name'] for entry in changes['files']):
(builddir / fname).rename(incomingdir / fname)
changesfile.rename(incomingdir / changesfile.name)
await build_lease.set_build_result('Uploaded')
elif loginfo['Status'] == 'attempted':
await build_lease.set_build_result('Attempted')
elif loginfo['Status'] == 'given-back':
await build_lease.set_build_result('Given-Back')
else:
warnings.warn(
f'Unrecognized status for build of {package} version {version}: {loginfo["Status"]}'
)
await build_lease.set_build_result('Internal-Error')
for changed_file in change_file['Checksums-Sha256']:
if os.path.splitext(changed_file['name'])[1] == '.buildinfo':
continue
if changed_file['sha256'] != checksums[changed_file['name']]:
raise Exception(
"Bad checksum on file {file}, local is {local}, bintray has {remote}"
.format(file=changed_file['name'],
local=changed_file['sha256'],
remote=checksums[changed_file['name']]))
else:
print("{file}: OK".format(file=changed_file['name']))
with io.open(changesFile, 'rb') as f:
d = deb822.Changes(f)
baseDir = os.path.dirname(os.path.realpath(changesFile))
parameters = {
'user': user,
'repo': repository,
'package': d['Source'],
'version': d['Version'],
'distributions': 'unstable',
'component': 'main',
'architectures': 'amd64,i386'
}
for file in d['Files']:
file_type = os.path.splitext(file['name'])[1]
if file_type == '.buildinfo':
continue
def parse_changes(changes_file):
""" Parse a changes file into a Changes class. """
with open(changes_file) as f:
return deb822.Changes(f)
