def update_bucketversions(bucketid, oops, key):
global bv_full_cf
global bv_day_cf
if 'ProblemType' not in oops:
return
key = uuid.UUID(key)
version = ''
package = oops.get('Package', '')
release = oops.get('DistroRelease', '')
# These are tuples of (value, timestamp)
if release:
release = release[0].encode('ascii', errors='ignore')
if package:
package = package[0].encode('ascii', errors='ignore')
package, version = split_package_and_version(package)
if not package:
counts['no_package'] += 1
if not release:
counts['no_release'] += 1
if bv_full_cf:
try:
bv_full_cf.insert((bucketid, release, version), {key: ''})
except:
print(repr(bucketid), type(bucketid))
print(repr(release), type(release))
print(repr(version), type(version))
print(repr(key), type(key))
raise
# Unlike the code in oops-repository, we use the day of the OOPS, so we
# don't end up with one very wide row for when we ran this. We'll do a
# repair on everything up to the current day after this is run.
if bv_day_cf:
ts = oops['ProblemType'][1]
day_key = time.strftime('%Y%m%d', time.gmtime(ts / 1000000))
try:
bv_day_cf.insert(day_key, {(bucketid, release, version): ''})
except:
print(repr(bucketid), type(bucketid))
print(repr(release), type(release))
print(repr(version), type(version))
print(repr(key), type(key))
raise
if not exec_path:
metrics.meter('missing.missing_executable_path')
if exec_path.endswith('apportcheckresume'):
# LP: #1316841 bad duplicate signatures
if release == 'Ubuntu 14.04' and \
apport_version == '2.14.1-0ubuntu3.1':
metrics.meter('missing.missing_suspend_resume_data')
return (False, 'Incomplete suspend resume data found in report.')
failure = data.get('Failure', '')
if failure == 'suspend/resume' and 'ProcMaps' in data:
# this is not useful as it is from the resuming system
data.pop('ProcMaps')
else:
metrics.meter('success.problem_type.%s' % problem_type)
package, version = utils.split_package_and_version(package)
# src_version is None and is never used, nor should it be.
src_package, src_version = utils.split_package_and_version(src_package)
fields = utils.get_fields_for_bucket_counters(problem_type, release,
package, version, pkg_arch,
rootfs_build, channel,
device_name, device_image)
# generic counter for crashes about a source package which is used by the
# phased-updater and only includes official Ubuntu packages and not those
# from systems under auto testing.
if not third_party and not automated_testing and problem_type == 'Crash':
update_release_pkg_counter(counters_fam, release, src_package, day_key)
if version == '':
metrics.meter('missing.missing_package_version')
else:
for bucket, instances in bucket_cf.get_range(include_timestamp=True,
buffer_size=2*1024):
print_totals()
str_instances = [str(instance) for instance in instances]
counts += 1
#if counts > 1000:
# break
# keep track of (system, version) as we only want to count each system once
# per version
insertions = []
for instance in chunks(str_instances, 3):
oopses = oops_cf.multiget(instance, columns=cols)
for oops in oopses:
data = oopses[oops]
if Counter(cols) != Counter(data.keys()):
continue
system = data.get('SystemIdentifier')
package = data.get('Package', '')
version = None
if package:
package, version = split_package_and_version(package)
if version == '':
continue
if (system, version) in insertions:
continue
key = (to_utf8(bucket), to_utf8(version))
#print('Would insert %s = {%s, ""}' % (key, to_utf8(system)))
insertions.append((system, version))
bucketversionsystems_cf.insert(key, {system: ''})
print_totals(force=True)
if not alias:
continue
# if alias and channel are the same we've already counted it
if alias == channel:
continue
else:
remove_channel = False
with open('channels-to-remove.txt', 'r') as f:
channels = f.readlines()
if '%s\n' % channel not in channels:
remove_channel = True
if remove_channel:
with open('channels-to-remove.txt', 'a') as f:
f.write('%s\n' % channel)
device_name = sii_dict.get('device name', '')
package, version = utils.split_package_and_version(
data.get('Package', ''))
if release and alias and package and version and pkg_arch:
keys.append('%s:%s:%s:%s:%s' %
(release, alias, package, version, pkg_arch))
keys.append('%s:%s:%s:%s' %
(release, alias, package, pkg_arch))
keys.append('%s:%s:%s' % (release, alias, pkg_arch))
keys.append('%s:%s:%s:%s' %
(alias, package, version, pkg_arch))
keys.append('%s:%s:%s' % (alias, package, pkg_arch))
keys.append('%s:%s' % (alias, pkg_arch))
if device_name:
keys.append('%s:%s:%s:%s:%s:%s' %
(release, alias, device_name, package, version,
pkg_arch))
keys.append(
# use a prepared statement which is less resource intensive
oops_lookup_stmt = session.prepare('SELECT * FROM "OOPS" WHERE key=?')
missing_data = {}
for oops in oopses:
data = {}
#hex_oops = '0x' + hexlify(oops.value)
# double quotes are needed to make table names case-sensitive
#oops_data = session.execute('SELECT * FROM "OOPS" WHERE key = %s' %
# hex_oops)
oops_data = session.execute(oops_lookup_stmt, [oops.value])
# all the column "names" are column1 so make a dictionary of keys: values
for od in oops_data:
data[od.column1] = od.value
if 'Package' not in data:
continue
package, version = utils.split_package_and_version(data['Package'])
if not package:
print("%s%s missing package name" % (URL, oops.value))
if not version:
if package not in missing_data:
missing_data[package] = [oops.value]
else:
missing_data[package].append(oops.value)
for datum in missing_data:
print("Errors regarding %s missing package version" % (datum))
for oops in missing_data[datum]:
print(" %s%s" % (URL, oops))