def handle(self, *args, **options):
if len(args) != 1:
raise CommandError("Need to (only) specify project/branch")
project = args[0]
server_params = urlparse(options['server'])
pc = PerfherderClient(protocol=server_params.scheme,
host=server_params.netloc)
signatures = pc.get_performance_signatures(
project,
time_interval=PerformanceTimeInterval.NINETY_DAYS)
if options['filter_props']:
for kv in options['filter_props']:
if ':' not in kv or len(kv) < 3:
raise CommandError("Must specify --filter-props as "
"'key:value'")
k, v = kv.split(':')
signatures = signatures.filter((k, v))
if options['time_interval'] is None:
time_intervals = PerformanceTimeInterval.all_valid_time_intervals()
else:
time_intervals = [options['time_interval']]
with concurrent.futures.ProcessPoolExecutor(
options['num_workers']) as executor:
futures = []
for signature_hash in signatures.get_signature_hashes():
futures.append(executor.submit(_add_series, server_params,
project,
time_intervals,
signature_hash,
signatures[signature_hash],
options['mysql_debug'],
options['verbose']))
for future in futures:
try:
future.result()
except Exception, e:
self.stderr.write("FAIL: {}".format(e))
# shutdown any pending tasks and exit (if something
# is in progress, no wait to stop it)
executor.shutdown(wait=False)
for future in futures:
future.cancel()
raise CommandError(
"Failed to import performance data: {}".format(e))
def handle(self, *args, **options):
if len(args) != 1:
raise CommandError("Need to (only) specify project/branch")
project = args[0]
server_params = urlparse(options['server'])
pc = PerfherderClient(protocol=server_params.scheme,
host=server_params.netloc)
signatures = pc.get_performance_signatures(
project, time_interval=PerformanceTimeInterval.NINETY_DAYS)
if options['filter_props']:
for kv in options['filter_props']:
if ':' not in kv or len(kv) < 3:
raise CommandError("Must specify --filter-props as "
"'key:value'")
k, v = kv.split(':')
signatures = signatures.filter((k, v))
if options['time_interval'] is None:
time_intervals = PerformanceTimeInterval.all_valid_time_intervals()
else:
time_intervals = [options['time_interval']]
with concurrent.futures.ProcessPoolExecutor(
options['num_workers']) as executor:
futures = []
for signature_hash in signatures.get_signature_hashes():
futures.append(
executor.submit(_add_series, server_params, project,
time_intervals, signature_hash,
signatures[signature_hash],
options['mysql_debug'],
options['verbose']))
for future in futures:
try:
future.result()
except Exception, e:
self.stderr.write("FAIL: {}".format(e))
# shutdown any pending tasks and exit (if something
# is in progress, no wait to stop it)
executor.shutdown(wait=False)
for future in futures:
future.cancel()
raise CommandError(
"Failed to import performance data: {}".format(e))
def _rewrite_data(self, project, mysql_debug):
signature_mapping = {}
extra_subtest_signatures = {}
with JobsModel(project) as jm:
jm.DEBUG = mysql_debug
summary = jm.get_performance_series_summary(
max(PerformanceTimeInterval.all_valid_time_intervals()))
# first pass: rewrite non-summary tests
for (signature_hash, signature_properties) in summary.iteritems():
if self._signature_needs_rewriting(signature_properties,
signature_hash) and \
'subtest_signatures' not in signature_properties:
new_hash = self._rewrite_series(jm, signature_hash,
signature_properties,
None, None)
signature_mapping[signature_hash] = new_hash
elif (not signature_properties.get('subtest_signatures') and
signature_properties.get('test') not in
Command.COUNTER_TESTS):
# in case this script got interrupted, keep track of
# subtest signatures which have already been converted
suitekey = self._get_suitekey(signature_properties)
if extra_subtest_signatures.get(suitekey):
extra_subtest_signatures[suitekey].append(
signature_hash)
else:
extra_subtest_signatures[suitekey] = [signature_hash]
# second pass: rewrite summary tests
for (signature_hash, signature_properties) in summary.iteritems():
if (self._signature_needs_rewriting(signature_properties,
signature_hash) and
signature_properties.get('subtest_signatures')):
self._rewrite_series(jm, signature_hash,
signature_properties,
signature_mapping,
extra_subtest_signatures)
def _rewrite_data(self, project, mysql_debug):
signature_mapping = {}
extra_subtest_signatures = {}
with JobsModel(project) as jm:
jm.DEBUG = mysql_debug
summary = jm.get_performance_series_summary(
max(PerformanceTimeInterval.all_valid_time_intervals()))
# first pass: rewrite non-summary tests
for (signature_hash, signature_properties) in summary.iteritems():
if self._signature_needs_rewriting(signature_properties,
signature_hash) and \
'subtest_signatures' not in signature_properties:
new_hash = self._rewrite_series(jm, signature_hash,
signature_properties, None,
None)
signature_mapping[signature_hash] = new_hash
elif (not signature_properties.get('subtest_signatures')
and signature_properties.get('test')
not in Command.COUNTER_TESTS):
# in case this script got interrupted, keep track of
# subtest signatures which have already been converted
suitekey = self._get_suitekey(signature_properties)
if extra_subtest_signatures.get(suitekey):
extra_subtest_signatures[suitekey].append(
signature_hash)
else:
extra_subtest_signatures[suitekey] = [signature_hash]
# second pass: rewrite summary tests
for (signature_hash, signature_properties) in summary.iteritems():
if (self._signature_needs_rewriting(signature_properties,
signature_hash)
and signature_properties.get('subtest_signatures')):
self._rewrite_series(jm, signature_hash,
signature_properties,
signature_mapping,
extra_subtest_signatures)
def _rewrite_series(self, jm, signature_hash, signature_properties,
subtest_signature_mapping, extra_subtest_signatures):
new_props = TalosDataAdapter._transform_signature_properties(
signature_properties,
significant_keys=Command.SIGNIFICANT_KEYS)
if 'subtest_signatures' in new_props:
suitekey = self._get_suitekey(new_props)
# rewrite a new set of subtest signatures
old_subtest_signatures = new_props['subtest_signatures']
new_subtest_signatures = set()
for old_signature in old_subtest_signatures:
try:
new_subtest_signatures.add(
subtest_signature_mapping[old_signature])
except:
# key may not exist if script interrupted, get
# suite signatures via extra_subtest_signatures
for sig in extra_subtest_signatures.get(suitekey, []):
new_subtest_signatures.add(sig)
new_props['subtest_signatures'] = sorted(new_subtest_signatures)
new_hash = TalosDataAdapter.get_series_signature(new_props)
print "%s -> %s" % (signature_hash, new_hash)
jm.set_series_signature(new_hash, new_props)
for time_interval in PerformanceTimeInterval.all_valid_time_intervals():
series_list = jm.get_performance_series_from_signatures(
[signature_hash], time_interval)
series = utils.decompress_if_needed(series_list[0]['blob'])
jm.store_performance_series(time_interval, 'talos_data',
str(new_hash), series)
jm.execute(proc='jobs.deletes.delete_performance_series',
placeholders=[signature_hash])
jm.execute(proc='jobs.deletes.delete_series_signature',
placeholders=[signature_hash])
return new_hash
def _rewrite_series(self, jm, signature_hash, signature_properties,
subtest_signature_mapping, extra_subtest_signatures):
new_props = TalosDataAdapter._transform_signature_properties(
signature_properties, significant_keys=Command.SIGNIFICANT_KEYS)
if 'subtest_signatures' in new_props:
suitekey = self._get_suitekey(new_props)
# rewrite a new set of subtest signatures
old_subtest_signatures = new_props['subtest_signatures']
new_subtest_signatures = set()
for old_signature in old_subtest_signatures:
try:
new_subtest_signatures.add(
subtest_signature_mapping[old_signature])
except:
# key may not exist if script interrupted, get
# suite signatures via extra_subtest_signatures
for sig in extra_subtest_signatures.get(suitekey, []):
new_subtest_signatures.add(sig)
new_props['subtest_signatures'] = sorted(new_subtest_signatures)
new_hash = TalosDataAdapter.get_series_signature(new_props)
print "%s -> %s" % (signature_hash, new_hash)
jm.set_series_signature(new_hash, new_props)
for time_interval in PerformanceTimeInterval.all_valid_time_intervals(
):
series_list = jm.get_performance_series_from_signatures(
[signature_hash], time_interval)
series = series_list[0]['blob']
jm.store_performance_series(time_interval, 'talos_data',
str(new_hash), series)
jm.jobs_execute(proc='jobs.deletes.delete_performance_series',
placeholders=[signature_hash])
jm.jobs_execute(proc='jobs.deletes.delete_series_signature',
placeholders=[signature_hash])
return new_hash
