"ordering": -1,
"stats": geo_mean(total)
}
)
new_records.append(new_record)
# ADD RECORD FOR GRAPH SERVER SUMMARY
new_record = Struct(
test_machine=r.test_machine,
datazilla=r.datazilla,
testrun=r.testrun,
test_build=r.test_build,
result={
"test_name": "summary_old",
"ordering": -1,
"stats": Stats(samples=Q.sort(total.mean)[:len(total)-1:])
}
)
new_records.append(new_record)
return new_records
except Exception, e:
Log.error("Transformation failure on id={{id}}", {"id":id}, e)
def stats(values):
"""
RETURN LOTS OF AGGREGATES
"""
if values == None:
return None
"length": len(CNV.object2JSON(line)),
"prefix": CNV.object2JSON(line)[0:130]
}, e)
missing_ids = missing_ids - existing_ids
#COPY MISSING DATA TO ES
try:
with ThreadedQueue(es, size=nvl(es.settings.batch_size, 100)) as es_sink:
with ThreadedQueue(File(settings.param.output_file), size=50) as file_sink:
simple_etl = functools.partial(etl, *[es_sink, file_sink, settings, transformer, max_existing_id])
num_not_found = 0
with Multithread(simple_etl, threads=settings.production.threads) as many:
results = many.execute([
{"id": id}
for id in Q.sort(missing_ids)[:nvl(settings.production.step, NUM_PER_BATCH):]
])
for result in results:
if not result:
num_not_found += 1
if num_not_found > nvl(settings.production.max_tries, 10):
many.inbound.pop_all() # CLEAR THE QUEUE OF OTHER WORK
many.stop()
break
else:
num_not_found = 0
except (KeyboardInterrupt, SystemExit):
Log.println("Shutdown Started, please be patient")
except Exception, e:
Log.error("Unusual shutdown!", e)
