def write_profiles(main_thread_profile): if cprofiler_stats is None: return from pyLibrary import convert from mo_files import File cprofiler_stats.add(pstats.Stats(main_thread_profile.cprofiler)) stats = cprofiler_stats.pop_all() Log.note("aggregating {{num}} profile stats", num=len(stats)) acc = stats[0] for s in stats[1:]: acc.add(s) stats = [ { "num_calls": d[1], "self_time": d[2], "total_time": d[3], "self_time_per_call": d[2] / d[1], "total_time_per_call": d[3] / d[1], "file": (f[0] if f[0] != "~" else "").replace("\\", "/"), "line": f[1], "method": f[2].lstrip("<").rstrip(">") } for f, d, in iteritems(acc.stats) ] stats_file = File(FILENAME, suffix=convert.datetime2string(datetime.now(), "_%Y%m%d_%H%M%S")) stats_file.write(convert.list2tab(stats)) Log.note("profile written to {{filename}}", filename=stats_file.abspath)
def write_profiles(main_thread_profile): if cprofiler_stats is None: return from pyLibrary import convert from mo_files import File cprofiler_stats.add(pstats.Stats(main_thread_profile.cprofiler)) stats = cprofiler_stats.pop_all() Log.note("aggregating {{num}} profile stats", num=len(stats)) acc = stats[0] for s in stats[1:]: acc.add(s) stats = [{ "num_calls": d[1], "self_time": d[2], "total_time": d[3], "self_time_per_call": d[2] / d[1], "total_time_per_call": d[3] / d[1], "file": (f[0] if f[0] != "~" else "").replace("\\", "/"), "line": f[1], "method": f[2].lstrip("<").rstrip(">") } for f, d, in iteritems(acc.stats)] stats_file = File(FILENAME, suffix=convert.datetime2string(datetime.now(), "_%Y%m%d_%H%M%S")) stats_file.write(convert.list2tab(stats)) Log.note("profile written to {{filename}}", filename=stats_file.abspath)
def write(profile_settings): from pyLibrary import convert from pyLibrary.env.files import File profs = list(profiles.values()) for p in profs: p.stats = p.stats.end() stats = [ { "description": p.description, "num_calls": p.stats.count, "total_time": p.stats.count * p.stats.mean, "total_time_per_call": p.stats.mean, } for p in profs if p.stats.count > 0 ] stats_file = File(profile_settings.filename, suffix=convert.datetime2string(datetime.now(), "_%Y%m%d_%H%M%S")) if stats: stats_file.write(convert.list2tab(stats)) else: stats_file.write("<no profiles>") stats_file2 = File( profile_settings.filename, suffix=convert.datetime2string(datetime.now(), "_series_%Y%m%d_%H%M%S") ) if not profs: return max_samples = MAX([len(p.samples) for p in profs if p.samples]) if not max_samples: return r = range(max_samples) profs.insert(0, Dict(description="index", samples=r)) stats = [{p.description: wrap(p.samples)[i] for p in profs if p.samples} for i in r] if stats: stats_file2.write(convert.list2tab(stats))
def write(profile_settings): from mo_files import File from mo_logs.convert import datetime2string from mo_math import MAX from pyLibrary.convert import list2tab profs = list(profiles.values()) for p in profs: p.stats = p.stats.end() stats = [{ "description": p.description, "num_calls": p.stats.count, "total_time": p.stats.count * p.stats.mean, "total_time_per_call": p.stats.mean } for p in profs if p.stats.count > 0] stats_file = File(profile_settings.filename, suffix=datetime2string(datetime.now(), "_%Y%m%d_%H%M%S")) if stats: stats_file.write(list2tab(stats)) else: stats_file.write("<no profiles>") stats_file2 = File(profile_settings.filename, suffix=datetime2string(datetime.now(), "_series_%Y%m%d_%H%M%S")) if not profs: return max_samples = MAX([len(p.samples) for p in profs if p.samples]) if not max_samples: return r = range(max_samples) profs.insert(0, Data(description="index", samples=r)) stats = [{p.description: wrap(p.samples)[i] for p in profs if p.samples} for i in r] if stats: stats_file2.write(list2tab(stats))
def main(num): try: Log.start() results = [] test_json(results, "jsons.encoder", json_encoder, num) test_json(results, "jsons.encoder (again)", json_encoder, num) test_json(results, "scrub before json.dumps", cPythonJSONEncoder().encode, num) test_json(results, "override JSONEncoder.default()", EnhancedJSONEncoder().encode, num) test_json(results, "default json.dumps", json.dumps, num) # WILL CRASH, CAN NOT HANDLE DIVERSITY OF TYPES try: test_json(results, "scrubbed ujson", ujson.dumps, num) except Exception: pass Log.note("\n{{summary}}", summary=convert.list2tab(results)) finally: Log.stop()
def __exit__(self, exc_type, exc_val, exc_tb): ParserElement._parse = self.previous_parse profile = jx.sort( [{ "parser": text(parser), "cache_hits": cache, "matches": match, "failures": fail, "call_count": match + fail + cache, "total_parse": parse, "total_overhead": all - parse, "per_parse": parse / (match + fail), "per_overhead": (all - parse) / (match + fail + cache), } for parser, (cache, match, fail, parse, all) in timing.items()], {"total_parse": "desc"}, ) self.file.add_suffix(Date.now().format("%Y%m%d_%H%M%S")).write( convert.list2tab(profile))
def main(num): try: Log.start() results = [] test_json(results, "mo-json encoder", json_encoder, num) test_json(results, "mo-json encoder (again)", json_encoder, num) test_json(results, "scrub before json.dumps", cPythonJSONEncoder().encode, num) test_json(results, "override JSONEncoder.default()", EnhancedJSONEncoder().encode, num) test_json(results, "default json.dumps", json.dumps, num) # WILL CRASH, CAN NOT HANDLE DIVERSITY OF TYPES test_json(results, "typed json", typed_encoder.encode, num) # test_json(results, "scrubbed ujson", ujson.dumps, num) # THIS PLAIN CRASHES Log.note(u"\n{{summary}}", summary=convert.list2tab(results)) finally: Log.stop()
def write_profile(profile_settings, stats): from pyLibrary import convert from pyLibrary.env.files import File acc = stats[0] for s in stats[1:]: acc.add(s) stats = [{ "num_calls": d[1], "self_time": d[2], "total_time": d[3], "self_time_per_call": d[2] / d[1], "total_time_per_call": d[3] / d[1], "file": (f[0] if f[0] != "~" else "").replace("\\", "/"), "line": f[1], "method": f[2].lstrip("<").rstrip(">") } for f, d, in acc.stats.iteritems() ] stats_file = File(profile_settings.filename, suffix=convert.datetime2string(datetime.now(), "_%Y%m%d_%H%M%S")) stats_file.write(convert.list2tab(stats))
def write_profile(profile_settings, stats): from pyLibrary import convert from mo_files import File acc = stats[0] for s in stats[1:]: acc.add(s) stats = [{ "num_calls": d[1], "self_time": d[2], "total_time": d[3], "self_time_per_call": d[2] / d[1], "total_time_per_call": d[3] / d[1], "file": (f[0] if f[0] != "~" else "").replace("\\", "/"), "line": f[1], "method": f[2].lstrip("<").rstrip(">") } for f, d, in acc.stats.iteritems()] stats_file = File(profile_settings.filename, suffix=convert.datetime2string(datetime.now(), "_%Y%m%d_%H%M%S")) stats_file.write(convert.list2tab(stats))
def main(): since = Date.today() - Duration(SCATTER_RANGE) if config.database.host not in listwrap( config.analysis.expected_database_host): Log.error("Expecting database to be one of {{expected}}", expected=config.analysis.expected_database_host) if not config.analysis.interesting: Log.alert( "Expecting config file to have `analysis.interesting` with a json expression. All series are included." ) # SETUP DESTINATION deviant_summary = bigquery.Dataset( config.deviant_summary).get_or_create_table( read_only=True, kwargs=config.deviant_summary) if config.args.id: # EXIT EARLY AFTER WE GOT THE SPECIFIC IDS if len(config.args.id) < 4: step_detector.SHOW_CHARTS = True for signature_hash in config.args.id: process( signature_hash, since=since, source=config.database, deviant_summary=deviant_summary, show=True, ) return # DOWNLOAD if config.args.download: # GET INTERESTING SERIES where_clause = BQLang[jx_expression( config.analysis.interesting)].to_bq(deviant_summary.schema) # GET ALL KNOWN SERIES docs = list( deviant_summary.sql_query(f""" SELECT * EXCEPT (_rank, values) FROM ( SELECT *, row_number() over (partition by id order by last_updated desc) as _rank FROM {quote_column(deviant_summary.full_name)} ) a WHERE _rank=1 and {sql_iso(where_clause)} LIMIT {quote_value(DOWNLOAD_LIMIT)} """)) if len(docs) == DOWNLOAD_LIMIT: Log.warning("Not all signatures downloaded") File(config.args.download).write(list2tab(docs, separator=",")) # DEVIANT show_sorted( config=config, since=since, source=config.database, deviant_summary=deviant_summary, sort={ "value": { "abs": "overall_dev_score" }, "sort": "desc" }, limit=config.args.deviant, show_old=False, show_distribution=True, ) # MODAL show_sorted( config=config, since=since, source=config.database, deviant_summary=deviant_summary, sort="overall_dev_score", limit=config.args.modal, where={"eq": { "overall_dev_status": "MODAL" }}, show_distribution=True, ) # OUTLIERS show_sorted( config=config, since=since, source=config.database, deviant_summary=deviant_summary, sort={ "value": "overall_dev_score", "sort": "desc" }, limit=config.args.outliers, where={"eq": { "overall_dev_status": "OUTLIERS" }}, show_distribution=True, ) # SKEWED show_sorted( config=config, since=since, source=config.database, deviant_summary=deviant_summary, sort={ "value": { "abs": "overall_dev_score" }, "sort": "desc" }, limit=config.args.skewed, where={"eq": { "overall_dev_status": "SKEWED" }}, show_distribution=True, ) # OK show_sorted( config=config, since=since, source=config.database, deviant_summary=deviant_summary, sort={ "value": { "abs": "overall_dev_score" }, "sort": "desc" }, limit=config.args.ok, where={"eq": { "overall_dev_status": "OK" }}, show_distribution=True, ) # NOISE show_sorted( config=config, since=since, source=config.database, deviant_summary=deviant_summary, sort={ "value": { "abs": "relative_noise" }, "sort": "desc" }, where={"gte": { "num_pushes": 30 }}, limit=config.args.noise, ) # EXTRA show_sorted( config=config, since=since, source=config.database, deviant_summary=deviant_summary, sort={ "value": { "abs": "max_extra_diff" }, "sort": "desc" }, where={"lte": { "num_new_segments": 7 }}, limit=config.args.extra, ) # MISSING show_sorted( config=config, since=since, source=config.database, deviant_summary=deviant_summary, sort={ "value": { "abs": "max_missing_diff" }, "sort": "desc" }, where={"lte": { "num_old_segments": 6 }}, limit=config.args.missing, ) # PATHOLOGICAL show_sorted( config=config, since=since, source=config.database, deviant_summary=deviant_summary, sort={ "value": "num_segments", "sort": "desc" }, limit=config.args.pathological, )
}}, "from": 0, "size": 200000, "sort": [], "facets": {}, "fields": ["bug_id", "modified_by", "modified_ts", "comment"] })) comments = convert.json2value(convert.utf82unicode(http.post(settings.comments.url, data=request_comments).all_content)).hits.hits.fields results = [] for c in comments: errors = parse_comment(bugs[c.bug_id], c) results.extend(errors) tab = convert.list2tab(results) File(settings.output.tab).write(tab) def parse_short_desc(bug): parts = bug.short_desc.split("|") if len(parts) in [2, 3]: bug.result.test = parts[0].strip() bug.result.message = parts[1].strip() elif any(map(parts[0].strip().endswith, [".html", ".py", ".js", ".xul"])) and len(parts)>2: bug.result.test = parts[0].strip() bug.result.message = parts[1].strip() elif len(parts) >= 4: set_default(bug.result, parse_status(parts[0])) bug.result.test = parts[1].strip()