def get(self): """Load bigquery stats from GCS.""" if not big_query.get_bucket(): logs.log_error('Loading stats to BigQuery failed: missing bucket name.') return # Retrieve list of fuzzers before iterating them, since the query can expire # as we create the load jobs. bigquery_client = big_query.get_api_client() for fuzzer in list(data_types.Fuzzer.query()): logs.log('Loading stats to BigQuery for %s.' % fuzzer.name) self._load_data(bigquery_client, fuzzer.name)
def get_gcs_stats_path(kind, fuzzer, timestamp): """Return gcs path in the format "/bucket/path/to/containing_dir/" for the given fuzzer, job, and timestamp or revision.""" bucket_name = big_query.get_bucket() if not bucket_name: return None datetime_value = datetime.datetime.utcfromtimestamp(timestamp) dir_name = data_types.coverage_information_date_to_string(datetime_value) path = '/%s/%s/%s/date/%s/' % (bucket_name, fuzzer, kind, dir_name) return path
def upload_stats(stats_list, filename=None): """Upload the fuzzer run to the bigquery bucket. Assumes that all the stats given are for the same fuzzer/job run.""" if not stats_list: logs.log_error("Failed to upload fuzzer stats: empty stats.") return if not isinstance(stats_list, list): raise AssertionError bucket_name = big_query.get_bucket() if not bucket_name: logs.log_error("Failed to upload fuzzer stats: missing bucket name.") return kind = stats_list[0].kind fuzzer = stats_list[0].fuzzer # Group all stats for fuzz targets. fuzzer_or_engine_name = get_fuzzer_or_engine_name(fuzzer) if not filename: # Generate a random filename. filename = "%016x" % random.randint(0, (1 << 64) - 1) + ".json" # Handle runs that bleed into the next day. def timestamp_start_of_day(s): return utils.utc_date_to_timestamp( datetime.datetime.utcfromtimestamp(s.timestamp).date()) stats_list.sort(key=lambda s: s.timestamp) for timestamp, stats in itertools.groupby(stats_list, timestamp_start_of_day): upload_data = "\n".join(stat.to_json() for stat in stats) day_path = ("gs:/" + get_gcs_stats_path( kind, fuzzer_or_engine_name, timestamp=timestamp) + filename) if not storage.write_data(upload_data, day_path): logs.log_error("Failed to upload FuzzerRun.")