def write_results_to_datastore(grouped): """ Saves results to a database """ from perf_result_datastore import PerfResultDataStore print 'Saving perf results to database' current_date = datetime.now() data_store = PerfResultDataStore(host=options.db_host, username=options.db_username, password=options.db_password, database_name=options.db_name) run_info_id = data_store.insert_run_info(options.lab_run_info) for results in all_query_results(grouped): first_query_result = results[RESULT_LIST][0] executor_name = first_query_result[EXECUTOR_NAME] workload = first_query_result[QUERY][WORKLOAD_NAME] scale_factor = first_query_result[QUERY][SCALE_FACTOR] query_name = first_query_result[QUERY][NAME] query = first_query_result[QUERY][QUERY_STR] file_format = first_query_result[QUERY][TEST_VECTOR][FILE_FORMAT] compression_codec = first_query_result[QUERY][TEST_VECTOR][COMPRESSION_CODEC] compression_type = first_query_result[QUERY][TEST_VECTOR][COMPRESSION_TYPE] avg_time = results[AVG] stddev = results[STDDEV] num_clients = results[NUM_CLIENTS] num_iterations = results[ITERATIONS] runtime_profile = first_query_result[RUNTIME_PROFILE] file_type_id = data_store.get_file_format_id( file_format, compression_codec, compression_type) if file_type_id is None: print 'Skipping unkown file type: %s / %s' % (file_format, compression) continue workload_id = data_store.get_workload_id(workload, scale_factor) if workload_id is None: workload_id = data_store.insert_workload_info(workload, scale_factor) query_id = data_store.get_query_id(query_name, query) if query_id is None: query_id = data_store.insert_query_info(query_name, query) data_store.insert_execution_result( query_id = query_id, workload_id = workload_id, file_type_id = file_type_id, num_clients = num_clients, cluster_name = options.cluster_name, executor_name = executor_name, avg_time = avg_time, stddev = stddev, run_date = current_date, version = options.build_version, notes = options.report_description, run_info_id = run_info_id, num_iterations = num_iterations, runtime_profile = runtime_profile, is_official = options.is_official)
def write_results_to_datastore(results): """ Saves results to a database """ current_date = datetime.now() data_store = PerfResultDataStore(host=options.db_host, username=options.db_username, password=options.db_password, database_name=options.db_name) run_info_id = data_store.insert_run_info(options.lab_run_info) for row in results: # We ignore everything after the stddev column executor, workload, scale_factor, query_name, query, file_format,\ compression, avg_time, stddev = row[0:STDDEV_IDX + 1] # Instead of storing 'N/A' in the database we want to store NULL avg_time = avg_time if avg_time and avg_time != 'N/A' else 'NULL' stddev = stddev if stddev and stddev != 'N/A' else 'NULL' file_type_id = data_store.get_file_format_id(file_format, compression) if file_type_id is None: print 'Skipping unkown file type: %s / %s' % (file_format, compression) continue workload_id = data_store.get_workload_id(workload, scale_factor) if workload_id is None: workload_id = data_store.insert_workload_info( workload, scale_factor) query_id = data_store.get_query_id(query_name, query) if query_id is None: query_id = data_store.insert_query_info(query_name, query) data_store.insert_execution_result( query_id=query_id, workload_id=workload_id, file_type_id=file_type_id, num_clients=int(row[NUM_CLIENTS_IDX]), cluster_name=options.cluster_name, executor_name=executor, avg_time=avg_time, stddev=stddev, run_date=current_date, version=options.build_version, notes=options.report_description, run_info_id=run_info_id, num_iterations=int(row[NUM_ITERS_IDX]), runtime_profile=row[RUNTIME_PROFILE_IDX], is_official=options.is_official)
def write_results_to_datastore(grouped): """ Saves results to a database """ from perf_result_datastore import PerfResultDataStore LOG.info('Saving perf results to database') run_date = str(datetime.now()) with PerfResultDataStore( host=options.db_host, port=options.db_port, database_name=options.db_name, use_secure_connection=options.use_secure_connection) as data_store: for results in all_query_results(grouped): for query_result in results[RESULT_LIST]: data_store.insert_execution_result( query_name=query_result[QUERY][NAME], query_string=query_result[QUERY][QUERY_STR], workload_name=query_result[QUERY][WORKLOAD_NAME], scale_factor=query_result[QUERY][SCALE_FACTOR], file_format=query_result[QUERY][TEST_VECTOR][FILE_FORMAT], compression_codec=query_result[QUERY][TEST_VECTOR][COMPRESSION_CODEC], compression_type=query_result[QUERY][TEST_VECTOR][COMPRESSION_TYPE], num_clients=results[NUM_CLIENTS], num_iterations=results[ITERATIONS], cluster_name=options.cluster_name, executor_name=query_result[EXECUTOR_NAME], exec_time=query_result[TIME_TAKEN], run_date=run_date, version=options.build_version, run_info=options.lab_run_info, user_name=options.run_user_name, runtime_profile = 'N/A' if options.hive_results else query_result[RUNTIME_PROFILE])
def write_results_to_datastore(results): """ Saves results to a database """ current_date = datetime.now() data_store = PerfResultDataStore(host=options.db_host, username=options.db_username, password=options.db_password, database_name=options.db_name) run_info_id = data_store.insert_run_info(options.lab_run_info) for row in results: # We ignore everything after the stddev column executor, workload, scale_factor, query_name, query, file_format,\ compression, avg_time, stddev = row[0:STDDEV_IDX + 1] # Instead of storing 'N/A' in the database we want to store NULL avg_time = avg_time if avg_time and avg_time != 'N/A' else 'NULL' stddev = stddev if stddev and stddev != 'N/A' else 'NULL' file_type_id = data_store.get_file_format_id(file_format, compression) if file_type_id is None: print 'Skipping unkown file type: %s / %s' % (file_format, compression) continue workload_id = data_store.get_workload_id(workload, scale_factor) if workload_id is None: workload_id = data_store.insert_workload_info(workload, scale_factor) query_id = data_store.get_query_id(query_name, query) if query_id is None: query_id = data_store.insert_query_info(query_name, query) data_store.insert_execution_result( query_id=query_id, workload_id=workload_id, file_type_id=file_type_id, num_clients=int(row[NUM_CLIENTS_IDX]), cluster_name=options.cluster_name, executor_name=executor, avg_time=avg_time, stddev=stddev, run_date=current_date, version=options.build_version, notes=options.report_description, run_info_id=run_info_id, num_iterations=int(row[NUM_ITERS_IDX]), runtime_profile=row[RUNTIME_PROFILE_IDX], is_official=options.is_official)