def execute_vizrequest(self, viz_request, options, schema, result_queue): viz = viz_request.viz sql_statement = viz.get_computed_filter_as_sql(schema) cursor = self.conn.cursor() viz_request.start_time = util.get_current_ms_time() cursor.execute(sql_statement) data = cursor.fetchall() viz_request.end_time = util.get_current_ms_time() cursor.close() results = {} for row in data: keys = [] for i, bin_desc in enumerate(viz.binning): if "width" in bin_desc: bin_width = bin_desc["width"] keys.append(str(int(row[i]))) else: keys.append(str(row[i])) key = ",".join(keys) results[key] = row[len(viz.binning):] viz_request.result = results result_queue.put(viz_request)
def execute_vizrequest(self, viz_request, options, schema, result_queue): viz = viz_request.viz sql_statement = viz.get_computed_filter_as_sql(schema) #calculate connection time # get a connection from the pool - block if non is available #connection = self.pool.get() connection = self.conn cursor = connection.cursor() viz_request.start_time = util.get_current_ms_time() cursor.execute(sql_statement) data = cursor.fetchall() viz_request.end_time = util.get_current_ms_time() # put connection back in the queue so the next thread can use it. cursor.close() #self.pool.put(connection) results = {} for row in data: keys = [] for i, bin_desc in enumerate(viz.binning): if "width" in bin_desc: bin_width = bin_desc["width"] keys.append(str(int(row[i]))) else: keys.append(str(row[i])) key = ",".join(keys) results[key] = row[len(viz.binning):] viz_request.result = results result_queue.put(viz_request)
def execute_vizrequest(self, viz_request, options, schema, result_queue): viz = viz_request.viz sql_statement = viz.get_computed_filter_as_sql(schema) #calculate connection time # get a connection from the pool - block if non is available # connection = self.pool.get() connection=self.conn viz_request.start_time = util.get_current_ms_time() try: editedSqlStatement = self.verdictdbedit(sql_statement) #print(editedSqlStatement) data = connection.sql(editedSqlStatement) except Exception as e: print(e, flush=True) viz_request.result = {} viz_request.margins = {} viz_request.end_time = util.get_current_ms_time() result_queue.put(viz_request) return viz_request.end_time = util.get_current_ms_time() # put connection back in the queue so the next thread can use it. #cursor.close() #connection.close() #connection=self.create_connection() #self.pool.put(connection) results = {} for i, row in data.iterrows(): keys = [] if row[0] is None: continue for i, bin_desc in enumerate(viz_request.viz.binning): if "width" in bin_desc: bin_width = bin_desc["width"] keys.append(str(int(row[0]))) else: keys.append(str(row[0]).strip()) key = ",".join(keys) row = list(row) for i, r in enumerate(row): if isinstance(r, decimal.Decimal): row[i] = float(r) results[key] = row[1] viz_request.result = results #viz_request.margins = margins viz_request.margins = {} result_queue.put(viz_request) print("delivering...")
def process_request(self, viz_request, options, schema, result_queue): # record start time viz_request.start_time = util.get_current_ms_time() # print SQL translation of request and simulate query execution # print(viz_request.viz.get_computed_filter_as_sql(schema)) time.sleep(random.randint(1, 1000)/float(1000)) # record end time viz_request.end_time = util.get_current_ms_time() # write an empty result to the viz_request viz_request.result = {} # notify IDEBench that processing is done by writing it to the result buffer result_queue.put(viz_request)
def run(self): self.vizgraph = VizGraph() with open(self.get_workflow_path()) as f: json_data = json.load(f) for s in json_data["setup"]: self.vizgraph.add_viz(s) for s in json_data["setup"]: self.vizgraph.apply_interaction(Operation(s)) self.workflow_interactions = json_data["interactions"] self.operation_results = OrderedDict({ "args": vars(self.options), "results": OrderedDict() }) self.current_interaction_index = 0 self.current_vizrequest_index = 0 self.benchmark_start_time = util.get_current_ms_time() try: logger.info("calling \"workflow_start\" on driver") self.driver.workflow_start() except AttributeError: pass global do_poll do_poll = True def poll_results(slf, queue): global count while do_poll: try: process_result = queue.get(timeout=1) except Empty: logger.info("result queue empty... trying again") continue if process_result is None: continue slf.deliver_viz_request([process_result]) logger.info("stopped polling results") try: while queue.get(timeout=0.01): pass except Empty: logger.info("result queue cleard") if not self.options.groundtruth: thread = Thread(target = poll_results, args = (self, IDEBench.result_queue)) thread.start() interaction_index = 0 while interaction_index < len(self.workflow_interactions): self.process_interaction(interaction_index) interaction_index +=1 do_poll = False if not self.options.groundtruth: thread.join() self.end_run()
def __init__(self, operation_id, parent_operation_id, viz): self.operation_id = operation_id self.parent_operation_id = parent_operation_id self.viz = viz self.start_time = util.get_current_ms_time() self.end_time = None self.result = None self.margins = None self.delivered = False self.bins = None self.timedout = False self.t_start = 0 self.t_pause = 0 self.progress = 0
def process_interaction(self, interaction_index): logger.info("interaction %i" % interaction_index) interaction = self.workflow_interactions[interaction_index] next_interaction = self.workflow_interactions[interaction_index + 1] if interaction_index +1 < len(self.workflow_interactions) else None vizs_to_request = self.vizgraph.apply_interaction(Operation(interaction)) expected_start_time = interaction["time"] viz_requests = [] for viz in vizs_to_request: viz_requests.append(VizRequest(self.current_vizrequest_index, self.current_interaction_index, expected_start_time, viz)) self.current_vizrequest_index += 1 # TODO: document this feature try: self.driver.before_requests(self.options, self.schema, IDEBench.result_queue) except AttributeError: pass procs = [] nprocs = len(viz_requests) if self.options.groundtruth or (hasattr(self.driver, "use_single_process") and self.driver.use_single_process): for viz_request in viz_requests: op_id = str(viz_request.operation_id) if op_id in self.gt_for_result["results"] and self.gt_for_result["results"][op_id]["output"]: self.driver.process_request(viz_request, self.options, self.schema, IDEBench.result_queue) self.deliver_viz_request([IDEBench.result_queue.get()]) self.counter += 1 else: for viz_request in viz_requests: thread = Thread(target = self.driver.process_request, args = (viz_request, self.options, self.schema, IDEBench.result_queue )) procs.append(thread) thread.start() time.sleep(0.002) # so the request threads do not overwhelm some of the drivers (particularly verdictdb) resultlist = [] delay = 0 think_time = 0 if "time" in interaction and next_interaction: original_think_time = next_interaction["time"] - interaction["time"] delay = min(0, next_interaction["time"] - (util.get_current_ms_time() - self.benchmark_start_time)) think_time = max(0, delay + original_think_time) else: think_time = self.options.settings_thinktime if not self.options.groundtruth: time.sleep(think_time / 1000) self.current_interaction_index += 1
def process_request(self, viz_request, options, schema, out_q): print("processsing..." + str(viz_request.operation_id)) if viz_request.viz.binning: sql_statement = viz_request.viz.get_computed_filter_as_sql(schema) sql_statement = sql_statement.replace( schema.get_fact_table_name(), "%s_%s%s" % (schema.get_fact_table_name(), options.settings_size, "n" if options.settings_normalized else "")) if self.can_execute_online(sql_statement): sql_statement = sql_statement.replace("SELECT ", "SELECT ONLINE ") sql_statement += " WITHTIME %s CONFIDENCE 95" % options.settings_time_requirement sql_statement += " REPORTINTERVAL %s;" % options.settings_time_requirement connection, cursor = self.create_connection( options.settings_time_requirement + 20) else: connection, cursor = self.create_connection( options.settings_time_requirement) viz_request.start_time = util.get_current_ms_time() try: cursor.execute(sql_statement) except psycopg2.extensions.QueryCanceledError as qce: viz_request.result = {} viz_request.margins = {} viz_request.timedout = True viz_request.end_time = util.get_current_ms_time() out_q.put(viz_request) return data = cursor.fetchall() viz_request.end_time = util.get_current_ms_time() connection.close() results = {} margins = {} for row in data: keys = [] if row[0] is None: continue startindex = 3 if self.can_execute_online(sql_statement) else 0 for i, bin_desc in enumerate(viz_request.viz.binning): if "width" in bin_desc: bin_width = bin_desc["width"] keys.append(str(int(row[i + startindex]))) else: keys.append(str(row[startindex + i]).strip()) key = ",".join(keys) row = list(row) for i, r in enumerate(row): if isinstance(r, decimal.Decimal): row[i] = float(r) if startindex == 3: results[key] = row[len(viz_request.viz.binning) + startindex:-1] else: results[key] = row[len(viz_request.viz.binning) + startindex:] if self.can_execute_online(sql_statement) and startindex == 3: margins[key] = row[len(row) - 1:] viz_request.result = results viz_request.margins = margins out_q.put(viz_request) print("delivering...")