def execute_vizrequest(self, viz_request, options, schema, result_queue):
viz = viz_request.viz
sql_statement = viz.get_computed_filter_as_sql(schema)
cursor = self.conn.cursor()
viz_request.start_time = util.get_current_ms_time()
cursor.execute(sql_statement)
data = cursor.fetchall()
viz_request.end_time = util.get_current_ms_time()
cursor.close()
results = {}
for row in data:
keys = []
for i, bin_desc in enumerate(viz.binning):
if "width" in bin_desc:
bin_width = bin_desc["width"]
keys.append(str(int(row[i])))
else:
keys.append(str(row[i]))
key = ",".join(keys)
results[key] = row[len(viz.binning):]
viz_request.result = results
result_queue.put(viz_request)
def execute_vizrequest(self, viz_request, options, schema, result_queue):
viz = viz_request.viz
sql_statement = viz.get_computed_filter_as_sql(schema)
#calculate connection time
# get a connection from the pool - block if non is available
#connection = self.pool.get()
connection = self.conn
cursor = connection.cursor()
viz_request.start_time = util.get_current_ms_time()
cursor.execute(sql_statement)
data = cursor.fetchall()
viz_request.end_time = util.get_current_ms_time()
# put connection back in the queue so the next thread can use it.
cursor.close()
#self.pool.put(connection)
results = {}
for row in data:
keys = []
for i, bin_desc in enumerate(viz.binning):
if "width" in bin_desc:
bin_width = bin_desc["width"]
keys.append(str(int(row[i])))
else:
keys.append(str(row[i]))
key = ",".join(keys)
results[key] = row[len(viz.binning):]
viz_request.result = results
result_queue.put(viz_request)
def execute_vizrequest(self, viz_request, options, schema, result_queue):
viz = viz_request.viz
sql_statement = viz.get_computed_filter_as_sql(schema)
#calculate connection time
# get a connection from the pool - block if non is available
# connection = self.pool.get()
connection=self.conn
viz_request.start_time = util.get_current_ms_time()
try:
editedSqlStatement = self.verdictdbedit(sql_statement)
#print(editedSqlStatement)
data = connection.sql(editedSqlStatement)
except Exception as e:
print(e, flush=True)
viz_request.result = {}
viz_request.margins = {}
viz_request.end_time = util.get_current_ms_time()
result_queue.put(viz_request)
return
viz_request.end_time = util.get_current_ms_time()
# put connection back in the queue so the next thread can use it.
#cursor.close()
#connection.close()
#connection=self.create_connection()
#self.pool.put(connection)
results = {}
for i, row in data.iterrows():
keys = []
if row[0] is None:
continue
for i, bin_desc in enumerate(viz_request.viz.binning):
if "width" in bin_desc:
bin_width = bin_desc["width"]
keys.append(str(int(row[0])))
else:
keys.append(str(row[0]).strip())
key = ",".join(keys)
row = list(row)
for i, r in enumerate(row):
if isinstance(r, decimal.Decimal):
row[i] = float(r)
results[key] = row[1]
viz_request.result = results
#viz_request.margins = margins
viz_request.margins = {}
result_queue.put(viz_request)
print("delivering...")
def process_request(self, viz_request, options, schema, result_queue):
# record start time
viz_request.start_time = util.get_current_ms_time()
# print SQL translation of request and simulate query execution
# print(viz_request.viz.get_computed_filter_as_sql(schema))
time.sleep(random.randint(1, 1000)/float(1000))
# record end time
viz_request.end_time = util.get_current_ms_time()
# write an empty result to the viz_request
viz_request.result = {}
# notify IDEBench that processing is done by writing it to the result buffer
result_queue.put(viz_request)
def run(self):
self.vizgraph = VizGraph()
with open(self.get_workflow_path()) as f:
json_data = json.load(f)
for s in json_data["setup"]:
self.vizgraph.add_viz(s)
for s in json_data["setup"]:
self.vizgraph.apply_interaction(Operation(s))
self.workflow_interactions = json_data["interactions"]
self.operation_results = OrderedDict({ "args": vars(self.options), "results": OrderedDict() })
self.current_interaction_index = 0
self.current_vizrequest_index = 0
self.benchmark_start_time = util.get_current_ms_time()
try:
logger.info("calling \"workflow_start\" on driver")
self.driver.workflow_start()
except AttributeError:
pass
global do_poll
do_poll = True
def poll_results(slf, queue):
global count
while do_poll:
try:
process_result = queue.get(timeout=1)
except Empty:
logger.info("result queue empty... trying again")
continue
if process_result is None:
continue
slf.deliver_viz_request([process_result])
logger.info("stopped polling results")
try:
while queue.get(timeout=0.01):
pass
except Empty:
logger.info("result queue cleard")
if not self.options.groundtruth:
thread = Thread(target = poll_results, args = (self, IDEBench.result_queue))
thread.start()
interaction_index = 0
while interaction_index < len(self.workflow_interactions):
self.process_interaction(interaction_index)
interaction_index +=1
do_poll = False
if not self.options.groundtruth:
thread.join()
self.end_run()
def __init__(self, operation_id, parent_operation_id, viz):
self.operation_id = operation_id
self.parent_operation_id = parent_operation_id
self.viz = viz
self.start_time = util.get_current_ms_time()
self.end_time = None
self.result = None
self.margins = None
self.delivered = False
self.bins = None
self.timedout = False
self.t_start = 0
self.t_pause = 0
self.progress = 0
def process_interaction(self, interaction_index):
logger.info("interaction %i" % interaction_index)
interaction = self.workflow_interactions[interaction_index]
next_interaction = self.workflow_interactions[interaction_index + 1] if interaction_index +1 < len(self.workflow_interactions) else None
vizs_to_request = self.vizgraph.apply_interaction(Operation(interaction))
expected_start_time = interaction["time"]
viz_requests = []
for viz in vizs_to_request:
viz_requests.append(VizRequest(self.current_vizrequest_index, self.current_interaction_index, expected_start_time, viz))
self.current_vizrequest_index += 1
# TODO: document this feature
try:
self.driver.before_requests(self.options, self.schema, IDEBench.result_queue)
except AttributeError:
pass
procs = []
nprocs = len(viz_requests)
if self.options.groundtruth or (hasattr(self.driver, "use_single_process") and self.driver.use_single_process):
for viz_request in viz_requests:
op_id = str(viz_request.operation_id)
if op_id in self.gt_for_result["results"] and self.gt_for_result["results"][op_id]["output"]:
self.driver.process_request(viz_request, self.options, self.schema, IDEBench.result_queue)
self.deliver_viz_request([IDEBench.result_queue.get()])
self.counter += 1
else:
for viz_request in viz_requests:
thread = Thread(target = self.driver.process_request, args = (viz_request, self.options, self.schema, IDEBench.result_queue ))
procs.append(thread)
thread.start()
time.sleep(0.002) # so the request threads do not overwhelm some of the drivers (particularly verdictdb)
resultlist = []
delay = 0
think_time = 0
if "time" in interaction and next_interaction:
original_think_time = next_interaction["time"] - interaction["time"]
delay = min(0, next_interaction["time"] - (util.get_current_ms_time() - self.benchmark_start_time))
think_time = max(0, delay + original_think_time)
else:
think_time = self.options.settings_thinktime
if not self.options.groundtruth:
time.sleep(think_time / 1000)
self.current_interaction_index += 1
def process_request(self, viz_request, options, schema, out_q):
print("processsing..." + str(viz_request.operation_id))
if viz_request.viz.binning:
sql_statement = viz_request.viz.get_computed_filter_as_sql(schema)
sql_statement = sql_statement.replace(
schema.get_fact_table_name(), "%s_%s%s" %
(schema.get_fact_table_name(), options.settings_size,
"n" if options.settings_normalized else ""))
if self.can_execute_online(sql_statement):
sql_statement = sql_statement.replace("SELECT ",
"SELECT ONLINE ")
sql_statement += " WITHTIME %s CONFIDENCE 95" % options.settings_time_requirement
sql_statement += " REPORTINTERVAL %s;" % options.settings_time_requirement
connection, cursor = self.create_connection(
options.settings_time_requirement + 20)
else:
connection, cursor = self.create_connection(
options.settings_time_requirement)
viz_request.start_time = util.get_current_ms_time()
try:
cursor.execute(sql_statement)
except psycopg2.extensions.QueryCanceledError as qce:
viz_request.result = {}
viz_request.margins = {}
viz_request.timedout = True
viz_request.end_time = util.get_current_ms_time()
out_q.put(viz_request)
return
data = cursor.fetchall()
viz_request.end_time = util.get_current_ms_time()
connection.close()
results = {}
margins = {}
for row in data:
keys = []
if row[0] is None:
continue
startindex = 3 if self.can_execute_online(sql_statement) else 0
for i, bin_desc in enumerate(viz_request.viz.binning):
if "width" in bin_desc:
bin_width = bin_desc["width"]
keys.append(str(int(row[i + startindex])))
else:
keys.append(str(row[startindex + i]).strip())
key = ",".join(keys)
row = list(row)
for i, r in enumerate(row):
if isinstance(r, decimal.Decimal):
row[i] = float(r)
if startindex == 3:
results[key] = row[len(viz_request.viz.binning) +
startindex:-1]
else:
results[key] = row[len(viz_request.viz.binning) +
startindex:]
if self.can_execute_online(sql_statement) and startindex == 3:
margins[key] = row[len(row) - 1:]
viz_request.result = results
viz_request.margins = margins
out_q.put(viz_request)
print("delivering...")
