class ManifestDataset():
def __init__(self, dbconfig=None, **request):
self.logger = logging.getLogger("manifest_dataset")
logging.basicConfig(
format=
'%(asctime)s.%(msecs)s:%(name)s:%(thread)d:%(levelname)s:%(process)d:%(message)s',
level=logging.INFO)
self.dbreader = DatabaseReader(dbconfig)
self.dbreader.connect()
self.cur_batch = None
self.next_batch = None
self.pre_fetch_thread = None
self.vehicle_id = request['vehicle_id']
self.scene_id = request['scene_id']
self.start_ts = int(request['start_ts'])
self.stop_ts = int(request['stop_ts'])
self.sensor_id = request['sensor_id']
self.step = int(request['step'])
self.fetch()
self.pre_fetch_thread.join()
def is_open(self):
return (self.start_ts <
self.stop_ts) or self.pre_fetch_thread or self.next_batch
def read(self, query=None):
self.next_batch = self.dbreader.query(query)
def fetch(self):
if self.pre_fetch_thread:
self.pre_fetch_thread.join()
self.end_ts = self.start_ts + self.step
if self.end_ts > self.stop_ts:
self.end_ts = self.stop_ts
self.pre_fetch_thread = None
self.cur_batch = self.next_batch
self.next_batch = None
if self.start_ts < self.stop_ts:
query = '''select s3_bucket, s3_key, data_ts from a2d2.drive_data where vehicle_id =
'{0}' and scene_id = '{1}' AND sensor_id = '{2}' AND data_ts >= {3}
AND data_ts < {4} order by data_ts;'''.format(
self.vehicle_id, self.scene_id, self.sensor_id, self.start_ts,
self.end_ts)
t = Thread(target=self.read, kwargs={"query": query})
self.pre_fetch_thread = t
t.start()
self.start_ts = self.end_ts
return self.cur_batch
def should_reject_chat(self, userid, agent_idx, min_tokens):
with self.conn:
controller = self.controller_map[userid]
cursor = self.conn.cursor()
chat_id = controller.get_chat_id()
ex = DatabaseReader.get_chat_example(cursor, chat_id, self.scenario_db).to_dict()
return reject_transcript(ex, agent_idx, min_tokens=min_tokens)
def should_reject_chat(self, userid, agent_idx):
with self.conn:
controller = self.controller_map[userid]
cursor = self.conn.cursor()
chat_id = controller.get_chat_id()
ex = DatabaseReader.get_chat_example(cursor, chat_id, self.scenario_db)
if sum([1 if event.agent == agent_idx and event.action == 'select' else 0 for event in ex.events]) > 3:
return True
return False
def should_reject_chat(self, userid, agent_idx, outcome):
with self.conn:
controller = self.controller_map[userid]
cursor = self.conn.cursor()
chat_id = controller.get_chat_id()
ex = DatabaseReader.get_chat_example(cursor, chat_id,
self.scenario_db).to_dict()
num_tokens = get_total_tokens_per_agent(ex)[agent_idx]
if num_tokens < 30:
return True
return False
def get_most_recent_chat(self, userid):
try:
with self.conn:
controller = self.controller_map[userid]
cursor = self.conn.cursor()
chat_id = controller.get_chat_id()
ex = DatabaseReader.get_chat_example(
cursor, chat_id, self.scenario_db).to_dict()
return ex
except sqlite3.IntegrityError:
print("WARNING: Rolled back transaction")
def __init__(self, dbconfig=None, **request):
self.logger = logging.getLogger("bus_dataset")
logging.basicConfig(
format=
'%(asctime)s.%(msecs)s:%(name)s:%(thread)d:%(levelname)s:%(process)d:%(message)s',
level=logging.INFO)
self.dbreader = DatabaseReader(dbconfig)
self.dbreader.connect()
self.cur_batch = None
self.next_batch = None
self.pre_fetch_thread = None
self.vehicle_id = request['vehicle_id']
self.scene_id = request['scene_id']
self.start_ts = int(request['start_ts'])
self.stop_ts = int(request['stop_ts'])
self.step = int(request['step'])
self.fetch()
self.pre_fetch_thread.join()
def should_reject_chat(self, userid, agent_idx, outcome):
with self.conn:
controller = self.controller_map[userid]
cursor = self.conn.cursor()
chat_id = controller.get_chat_id()
ex = DatabaseReader.get_chat_example(cursor, chat_id,
self.scenario_db).to_dict()
try:
valid = outcome['valid_deal']
except (TypeError, KeyError) as e:
valid = False
num_turns = get_total_turns(ex)
if not valid and num_turns < 4:
return True
return False
def get_margin(self, controller, agent_idx):
with self.conn:
cursor = self.conn.cursor()
chat_id = controller.get_chat_id()
ex = DatabaseReader.get_chat_example(cursor, chat_id,
self.scenario_db)
outcome = controller.get_outcome()
role = ex.scenario.kbs[agent_idx].facts['personal']['Role']
if outcome['reward'] == 0:
return role, None
else:
try:
price = float(outcome['offer']['price'])
except (KeyError, ValueError) as e:
return role, None
margin = StrategyAnalyzer.get_margin(ex,
price,
agent_idx,
role,
remove_outlier=False)
return role, margin