def make_update_list(pb_file_list, folder_path):
update_list = []
bad_update_header_list = []
dirname = folder_path
for pb_file in pb_file_list:
pb_file_path = os.path.join(dirname, pb_file)
with open(pb_file_path, 'rb') as f:
feed = gtfs_realtime_pb2.FeedMessage()
feed.ParseFromString(f.read())
dict_obj = MessageToDict(feed)
if 'entity' in dict_obj.keys():
for update_idx in range(len(dict_obj['entity'])):
update_dict = {}
j_in = json.dumps(dict_obj['entity'][update_idx])
j_out = json.loads(j_in)
if 'tripUpdate' in j_out.keys():
update_dict['delay'] = j_out['tripUpdate']['delay']
update_dict['stop_update_departure'] = j_out['tripUpdate'][
'stopTimeUpdate'][0]['departure']['time']
update_dict['stop_id'] = j_out['tripUpdate'][
'stopTimeUpdate'][0]['stopId']
update_dict['timestamp'] = j_out['tripUpdate']['timestamp']
update_dict['route_id'] = j_out['tripUpdate']['trip'][
'routeId']
update_dict['trip_id'] = j_out['tripUpdate']['trip'][
'tripId']
update_dict['vehicle_id'] = j_out['tripUpdate']['vehicle'][
'id']
update_list.append(update_dict)
else:
bad_update_header_list.append(dict_obj['header'])
else:
bad_update_header_list.append(dict_obj['header'])
return update_list, bad_update_header_list
async def play(self, ws, observation):
success = True
request_data = api.Request(
data=api.RequestData(ability_id=True, unit_type_id=True, upgrade_id=True)
)
await asyncio.wait_for(ws.send(request_data.SerializeToString()), 5)
try:
result = await asyncio.wait_for(ws.recv(), 5)
data_response = api.Response.FromString(result)
game_data = data_response.data
request_game_info = api.Request(game_info=api.RequestGameInfo())
await asyncio.wait_for(ws.send(request_game_info.SerializeToString()), 5)
result = await asyncio.wait_for(ws.recv(), 5)
game_info_response = api.Response.FromString(result)
# If game is still on
if game_data.units:
obj = decode_observation(observation.observation.observation, game_data, game_info_response)
obs = MessageToDict(observation)
obs = str(obs)
obs = obs.replace("\'", "\"")
obs = obs.replace("False", "false")
obs = obs.replace("True", "true")
obs = json.loads(obs,encoding="UTF-8")
game_meta = api.Request(
game_info=api.RequestGameInfo()
)
await ws.send(game_meta.SerializeToString())
result = await ws.recv()
game_meta = api.Response.FromString(result)
game_meta = MessageToDict(game_meta)
game_meta = str(game_meta)
game_meta = game_meta.replace("\'", "\"")
game_meta = game_meta.replace("False", "false")
game_meta = game_meta.replace("True", "true")
game_meta = json.loads(game_meta,encoding="UTF-8")
game_meta = game_meta.get("gameInfo", None)
game_meta.pop("modNames")
game_meta.pop("options")
game_meta.pop("mapName")
if("localMapPath" in game_meta.keys()):
game_meta.pop("localMapPath")
game_meta.pop("playerInfo")
game_meta.update(game_meta["startRaw"])
game_meta.pop("startRaw")
game_meta.pop("mapSize")
await self.process_step(ws, obj, raw=(obs, game_meta, game_data))
# function = self.decision_function
# alvailable_actions = self.query_alvailable_actions()
# to_do_action = function(observation, alvailable_actions)
# while(to_do_action and alvailable_actions):
# self.send_order(self, to_do_action)
# to_do_action = self.query_alvailable_actions()
except asyncio.TimeoutError:
return False
return True
def normalize_list_from_config(config):
""" Construct list of attrs to normalize
from protobuf config
"""
sarsa_as_dict = MessageToDict(config)
attr_mapping = Sarsa.proto_name_to_attr_dict()
return [
v for k, v in attr_mapping.items()
if k not in sarsa_as_dict.keys()
]
def asset_edit(asset_id=None):
form = AssetForm()
if form.validate_on_submit():
return asset_submit(form, asset_id)
else:
logging.info('loading current values because: {}'.format(form.errors))
old_asset = assets.GetAsset(a.GetAssetRequest(_id=asset_id))
# All of this fuckery is needed because the proto object validates field types,
# so we can't just change the field to a datetime object but need a new object
asset_dict = MessageToDict(message=old_asset, preserving_proto_field_name=True)
asset_dict['acquired'] = date.fromtimestamp(old_asset.acquired)
# Have to delete _id since it's not a valid field for a namedtuple
del asset_dict['_id']
asset_obj = namedtuple("Asset", asset_dict.keys()) (*asset_dict.values())
form = AssetForm(obj=asset_obj)
return render_template('asset_edit.html', form=form, view='Edit Asset')
def get_sketch(request: schemas.ColorInput):
knn = request.max_results
color_list = []
c0 = closest_color([request.c0.red, request.c0.green, request.c0.blue])
c1 = closest_color([request.c1.red, request.c1.green, request.c1.blue])
c2 = closest_color([request.c2.red, request.c2.green, request.c2.blue])
c3 = closest_color([request.c3.red, request.c3.green, request.c3.blue])
c4 = closest_color([request.c4.red, request.c4.green, request.c4.blue])
c5 = closest_color([request.c5.red, request.c5.green, request.c5.blue])
c6 = closest_color([request.c6.red, request.c6.green, request.c6.blue])
c7 = closest_color([request.c7.red, request.c7.green, request.c7.blue])
c8 = closest_color([request.c8.red, request.c8.green, request.c8.blue])
c9 = closest_color([request.c9.red, request.c9.green, request.c9.blue])
c10 = closest_color([request.c10.red, request.c10.green, request.c10.blue])
c11 = closest_color([request.c11.red, request.c11.green, request.c11.blue])
color_list.extend([c0, c1, c2, c3, c4, c5, c6, c7, c8, c9, c10, c11])
color_query = [item for t in color_list for item in t]
with CottontailDBClient('localhost', 1865) as client:
result = client.knn(
color_query, "tal_db", "color_image", "dominant_color_vector",
["video_id", "keyframe_id", "start_time", "distance"], knn)
result = MessageToDict(list(result)[0])
response = {}
columns = result["columns"]
if 'tuples' in result.keys():
results = result["tuples"]
else:
return {"results": []}
for i, tuple in enumerate(results):
response[f"{i}"] = dict()
response[f"{i}"][columns[0]
["name"]] = tuple["data"][0]["doubleData"]
response[f"{i}"][columns[1]
["name"]] = tuple["data"][1]["stringData"]
response[f"{i}"][columns[2]["name"]] = tuple["data"][2]["intData"]
response[f"{i}"][columns[3]
["name"]] = tuple["data"][3]["floatData"]
return {"results": list(response.values())}
def read_tfrecords(tfrecords):
tfrecords_bytes = io.BytesIO(tfrecords)
examples = []
while True:
length_header = 12
buf = tfrecords_bytes.read(length_header)
if not buf:
# reached end of tfrecord buffer, return examples
return examples
if len(buf) != length_header:
raise ValueError("TFrecord is fewer than %d bytes" % length_header)
length, length_mask = struct.unpack("<QI", buf)
length_mask_actual = _masked_crc32c(buf[:8])
if length_mask_actual != length_mask:
raise ValueError("TFRecord does not contain a valid length mask")
length_data = length + 4
buf = tfrecords_bytes.read(length_data)
if len(buf) != length_data:
raise ValueError(
"TFRecord data payload has fewer bytes than specified in header"
)
data, data_mask_expected = struct.unpack("<%dsI" % length, buf)
data_mask_actual = _masked_crc32c(data)
if data_mask_actual != data_mask_expected:
raise ValueError("TFRecord has an invalid data crc32c")
# Deserialize the tf.Example proto
example = tf.train.Example()
example.ParseFromString(data)
# Extract a feature map from the example object
example_feature = MessageToDict(example.features)["feature"]
feature_dict = {}
for feature_key in example_feature.keys():
feature_dict[feature_key] = example_feature[feature_key][list(
example_feature[feature_key].keys())[0]]["value"][0]
examples.append(feature_dict)
def read_tfrecords(tfrecords):
tfrecords_bytes = io.BytesIO(tfrecords)
examples = []
while True:
length_header = 12
buf = tfrecords_bytes.read(length_header)
if not buf:
# reached end of tfrecord buffer, return examples
return examples
if len(buf) != length_header:
raise ValueError('TFrecord is fewer than %d bytes' % length_header)
length, length_mask = struct.unpack('<QI', buf)
length_mask_actual = _masked_crc32c(buf[:8])
if length_mask_actual != length_mask:
raise ValueError('TFRecord does not contain a valid length mask')
length_data = length + 4
buf = tfrecords_bytes.read(length_data)
if len(buf) != length_data:
raise ValueError('TFRecord data payload has fewer bytes than specified in header')
data, data_mask_expected = struct.unpack('<%dsI' % length, buf)
data_mask_actual = _masked_crc32c(data)
if data_mask_actual != data_mask_expected:
raise ValueError('TFRecord has an invalid data crc32c')
# Deserialize the tf.Example proto
example = tf.train.Example()
example.ParseFromString(data)
# Extract a feature map from the example object
example_feature = MessageToDict(example.features)['feature']
feature_dict = {}
for feature_key in example_feature.keys():
feature_dict[feature_key] = example_feature[feature_key][list(example_feature[feature_key].keys())[0]]['value'][0]
examples.append(feature_dict)
def make_vehicle_list(pb_file_list, folder_path):
vehicle_list = []
bad_vehicle_header_list = []
dirname = folder_path
for pb_file in pb_file_list:
pb_file_path = os.path.join(dirname, pb_file)
with open(pb_file_path, 'rb') as f:
feed = gtfs_realtime_pb2.FeedMessage()
feed.ParseFromString(f.read())
dict_obj = MessageToDict(feed)
if 'entity' in dict_obj.keys():
for vehicles_idx in range(len(dict_obj['entity'])):
vehicle_dict = {}
j_in = json.dumps(dict_obj['entity'][vehicles_idx])
j_out = json.loads(j_in)
if 'position' in j_out['vehicle'] and 'trip' in j_out[
'vehicle']:
vehicle_dict['vehicle_id'] = j_out['vehicle']['vehicle'][
'id']
vehicle_dict['timestamp'] = j_out['vehicle']['timestamp']
vehicle_dict['trip_id'] = j_out['vehicle']['trip'][
'tripId']
vehicle_dict['route_id'] = j_out['vehicle']['trip'][
'routeId']
vehicle_dict['vehicle_lat'] = j_out['vehicle']['position'][
'latitude']
vehicle_dict['vehicle_long'] = j_out['vehicle'][
'position']['longitude']
#trip_id = j_out['vehicle']['trip']['tripId']
#route_id = j_out['vehicle']['trip']['routeId']
#vehicle_dict['shape_id'] = get_shape_id_from_triproute(trip_id, route_id, schedule_df)
vehicle_list.append(vehicle_dict)
else:
bad_vehicle_header_list.append(dict_obj['header'])
else:
bad_vehicle_header_list.append(dict_obj['header'])
return vehicle_list, bad_vehicle_header_list
def user_edit(user_id=None):
form = UserForm()
form.submit.label.text = 'Update'
if form.validate_on_submit():
# check that old_password matches the current user's
if bcrypt.check_password_hash(current_user.user.password, form.old_password.data):
return user_submit(form, user_id)
else:
flash('Current password incorrect!')
else:
logging.info('loading current values because: {}'.format(form.errors))
old_user = users.GetUser(u.GetUserRequest(_id=user_id))
# All of this fuckery is needed because the proto object validates field types,
# so we can't just change the field to a datetime object but need a new object
user_dict = MessageToDict(message=old_user, preserving_proto_field_name=True)
# Have to delete _id since it's not a valid field for a namedtuple
del user_dict['_id']
user_obj = namedtuple("User", user_dict.keys()) (*user_dict.values())
form = UserForm(obj=user_obj)
return render_template('user_edit.html', form=form, view='Edit User')
def get_sketch(request: schemas.ObjectSketchInput):
knn = request.max_results
object_query = request.object
# (x1,y1) is lower left and (x2,y2) is upper right
sketch_query = [
request.sketch.x1, request.sketch.y1, request.sketch.x2,
request.sketch.y2
] # list of 4 elements
with CottontailDBClient('localhost', 1865) as client:
result = client.knn_where(
sketch_query, "tal_db", "sketch", "sketch_vector", "object",
["video_id", "keyframe_id", "distance", "start_time", "object"],
[object_query], knn)
result = MessageToDict(list(result)[0])
response = {}
columns = result["columns"]
if 'tuples' in result.keys():
results = result["tuples"]
else:
return {"results": []}
for i, tuple in enumerate(results):
response[f"{i}"] = dict()
response[f"{i}"][columns[0]
["name"]] = tuple["data"][0]["stringData"]
response[f"{i}"][columns[1]
["name"]] = tuple["data"][1]["doubleData"]
response[f"{i}"][columns[2]
["name"]] = tuple["data"][2]["stringData"]
response[f"{i}"][columns[3]["name"]] = tuple["data"][3]["intData"]
response[f"{i}"][columns[4]
["name"]] = tuple["data"][4]["floatData"]
return {"results": list(response.values())}
def get_gas_price(url):
keywords = [
'regular', 'reg', 'plus', 'premium', 'pre', 'prem', 'unleaded',
'super', 'v-power', 'special', 'super+', 'unlead', 'noethnl', 'diesel',
'extra', 'midgrade', 'mid-grade', 'mid', 'silver', 'ultimate',
'unleaded plus', 'unleaded premium'
]
spell = SpellChecker()
spell.word_frequency.load_words(keywords)
client = vision.ImageAnnotatorClient()
image = vision.types.Image()
image.source.image_uri = url
texts = MessageToDict(client.text_detection(image=image))
while "error" in texts.keys():
texts = MessageToDict(client.text_detection(image=image))
sleep(5)
if len(texts.keys()) == 0:
return {"ERROR": "No text found"}
texts["textAnnotations"].pop(0)
strings = [{
"text":
x["description"],
"bounds": [(y["x"], y["y"]) if "y" in y.keys() else (y["x"], 0)
for y in x["boundingPoly"]["vertices"]]
} for x in texts["textAnnotations"]]
# Get number prices
numbers = []
for i in range(len(strings)):
if strings[i]["text"].replace(".", "").replace(" ", "").replace(
"$", "")[:3].isnumeric():
string = strings[i]["text"].replace(".",
"").replace(" ", "").replace(
"$", "")[:3]
if len(string) >= 3:
strings[i]["text"] = "$" + string[0] + "." + string[1:]
numbers.append(strings[i])
# Get adaptive keyword list
keywords = adaptive_keywords(keywords, strings)
# Get keyword
keyword_indexes = [
i for i in range(len(strings))
if strings[i]["text"].lower() in keywords
]
if len(keyword_indexes) == 0:
return {str(i): numbers[i]["text"] for i in range(len(numbers))}
keyword = strings[keyword_indexes[0]]
keyword_midpoint = get_midpoint(keyword["bounds"])
# Get vector
vector = (10000, 10000)
for i in range(len(numbers)):
number_midpoint = get_midpoint(numbers[i]["bounds"])
if math.sqrt((keyword_midpoint[0] - number_midpoint[0])**2 +
(keyword_midpoint[1] - number_midpoint[1])**2
) < math.sqrt(vector[0]**2 + vector[1]**2):
vector = (keyword_midpoint[0] - number_midpoint[0],
keyword_midpoint[1] - number_midpoint[1])
# Get gas type and price association
output = {}
for i in range(len(numbers)):
number_midpoint = get_midpoint(numbers[i]["bounds"])
gas_type = get_text_element(
strings,
(vector[0] + number_midpoint[0], vector[1] + number_midpoint[1]))
if gas_type:
output[spell.correction(
gas_type["text"]).capitalize()] = numbers[i]["text"]
else:
# Attempt to cast a wider net around the vector
output[str(i)] = numbers[i]["text"]
for n in range(-50, 51, 10):
gas_type = get_text_element(
strings, (vector[0] + number_midpoint[0] + n,
vector[1] + number_midpoint[1]))
if gas_type and gas_type["text"].lower() in keywords:
output.pop(str(i))
output[spell.correction(
gas_type["text"]).capitalize()] = numbers[i]["text"]
break
gas_type = get_text_element(
strings, (vector[0] + number_midpoint[0],
vector[1] + number_midpoint[1] + n))
if gas_type and gas_type["text"].lower() in keywords:
output.pop(str(i))
output[spell.correction(
gas_type["text"]).capitalize()] = numbers[i]["text"]
break
for k in output.keys():
if k.isnumeric():
output["Unknown " + str(k)] = output.pop(k)
return output
def get_text(text: str):
initial_text_list = []
text_list = []
initial_text_list = list(text.split(" "))
for element in initial_text_list:
element = stemming_algo(element)
text_list.append(element)
if len(text_list) == 1:
with CottontailDBClient('localhost', 1865) as client:
result = client.select_where("tal_db", "transcription",
["video_id", "audio_transcription"],
"audio_transcription",
[f"%{text[0]}%"])
result = MessageToDict(list(result)[0])
response = {}
columns = result["columns"]
if 'tuples' in result.keys():
results = result["tuples"]
print(results)
else:
return {"results": []}
for i, tuple in enumerate(results):
response[f"{i}"] = dict()
response[f"{i}"][columns[0]
["name"]] = tuple["data"][0]["stringData"]
response[f"{i}"][columns[1]
["name"]] = tuple["data"][1]["stringData"]
return {"results": list(response.values())}
elif len(text_list) == 2:
with CottontailDBClient('localhost', 1865) as client:
result1_text = client.select_where(
"tal_db", "transcription", ["video_id", "audio_transcription"],
"audio_transcription", [f"%{text_list[0]}%"])
df_text1 = cottontail_simple_text_where_to_df(
result1_text, "audio_transcription")
print(df_text1)
result2_text = client.select_where(
"tal_db", "transcription", ["video_id", "audio_transcription"],
"audio_transcription", [f"%{text_list[1]}%"])
df_text2 = cottontail_simple_text_where_to_df(
result2_text, "audio_transcription")
print(df_text2)
merged_df = pd.merge(df_text1, df_text2, on=['video_id'])
print(merged_df)
merged_df = merged_df.drop(
['audio_transcription_x', 'audio_transcription'],
axis=1).sort_values(by=['video_id'])
response = merged_df.head(20000000).to_dict(orient="records")
print(response)
return {"results": response}
elif len(text_list) == 3:
with CottontailDBClient('localhost', 1865) as client:
result1_text = client.select_where(
"tal_db", "transcription", ["video_id", "audio_transcription"],
"audio_transcription", [f"%{text_list[0]}%"])
df_text1 = cottontail_simple_text_where_to_df(
result1_text, "audio_transcription")
print(df_text1)
result2_text = client.select_where(
"tal_db", "transcription", ["video_id", "audio_transcription"],
"audio_transcription", [f"%{text_list[1]}%"])
df_text2 = cottontail_simple_text_where_to_df(
result2_text, "audio_transcription")
print(df_text2)
result3_text = client.select_where(
"tal_db", "transcription", ["video_id", "audio_transcription"],
"audio_transcription", [f"%{text_list[2]}%"])
df_text3 = cottontail_simple_text_where_to_df(
result3_text, "audio_transcription")
print(df_text3)
merged_df = df_text1.merge(df_text2,
on=['video_id']).merge(df_text3,
on=['video_id'])
print(merged_df)
merged_df = merged_df.drop(
['audio_transcription_x', 'audio_transcription_y'],
axis=1).sort_values(by=['video_id'])
response = merged_df.head(20000000).to_dict(orient="records")
print(response)
return {"results": response}
elif len(text_list) > 3:
with CottontailDBClient('localhost', 1865) as client:
result = client.select_where("tal_db", "transcription",
["video_id", "audio_transcription"],
"audio_transcription", [f"%{text}%"])
result = MessageToDict(list(result)[0])
response = {}
columns = result["columns"]
if 'tuples' in result.keys():
results = result["tuples"]
print(results)
else:
return {"results": []}
for i, tuple in enumerate(results):
response[f"{i}"] = dict()
response[f"{i}"][columns[0]
["name"]] = tuple["data"][0]["stringData"]
response[f"{i}"][columns[1]
["name"]] = tuple["data"][1]["stringData"]
return {"results": list(response.values())}
async def load_replay(self, replay_file, id=0):
print(replay_file)
async with websockets.connect("ws://{0}:{1}/sc2api".format(
self.host.address, self.host.port)) as ws:
replay_meta = api.Request(replay_info=api.RequestReplayInfo(
replay_path=replay_file))
await ws.send(replay_meta.SerializeToString())
result = await ws.recv()
metadata = api.Response.FromString(result)
self.replay_info = {
"map":
metadata.replay_info.map_name,
"races": [
metadata.replay_info.player_info[0].player_info.
race_requested,
metadata.replay_info.player_info[1].player_info.
race_requested,
],
"results": [
metadata.replay_info.player_info[0].player_result.result,
metadata.replay_info.player_info[1].player_result.result,
],
}
print(self.replay_info)
msg = api.Request(start_replay=api.RequestStartReplay(
replay_path=replay_file,
observed_player_id=id,
options=api.InterfaceOptions(raw=True, score=False),
))
await ws.send(msg.SerializeToString())
time.sleep(1)
result = await ws.recv()
response = api.Response.FromString(result)
print(response)
game_meta = api.Request(game_info=api.RequestGameInfo())
await ws.send(game_meta.SerializeToString())
result = await ws.recv()
game_meta = api.Response.FromString(result)
game_meta = MessageToDict(game_meta)
game_meta = str(game_meta)
game_meta = game_meta.replace("\'", "\"")
game_meta = game_meta.replace("False", "false")
game_meta = game_meta.replace("True", "true")
game_meta = json.loads(game_meta, encoding="UTF-8")
if "gameInfo" in game_meta.keys():
game_meta = game_meta.get("gameInfo", None)
game_meta.pop("modNames")
game_meta.pop("options")
game_meta.pop("mapName")
if ("localMapPath" in game_meta.keys()):
game_meta.pop("localMapPath")
game_meta.pop("playerInfo")
game_meta.update(game_meta["startRaw"])
game_meta.pop("startRaw")
game_meta.pop("mapSize")
self.game_info = game_meta
self.status = "started"
return True
else:
return False
