def transition_to_datastore():
"""Transition from file backend to datastore backend.
Checks if the log message has already been transitioned.
"""
client = get_client()
for focus_group in FOCUS_GROUPS:
yield "data: transitioning {} to datastore\n\n".format(focus_group)
in_datastore = datastore.all_content(focus_group, _add_links=False)
for day in files.all_content(focus_group, _add_links=False):
datastore_day = {}
ents = [] # today's entities
for ds_day in in_datastore:
if ds_day["date"] == day["date"]:
datastore_day = ds_day
break
for log in day["logs"]:
if focus_group == "legacy":
timestamp = datetime.strptime(
log["time"].split("M")[0],
"%Y-%m-%d %H:%M:%S",
)
if "capitals" in log["name"]:
focus_group_name = "capitals"
else:
focus_group_name = "tactical-destroyers"
else:
ts = [day["date"].year, day["date"].month, day["date"].day]
ts.extend(int(x) for x in log["time"].split(":"))
timestamp = datetime(*ts)
focus_group_name = focus_group
for datastore_log in datastore_day.get("logs", []):
if datastore_log["user"] == log["user"] and \
datastore_log["message"] == log["message"] and \
datastore_log["time"] == timestamp:
break
else:
ents.append(get_entity(
focus_group_name,
log["user"],
log["message"],
timestamp,
))
client.put_multi(ents)
yield "data: sent {} records to datastore\n\n".format(len(ents))
yield "data: finished moving {} to datastore\n\n".format(focus_group)
def get_entity(channel, speaker, message, time=None):
"""Builds an entity for the google datastore for this message."""
if time is None:
time = datetime.now()
client = get_client()
entity = datastore.Entity(client.key("#{}_focusgroup".format(channel)))
entity["speaker"] = speaker
entity["messsage"] = messsage
entity["time"] = time
return entity
