def store_user_data(user, tag_history, tag_graph, start, end, append_to):
user_entity = User.get_by_id(user, namespace=DS_VERSION)
if user_entity is None:
return False
if append_to is not None:
# merge histories
hist_frag = TagHistory.get_by_id(append_to,
parent=user_entity.key,
namespace=DS_VERSION)
hist_frag.tag_history['weeks'] += tag_history['weeks']
hist_frag.end = end
hist_frag.put()
# merge graphs
graph_frag = TagGraph.get_by_id(append_to,
parent=user_entity.key,
namespace=DS_VERSION)
old_graph = graph_frag.tag_graph
for tag in tag_graph:
if tag in old_graph:
old_graph[tag]['plays'] += tag_graph[tag]['plays']
old_graph[tag]['adj'] = old_graph[tag]['adj'].union(
tag_graph[tag]['adj'])
else:
old_graph[tag] = tag_graph[tag]
graph_frag.end = end
graph_frag.put()
# update end timestamp in user fragments chart
frag_info = next(frag_info for frag_info in user_entity.fragments
if frag_info['start'] == hist_frag.start)
frag_info['end'] = end
else:
user_entity.fragments.append({'start': start, 'end': end})
if tag_history['weeks']:
TagHistory(id=user + str(start),
tag_history=tag_history,
start=start,
end=end,
parent=user_entity.key,
namespace=DS_VERSION).put_async()
TagGraph(id=user + str(start),
tag_graph=tag_graph,
start=start,
end=end,
parent=user_entity.key,
namespace=DS_VERSION).put_async()
user_entity.put()
return True
def build_response(self, user, request):
tag_graph = {}
user_entity = User.get_by_id(user, namespace=DS_VERSION)
start = end = -1
qry = TagGraph.query(ancestor=user_entity.key,
namespace=DS_VERSION).order(TagGraph.start)
# NOTE: 'from'/'to' are not exactly analagous to 'start'/'end'.
# 'from' and 'to' both correspond to fragment 'start' properties
# (as opposed to 'to' corresponsing to an 'end' value).
# Since filter properties must be same as order property,
# we filter only by fragment 'start' dates.
if request.get('from') and request.get('from').isdigit():
qry = qry.filter(TagGraph.start >= int(request.get('from')))
if request.get('to') and request.get('to').isdigit():
qry = qry.filter(TagGraph.start <= int(request.get('to')))
fetch_results = qry.fetch()
for graph_entity in fetch_results:
# merge graph to aggregate graph
sub_graph = graph_entity.tag_graph
for tag in sub_graph:
if tag in tag_graph:
tag_graph[tag]['plays'] += sub_graph[tag]['plays']
tag_graph[tag]['adj'] = tag_graph[tag]['adj'].union(
sub_graph[tag]['adj'])
else:
tag_graph[tag] = sub_graph[tag]
if fetch_results:
start = fetch_results[0].start
end = fetch_results[-1].end
# format JSON
tag_objs = [{
'tag': tag,
'plays': v['plays'],
'adj': list(v['adj'])
} for tag, v in tag_graph.iteritems()]
tag_objs.sort(key=lambda e: e['plays'], reverse=True)
if request.get('tp') and request.get('tp').isdigit():
top_percent = int(request.get('tp')) / 100.0
tag_objs = tag_objs[:int(len(tag_objs) * top_percent)]
top_tags = set([obj['tag'] for obj in tag_objs])
for obj in tag_objs:
adj = [
syn_tag for syn_tag in obj['adj'] if syn_tag in top_tags
]
obj['adj'] = adj
return {'user': user, 'tags': tag_objs, 'start': start, 'end': end}
def build_response(self, user, request):
tag_graph = {}
user_entity = User.get_by_id(user, namespace=DS_VERSION)
start = end = -1
qry = TagGraph.query(ancestor=user_entity.key,
namespace=DS_VERSION).order(
TagGraph.start)
# NOTE: 'from'/'to' are not exactly analagous to 'start'/'end'.
# 'from' and 'to' both correspond to fragment 'start' properties
# (as opposed to 'to' corresponsing to an 'end' value).
# Since filter properties must be same as order property,
# we filter only by fragment 'start' dates.
if request.get('from') and request.get('from').isdigit():
qry = qry.filter(TagGraph.start >= int(request.get('from')))
if request.get('to') and request.get('to').isdigit():
qry = qry.filter(TagGraph.start <= int(request.get('to')))
fetch_results = qry.fetch()
for graph_entity in fetch_results:
# merge graph to aggregate graph
sub_graph = graph_entity.tag_graph
for tag in sub_graph:
if tag in tag_graph:
tag_graph[tag]['plays'] += sub_graph[tag]['plays']
tag_graph[tag]['adj'] = tag_graph[tag]['adj'].union(
sub_graph[tag]['adj'])
else:
tag_graph[tag] = sub_graph[tag]
if fetch_results:
start = fetch_results[0].start
end = fetch_results[-1].end
# format JSON
tag_objs = [{'tag': tag, 'plays': v['plays'], 'adj': list(v['adj'])}
for tag, v in tag_graph.iteritems()]
tag_objs.sort(key=lambda e: e['plays'], reverse=True)
if request.get('tp') and request.get('tp').isdigit():
top_percent = int(request.get('tp')) / 100.0
tag_objs = tag_objs[:int(len(tag_objs) * top_percent)]
top_tags = set([obj['tag'] for obj in tag_objs])
for obj in tag_objs:
adj = [syn_tag for syn_tag in obj['adj']
if syn_tag in top_tags]
obj['adj'] = adj
return {'user': user, 'tags': tag_objs, 'start': start, 'end': end}
def store_user_data(user, tag_history, tag_graph, start, end, append_to):
user_entity = User.get_by_id(user, namespace=DS_VERSION)
if user_entity is None:
return False
if append_to is not None:
# merge histories
hist_frag = TagHistory.get_by_id(append_to,
parent=user_entity.key, namespace=DS_VERSION)
hist_frag.tag_history['weeks'] += tag_history['weeks']
hist_frag.end = end
hist_frag.put()
# merge graphs
graph_frag = TagGraph.get_by_id(append_to,
parent=user_entity.key, namespace=DS_VERSION)
old_graph = graph_frag.tag_graph
for tag in tag_graph:
if tag in old_graph:
old_graph[tag]['plays'] += tag_graph[tag]['plays']
old_graph[tag]['adj'] = old_graph[tag]['adj'].union(
tag_graph[tag]['adj'])
else:
old_graph[tag] = tag_graph[tag]
graph_frag.end = end
graph_frag.put()
# update end timestamp in user fragments chart
frag_info = next(frag_info for frag_info in user_entity.fragments
if frag_info['start'] == hist_frag.start)
frag_info['end'] = end
else:
user_entity.fragments.append({'start': start, 'end': end})
if tag_history['weeks']:
TagHistory(id=user+str(start), tag_history=tag_history,
start=start, end=end,
parent=user_entity.key, namespace=DS_VERSION).put_async()
TagGraph(id=user+str(start), tag_graph=tag_graph,
start=start, end=end,
parent=user_entity.key, namespace=DS_VERSION).put_async()
user_entity.put()
return True
