def store_user_data(user, tag_history, tag_graph, start, end, append_to):
user_entity = User.get_by_id(user, namespace=DS_VERSION)
if user_entity is None:
return False
if append_to is not None:
# merge histories
hist_frag = TagHistory.get_by_id(append_to,
parent=user_entity.key,
namespace=DS_VERSION)
hist_frag.tag_history['weeks'] += tag_history['weeks']
hist_frag.end = end
hist_frag.put()
# merge graphs
graph_frag = TagGraph.get_by_id(append_to,
parent=user_entity.key,
namespace=DS_VERSION)
old_graph = graph_frag.tag_graph
for tag in tag_graph:
if tag in old_graph:
old_graph[tag]['plays'] += tag_graph[tag]['plays']
old_graph[tag]['adj'] = old_graph[tag]['adj'].union(
tag_graph[tag]['adj'])
else:
old_graph[tag] = tag_graph[tag]
graph_frag.end = end
graph_frag.put()
# update end timestamp in user fragments chart
frag_info = next(frag_info for frag_info in user_entity.fragments
if frag_info['start'] == hist_frag.start)
frag_info['end'] = end
else:
user_entity.fragments.append({'start': start, 'end': end})
if tag_history['weeks']:
TagHistory(id=user + str(start),
tag_history=tag_history,
start=start,
end=end,
parent=user_entity.key,
namespace=DS_VERSION).put_async()
TagGraph(id=user + str(start),
tag_graph=tag_graph,
start=start,
end=end,
parent=user_entity.key,
namespace=DS_VERSION).put_async()
user_entity.put()
return True
def get_worker_intervals(user_entity, register_date, weeks):
date_floor = register_date
result = []
if user_entity.last_updated is not None:
# get end of incomplete fragment as we must finish it
last_frag = TagHistory.query(ancestor=user_entity.key
, namespace=DS_VERSION).order(
-TagHistory.start).get()
date_floor = user_entity.last_updated
frag_size = get_interval_size(weeks, last_frag.start, last_frag.end)
if frag_size < FRAGMENT_SIZE:
start_i = bisect.bisect_right(weeks, date_floor)
weeks_remaining = FRAGMENT_SIZE - frag_size
end_week = weeks[min(len(weeks)-1, start_i+weeks_remaining-1)]
result.append((weeks[start_i], end_week, last_frag.key.id()))
date_floor = end_week
# get the first week after 'date_floor'
start_i = bisect.bisect_right(weeks, date_floor)
for i in xrange(start_i+FRAGMENT_SIZE-1,
len(weeks), FRAGMENT_SIZE):
result.append((weeks[i-FRAGMENT_SIZE+1], weeks[i], None))
remainder = (len(weeks)-start_i) % FRAGMENT_SIZE
if remainder > 0:
# submit job for last fragment
result.append((weeks[len(weeks)-remainder], weeks[-1], None))
return result
def get_worker_intervals(user_entity, register_date, weeks):
date_floor = register_date
result = []
if user_entity.last_updated is not None:
# get end of incomplete fragment as we must finish it
last_frag = TagHistory.query(ancestor=user_entity.key, namespace=DS_VERSION).order(-TagHistory.start).get()
date_floor = user_entity.last_updated
frag_size = get_interval_size(weeks, last_frag.start, last_frag.end)
if frag_size < FRAGMENT_SIZE:
start_i = bisect.bisect_right(weeks, date_floor)
weeks_remaining = FRAGMENT_SIZE - frag_size
end_week = weeks[min(len(weeks) - 1, start_i + weeks_remaining - 1)]
result.append((weeks[start_i], end_week, last_frag.key.id()))
date_floor = end_week
# get the first week after 'date_floor'
start_i = bisect.bisect_right(weeks, date_floor)
for i in xrange(start_i + FRAGMENT_SIZE - 1, len(weeks), FRAGMENT_SIZE):
result.append((weeks[i - FRAGMENT_SIZE + 1], weeks[i], None))
remainder = (len(weeks) - start_i) % FRAGMENT_SIZE
if remainder > 0:
# submit job for last fragment
result.append((weeks[len(weeks) - remainder], weeks[-1], None))
return result
def post(self):
url = self.request.get('url')
tag = self.request.get('tag')
start_day = self.request.get('start_day')
content = simplejson.loads(urlfetch.fetch(url, deadline=10).content)
try:
refinements = content['response']['refinementGroups'][0]['refinements']
for section in refinements:
t = TagHistory(
name=tag,
related_section=section['displayName'],
data_date=datetime.datetime.strptime(start_day, "%Y-%m-%d"),
content_count=int(section['count']),
)
t.save()
except KeyError:
logging.error("no data for %s" % url)
return self.response.out.write('done')
def build_response(self, user, request):
resp = {'user': user, 'weeks': []}
user_entity = User.get_by_id(user, namespace=DS_VERSION)
qry = TagHistory.query(ancestor=user_entity.key,
namespace=DS_VERSION).order(TagHistory.start)
for hist_frag in qry.fetch():
resp['weeks'] += hist_frag.tag_history['weeks']
return resp
def build_response(self, user, request):
resp = {'user': user, 'weeks':[]}
user_entity = User.get_by_id(user, namespace=DS_VERSION)
qry = TagHistory.query(ancestor=user_entity.key,
namespace=DS_VERSION).order(
TagHistory.start)
for hist_frag in qry.fetch():
resp['weeks'] += hist_frag.tag_history['weeks']
return resp
def store_user_data(user, tag_history, tag_graph, start, end, append_to):
user_entity = User.get_by_id(user, namespace=DS_VERSION)
if user_entity is None:
return False
if append_to is not None:
# merge histories
hist_frag = TagHistory.get_by_id(append_to,
parent=user_entity.key, namespace=DS_VERSION)
hist_frag.tag_history['weeks'] += tag_history['weeks']
hist_frag.end = end
hist_frag.put()
# merge graphs
graph_frag = TagGraph.get_by_id(append_to,
parent=user_entity.key, namespace=DS_VERSION)
old_graph = graph_frag.tag_graph
for tag in tag_graph:
if tag in old_graph:
old_graph[tag]['plays'] += tag_graph[tag]['plays']
old_graph[tag]['adj'] = old_graph[tag]['adj'].union(
tag_graph[tag]['adj'])
else:
old_graph[tag] = tag_graph[tag]
graph_frag.end = end
graph_frag.put()
# update end timestamp in user fragments chart
frag_info = next(frag_info for frag_info in user_entity.fragments
if frag_info['start'] == hist_frag.start)
frag_info['end'] = end
else:
user_entity.fragments.append({'start': start, 'end': end})
if tag_history['weeks']:
TagHistory(id=user+str(start), tag_history=tag_history,
start=start, end=end,
parent=user_entity.key, namespace=DS_VERSION).put_async()
TagGraph(id=user+str(start), tag_graph=tag_graph,
start=start, end=end,
parent=user_entity.key, namespace=DS_VERSION).put_async()
user_entity.put()
return True
def get(self, tag):
tag_history = TagHistory.gql("WHERE name = :tag", tag=tag)
return render(self.response, {'tag_history': tag_history}, 'index.html')