def test_more(self): request = make_request("/admin/global_dictionary/add_words", "POST", True, 'json={0}'.format(json.dumps(["a{0}".format(i) for i in range(201)]))) response = request.get_response(main.app) task_response = self.run_tasks(1) self.assertEqual(response.status_int, 200) self.assertEqual(task_response[0].status_int, 200) self.assertEqual(GlobalDictionaryWord.query().count(), 201) request = make_request("/admin/global_dictionary/update_json", "POST", True, '0') response = request.get_response(main.app) task_response = self.run_tasks(1) self.assertEqual(response.status_int, 200) self.assertEqual(GlobalDictionaryWord.query().count(), 201) self.assertEqual(GlobalDictionaryJson.query().count(), 1) self.assertEqual(task_response[0].status_int, 200) request = make_request("/api/global_dictionary/get_words/0", "GET", True, '0') response = request.get_response(main.app) server_json = json.loads(response.body) timestamp = server_json["timestamp"] words_time = datetime.fromtimestamp(0) for word in GlobalDictionaryWord.query().fetch(): words_time = max(words_time, word.timestamp) self.assertEqual(time.mktime(words_time.timetuple()) * 1000, timestamp) self.assertEqual(201, len(server_json["words"]))
def test_more(self): request = make_request( "/admin/global_dictionary/add_words", "POST", True, 'json={0}'.format( json.dumps(["a{0}".format(i) for i in range(201)]))) response = request.get_response(main.app) task_response = self.run_tasks(1) self.assertEqual(response.status_int, 200) self.assertEqual(task_response[0].status_int, 200) self.assertEqual(GlobalDictionaryWord.query().count(), 201) request = make_request("/admin/global_dictionary/update_json", "POST", True, '0') response = request.get_response(main.app) task_response = self.run_tasks(1) self.assertEqual(response.status_int, 200) self.assertEqual(GlobalDictionaryWord.query().count(), 201) self.assertEqual(GlobalDictionaryJson.query().count(), 1) self.assertEqual(task_response[0].status_int, 200) request = make_request("/api/global_dictionary/get_words/0", "GET", True, '0') response = request.get_response(main.app) server_json = json.loads(response.body) timestamp = server_json["timestamp"] words_time = datetime.fromtimestamp(0) for word in GlobalDictionaryWord.query().fetch(): words_time = max(words_time, word.timestamp) self.assertEqual(time.mktime(words_time.timetuple()) * 1000, timestamp) self.assertEqual(201, len(server_json["words"]))
def post(self): import numpy, matplotlib, matplotlib.pyplot heatmap_plot = ndb.Key(Plot, "d_plot").get() if heatmap_plot is not None: heatmap_plot.key.delete() words = GlobalDictionaryWord.query(GlobalDictionaryWord.used_times > 0).fetch() matplotlib.pyplot.title("heatmap") x = [] y = [] max_used = 0 for word in words: max_used = max(word.used_times, max_used) x.append(word.used_times) y.append(word.D) max_used = 8 heatmap, xedges, yedges = numpy.histogram2d(y, x, bins=[30, max_used], range=[[0, 30], [0, max_used]]) extent = [0, max_used, 0, 30] matplotlib.pyplot.clf() matplotlib.pyplot.axis(extent) matplotlib.pyplot.imshow(heatmap, extent=extent, aspect="auto", origin="lower") matplotlib.pyplot.title("heatmap for word used times to D") matplotlib.pyplot.xlabel("times used", fontsize=12) matplotlib.pyplot.ylabel("word D", fontsize=12) rv = StringIO.StringIO() matplotlib.pyplot.savefig(rv, format="png", dpi=100) Plot(plot=rv.getvalue(), id="d_plot").put() matplotlib.pyplot.close() rv.close()
def post(self): import numpy, matplotlib, matplotlib.pyplot N = self.request.get("N") heatmap_plot = ndb.Key(Plot, "heatmap_plot_" + N).get() if heatmap_plot is not None: heatmap_plot.key.delete() q = GlobalDictionaryWord.query(GlobalDictionaryWord.used_times > 0).order(-GlobalDictionaryWord.used_times) count = int(q.count() * int(N) / 100) words = q.fetch(count) matplotlib.pyplot.title("heatmap") x = [] y = [] for word in words: x.append(len(word.word)) y.append(int(word.E)) heatmap, xedges, yedges = numpy.histogram2d(y, x, bins=[30, 25], range=[[0, 100], [0, 25]]) extent = [0, 25, 0, 100] matplotlib.pyplot.clf() matplotlib.pyplot.axis([0, 25, 0, 100]) matplotlib.pyplot.imshow(heatmap, vmin=1, extent=extent, aspect="auto", origin="lower") matplotlib.pyplot.title("heatmap for words in top {0} % used times".format(N)) matplotlib.pyplot.xlabel("word length", fontsize=12) matplotlib.pyplot.ylabel("word difficulty", fontsize=12) rv = StringIO.StringIO() matplotlib.pyplot.savefig(rv, format="png", dpi=100) Plot(plot=rv.getvalue(), id="heatmap_plot_" + N).put() matplotlib.pyplot.close() rv.close()
def post(self): self.stage = int(self.request.get('stage', 1)) self.start_cursor = ndb.Cursor(urlsafe=self.request.get('cursor')) queue = taskqueue.Queue('logs-processing') if self.stage == 1: RecalcAllLogs.delete_all_stat() self.next_stage() self.abort(200) elif self.stage == 2: words = self.fetch_portion(GlobalDictionaryWord.query(GlobalDictionaryWord.used_times > 0)) for fut in map(self.reset_word, words): fut.get_result() elif self.stage == 3: logs = self.fetch_portion(GameLog.query()) for el in logs: if not el.ignored: el.ignored = False el.put() elif self.stage == 4: map(lambda k: queue.add_async(taskqueue.Task(url='/internal/add_game_to_statistic', params={'game_key': k.urlsafe()})), self.fetch_portion(GameLog.query(GameLog.ignored == False), keys_only=True)) elif self.stage == 5: map(lambda k: queue.add_async(taskqueue.Task(url='/internal/add_game_to_statistic', params={'game_key': k.urlsafe()})), self.fetch_portion(GameHistory.query(GameHistory.ignored == False), keys_only=True)) if self.more and self.cursor: self.next_portion() else: self.next_stage()
def test_delete(self): url = "/admin/global_dictionary/delete" request = make_request(url, "POST", True, '0') response = request.get_response(main.app) task_response = self.run_tasks(1) self.assertEqual(GlobalDictionaryWord.query().count(), 0) self.assertEqual(GlobalDictionaryJson.query().count(), 0)
def test_delete(self): url = "/admin/global_dictionary/delete" request = make_request(url, "POST", True, '0') response = request.get_response(main.app) task_response = self.run_tasks(1) self.assertEqual(GlobalDictionaryWord.query().count(), 0) self.assertEqual(GlobalDictionaryJson.query().count(), 0)
def post(self): import matplotlib, matplotlib.pyplot N = self.request.get("N") scatter_plot = ndb.Key(Plot, "scatter_plot_" + N).get() if scatter_plot is not None: scatter_plot.key.delete() q = GlobalDictionaryWord.query(GlobalDictionaryWord.used_times > 0).order(-GlobalDictionaryWord.used_times) count = int(q.count() * int(N) / 100) words = q.fetch(count) dict_words = {word.word: word.frequency for word in ndb.gql('SELECT * FROM WordFrequency').fetch()} logging.info('{0} words in freq dictionary'.format(len(dict_words))) x = [] y = [] for word in words: if word.word in dict_words: x.append(dict_words[word.word]) y.append(int(word.E)) fig, ax = matplotlib.pyplot.subplots() ax.set_title("Scatter plot for words in top {0} % used times".format(N), fontsize=14) ax.set_xlabel("uses per million words", fontsize=12) ax.set_ylabel("difficulty", fontsize=12) ax.grid(True, linestyle='-', color='0.75') ax.plot(x, y, 'o', color="green", markersize=2) ax.set_xscale('log') ax.set_ylim([0, 100]) rv = StringIO.StringIO() fig.savefig(rv, format="png", dpi=100) Plot(plot=rv.getvalue(), id="scatter_plot_" + N).put() matplotlib.pyplot.close() rv.close()
def check_word(word): if GlobalDictionaryWord.get(word) is None: on_server = ndb.Key(UnknownWord, word).get() if on_server is None: UnknownWord(word=word, id=word, times_used=1).put() elif not on_server.ignored: on_server.times_used += 1 on_server.put()
def check_word(word): if GlobalDictionaryWord.get(word) is None: on_server = ndb.Key(UnknownWord, word).get() if on_server is None: UnknownWord(word=word, id=word, times_used=1).put() elif not on_server.ignored: on_server.times_used += 1 on_server.put()
def test_delete_from_global_dictionary(self): words = ["a", "b", "c", "d"] for i in words: GlobalDictionaryWord(word=i, id=i, tags="").put() ComplainedWord(word="c").put() ComplainedWord(word="c").put() ComplainedWord(word="d").put() request = make_request("/admin/complain/remove", "POST", True, 'word=c') response = request.get_response(main.app) self.assertEqual(response.status_int, 200) self.assertEqual(ComplainedWord.query().count(), 1) self.assertEqual(GlobalDictionaryWord.query().count(), 4) self.assertEqual(ndb.Key(GlobalDictionaryWord, "c").get().tags, "-deleted") ComplainedWord(word="c").put() response = request.get_response(main.app) self.assertEqual(ComplainedWord.query().count(), 1) self.assertEqual(GlobalDictionaryWord.query().count(), 4) self.assertEqual(ndb.Key(GlobalDictionaryWord, "c").get().tags, "-deleted")
def post(self): timestamp = int(self.request.get("timestamp")) max_timestamp = 0 word_list = [] for word in GlobalDictionaryWord.query().fetch(): word_time = int(time.mktime(word.timestamp.timetuple()) * 1000) if word_time > timestamp: max_timestamp = max(max_timestamp, word_time) word_list.append({"word": word.word, "E": word.E, "D": word.D, "U": word.used_times, "tags": word.tags}) if word_list: GlobalDictionaryJson(json=json.dumps(word_list), timestamp=max_timestamp).put()
def get(self, *args, **kwargs): tab = self.request.get('tab', 'info') data = {} total = TotalStatistics.get() if tab == 'info': data['words_in_dictionary'] = cache( 'dict_word', lambda: GlobalDictionaryWord.query().count()) data['used_words'] = cache( 'used_words', lambda: GlobalDictionaryWord.query( GlobalDictionaryWord.used_times > 0).count()) longest = cache( 'longest_explanation', lambda: GlobalDictionaryWord.query(). order(-GlobalDictionaryWord.total_explanation_time).get()) if longest: data['longest_word'], data[ 'longest_time'] = longest.word, longest.total_explanation_time data['total_words'], data[ 'total_games'] = total.words_used, total.games data['games_for_players'] = cache( 'for_player_count', lambda: GamesForPlayerCount.query().order( GamesForPlayerCount.player_count).fetch()) elif tab == 'daily': data['daily_statistics'] = cache( 'daily', lambda: DailyStatistics.query().order(DailyStatistics. date).fetch()) daily = [] for el in data['daily_statistics']: daily.append((el.games, el.words_used, el.players_participated, el.total_game_duration // 60 * 60, el.date.strftime("%Y-%m-%d"))) by_hour = [0 for i in range(24)] by_day = [0 for i in range(7)] for hour, games in enumerate(total.by_hour): by_hour[hour % 24] += games by_day[(hour // 24 + 3) % 7] += games data['daily'] = daily data['by_hour'] = by_hour data['by_day'] = by_day data['by_hour_and_day'] = total.by_hour self.draw_page("statistics/total_statistic", tab=tab, **data)
def test_delete_from_global_dictionary(self): words = ["a", "b", "c", "d"] for i in words: GlobalDictionaryWord(word=i, id=i, tags="").put() ComplainedWord(word="c").put() ComplainedWord(word="c").put() ComplainedWord(word="d").put() request = make_request("/admin/complain/remove", "POST", True, 'word=c') response = request.get_response(main.app) self.assertEqual(response.status_int, 200) self.assertEqual(ComplainedWord.query().count(), 1) self.assertEqual(GlobalDictionaryWord.query().count(), 4) self.assertEqual( ndb.Key(GlobalDictionaryWord, "c").get().tags, "-deleted") ComplainedWord(word="c").put() response = request.get_response(main.app) self.assertEqual(ComplainedWord.query().count(), 1) self.assertEqual(GlobalDictionaryWord.query().count(), 4) self.assertEqual( ndb.Key(GlobalDictionaryWord, "c").get().tags, "-deleted")
def post(self, *args, **kwargs): words = json.loads(self.request.get("json")) for word in words: g_word = GlobalDictionaryWord.get(word["word"]) if not g_word or g_word.tags.find("-deleted") >= 0: continue current_word = ComplainedWord(device=self.device_key, word=word["word"], reason=word["reason"]) if "replace_word" in word: current_word.replacement_word = \ word["replace_word"] current_word.put()
def post(self, *args, **kwargs): words = json.loads(self.request.get("json")) for word in words: g_word = GlobalDictionaryWord.get(word["word"]) if not g_word or g_word.tags.find("-deleted") >= 0: continue current_word = ComplainedWord(device=self.device_key, word=word["word"], reason=word["reason"]) if "replace_word" in word: current_word.replacement_word = \ word["replace_word"] current_word.put()
def get(self, *args, **kwargs): tab = self.request.get('tab', 'info') data = {} total = TotalStatistics.get() if tab == 'info': data['words_in_dictionary'] = cache('dict_word', lambda: GlobalDictionaryWord.query().count()) data['used_words'] = cache('used_words', lambda: GlobalDictionaryWord.query(GlobalDictionaryWord.used_times > 0).count()) longest = cache('longest_explanation', lambda: GlobalDictionaryWord.query().order( -GlobalDictionaryWord.total_explanation_time).get()) if longest: data['longest_word'], data['longest_time'] = longest.word, longest.total_explanation_time data['total_words'], data['total_games'] = total.words_used, total.games data['games_for_players'] = cache('for_player_count', lambda: GamesForPlayerCount.query().order( GamesForPlayerCount.player_count).fetch()) elif tab == 'daily': data['daily_statistics'] = cache('daily', lambda: DailyStatistics.query().order(DailyStatistics.date).fetch()) daily = [] for el in data['daily_statistics']: daily.append((el.games, el.words_used, el.players_participated, el.total_game_duration // 60 * 60, el.date.strftime("%Y-%m-%d"))) by_hour = [0 for i in range(24)] by_day = [0 for i in range(7)] for hour, games in enumerate(total.by_hour): by_hour[hour % 24] += games by_day[(hour // 24 + 3) % 7] += games data['daily'] = daily data['by_hour'] = by_hour data['by_day'] = by_day data['by_hour_and_day'] = total.by_hour self.draw_page("statistics/total_statistic", tab=tab, **data)
def get(self, *args, **kwargs): word = self.request.get('word', None) entity, games, top, bottom, rand, danger_top = None, None, None, None, None, None if word: entity = GlobalDictionaryWord.get(word) if not entity: danger_top = memcache.get("danger_top") if not danger_top: danger_top = GlobalDictionaryWord.query().order( -GlobalDictionaryWord.danger).fetch(limit=10) memcache.set("danger_top", danger_top, time=60 * 60 * 12) top = memcache.get("words_top") if not top: top = GlobalDictionaryWord.query().order( -GlobalDictionaryWord.E).fetch(limit=10) memcache.set("words_top", top, time=60 * 60 * 12) bottom = memcache.get("words_bottom") if not bottom: bottom = GlobalDictionaryWord.query().order( GlobalDictionaryWord.E).fetch(limit=10) memcache.set("words_bottom", bottom, time=60 * 60 * 12) q = GlobalDictionaryWord.query().filter( GlobalDictionaryWord.used_times > 0) c = memcache.get("used_words_count") if not c: c = q.count() memcache.set("used_words_count", c, time=60 * 60 * 12) if c >= 10: rand = q.fetch(limit=10, offset=randint(0, c - 10)) self.draw_page('statistics/word_statistic', word=word, word_entity=entity, top=top if top else [], bottom=bottom if bottom else [], rand=rand if rand else [], danger=danger_top if danger_top else [])
def get(self, *args, **kwargs): word = self.request.get("word", None) entity, games, top, bottom, rand, danger_top = None, None, None, None, None, None if word: entity = GlobalDictionaryWord.get(word) if not entity: danger_top = memcache.get("danger_top") if not danger_top: danger_top = GlobalDictionaryWord.query().order(-GlobalDictionaryWord.danger).fetch(limit=10) memcache.set("danger_top", danger_top, time=60 * 60 * 12) top = memcache.get("words_top") if not top: top = GlobalDictionaryWord.query().order(-GlobalDictionaryWord.E).fetch(limit=10) memcache.set("words_top", top, time=60 * 60 * 12) bottom = memcache.get("words_bottom") if not bottom: bottom = GlobalDictionaryWord.query().order(GlobalDictionaryWord.E).fetch(limit=10) memcache.set("words_bottom", bottom, time=60 * 60 * 12) q = GlobalDictionaryWord.query().filter(GlobalDictionaryWord.used_times > 0) c = memcache.get("used_words_count") if not c: c = q.count() memcache.set("used_words_count", c, time=60 * 60 * 12) if c >= 10: rand = q.fetch(limit=10, offset=randint(0, c - 10)) self.draw_page( "statistics/word_statistic", word=word, word_entity=entity, top=top if top else [], bottom=bottom if bottom else [], rand=rand if rand else [], danger=danger_top if danger_top else [], )
def post(self): results = {} functions = {} for function in Function.query().fetch(): exec function.code in functions results[function.name] = [] for index, word in enumerate(GlobalDictionaryWord.query().fetch()): for name, result in results: res = functions[name](word) if res is not None: if len(result) <= 50: result.append(Elem(res, word.word)) else: heapq.heappushpop(result, Elem(res, word.word)) for name, result in results: FunctionResult(top=result, id=name).put()
def post(self): results = {} functions = {} for function in Function.query().fetch(): exec function.code in functions results[function.name] = [] for index, word in enumerate(GlobalDictionaryWord.query().fetch()): for name, result in results: res = functions[name](word) if res is not None: if len(result) <= 50: result.append(Elem(res, word.word)) else: heapq.heappushpop(result, Elem(res, word.word)) for name, result in results: FunctionResult(top=result, id=name).put()
def get(self, *args, **kwargs): #TODO : add normal editor template = JINJA_ENVIRONMENT.get_template('templates/global_word_editor.html') curr_json = GlobalDictionaryJson.get_by_key_name('json') render_data = {'words':[]} num = 0 word_cnt = 0 id = int(kwargs.get('id')) if curr_json is not None: for i in json.loads(curr_json.json): word_cnt += 1 if i['tags'].find('-deleted') != -1 or word_cnt <= id * 200 or word_cnt > (id + 1) * 200: continue word = GlobalDictionaryWord(cnt=num, word=i['word'], E=i['E'], D=i['D'], tags=i['tags']) num += 1 render_data['words'].append(word) render_data['quantity'] = num render_data['all_num'] = word_cnt self.response.write(template.render(render_data))
def post(self): results = {} functions = {} names = [] for function in Function.query().fetch(): exec function.code in functions results[function.name] = [] names.append(function.name) for index, word in enumerate(GlobalDictionaryWord.query().fetch()): for function_name in names: res = functions[function_name](word) if res is not None: if len(results[function_name]) <= 50: results[function_name].append(elem(res, word.word)) else: heapq.heappushpop(results[function_name], elem(res, word.word)) for function_name in results: top50 = {i.word: i.res for i in results[function_name]} taskqueue.add(url='/internal/statistics/functions/update/task_queue/push_results', params={'name': function_name, 'top': json.dumps(top50)})
def post(self): for dictionary in Dictionary.query(): data_object = [] key = str(int(time.time())) words = GlobalDictionaryWord.query(GlobalDictionaryWord.deleted == False, GlobalDictionaryWord.lang == dictionary.key.id()).order(GlobalDictionaryWord.E).fetch() chunk_size = len(words) // 100 for i, word in enumerate(words): data_object.append({"word": word.word, "diff": i // chunk_size, "used": word.used_times, "tags": word.tags}) output_file = gcs.open(get_gcs_filename(key), "w", "application/json") json.dump(data_object, output_file) output_file.close() old_key = dictionary.gcs_key dictionary.gcs_key = key dictionary.put() if old_key: gcs.delete(get_gcs_filename(old_key))
def update_word(self, word, word_outcome, explanation_time, rating, game_key): word_db = GlobalDictionaryWord.get(word) if not word_db: return word_db.used_times += 1 if word_outcome == 'guessed': word_db.guessed_times += 1 elif word_outcome == 'failed': word_db.failed_times += 1 time_sec = explanation_time word_db.total_explanation_time += time_sec if word_outcome == 'guessed': pos = time_sec // 5 l = word_db.counts_by_expl_time while pos >= len(l): l.append(0) l[pos] += 1 word_db.used_games.append(game_key.urlsafe()) word_db.E = rating.mu word_db.D = rating.sigma word_db.put()
def post(self): import numpy, matplotlib, matplotlib.pyplot N = self.request.get("N") heatmap_plot = ndb.Key(Plot, "heatmap_plot_" + N).get() if heatmap_plot is not None: heatmap_plot.key.delete() q = GlobalDictionaryWord.query( GlobalDictionaryWord.used_times > 0).order( -GlobalDictionaryWord.used_times) count = int(q.count() * int(N) / 100) words = q.fetch(count) matplotlib.pyplot.title("heatmap") x = [] y = [] for word in words: x.append(len(word.word)) y.append(int(word.E)) heatmap, xedges, yedges = numpy.histogram2d(y, x, bins=[30, 25], range=[[0, 100], [0, 25]]) extent = [0, 25, 0, 100] matplotlib.pyplot.clf() matplotlib.pyplot.axis([0, 25, 0, 100]) matplotlib.pyplot.imshow(heatmap, vmin=1, extent=extent, aspect="auto", origin="lower") matplotlib.pyplot.title( "heatmap for words in top {0} % used times".format(N)) matplotlib.pyplot.xlabel("word length", fontsize=12) matplotlib.pyplot.ylabel("word difficulty", fontsize=12) rv = StringIO.StringIO() matplotlib.pyplot.savefig(rv, format="png", dpi=100) Plot(plot=rv.getvalue(), id="heatmap_plot_" + N).put() matplotlib.pyplot.close() rv.close()
def post(self): self.stage = int(self.request.get('stage', 1)) self.start_cursor = ndb.Cursor(urlsafe=self.request.get('cursor')) queue = taskqueue.Queue('logs-processing') if self.stage == 1: RecalcAllLogs.delete_all_stat() self.next_stage() self.abort(200) elif self.stage == 2: words = self.fetch_portion( GlobalDictionaryWord.query( GlobalDictionaryWord.used_times > 0)) for fut in map(self.reset_word, words): fut.get_result() elif self.stage == 3: logs = self.fetch_portion(GameLog.query()) for el in logs: if not el.ignored: el.ignored = False el.put() elif self.stage == 4: map( lambda k: queue.add_async( taskqueue.Task(url='/internal/add_game_to_statistic', params={'game_key': k.urlsafe()})), self.fetch_portion(GameLog.query(GameLog.ignored == False), keys_only=True)) elif self.stage == 5: map( lambda k: queue.add_async( taskqueue.Task(url='/internal/add_game_to_statistic', params={'game_key': k.urlsafe()})), self.fetch_portion( GameHistory.query(GameHistory.ignored == False), keys_only=True)) if self.more and self.cursor: self.next_portion() else: self.next_stage()
def update_word(self, word, word_outcome, explanation_time, rating, game_key): word_db = GlobalDictionaryWord.get(word) if not word_db: return word_db.used_times += 1 if word_outcome == 'guessed': word_db.guessed_times += 1 elif word_outcome == 'failed': word_db.failed_times += 1 time_sec = explanation_time word_db.total_explanation_time += time_sec if word_outcome == 'guessed': pos = time_sec // 5 l = word_db.counts_by_expl_time while pos >= len(l): l.append(0) l[pos] += 1 word_db.used_games.append(game_key.urlsafe()) word_db.E = rating.mu word_db.D = rating.sigma word_db.put()
def post(self): import matplotlib, matplotlib.pyplot N = self.request.get("N") scatter_plot = ndb.Key(Plot, "scatter_plot_" + N).get() if scatter_plot is not None: scatter_plot.key.delete() q = GlobalDictionaryWord.query( GlobalDictionaryWord.used_times > 0).order( -GlobalDictionaryWord.used_times) count = int(q.count() * int(N) / 100) words = q.fetch(count) dict_words = { word.word: word.frequency for word in ndb.gql('SELECT * FROM WordFrequency').fetch() } logging.info('{0} words in freq dictionary'.format(len(dict_words))) x = [] y = [] for word in words: if word.word in dict_words: x.append(dict_words[word.word]) y.append(int(word.E)) fig, ax = matplotlib.pyplot.subplots() ax.set_title( "Scatter plot for words in top {0} % used times".format(N), fontsize=14) ax.set_xlabel("uses per million words", fontsize=12) ax.set_ylabel("difficulty", fontsize=12) ax.grid(True, linestyle='-', color='0.75') ax.plot(x, y, 'o', color="green", markersize=2) ax.set_xscale('log') ax.set_ylim([0, 100]) rv = StringIO.StringIO() fig.savefig(rv, format="png", dpi=100) Plot(plot=rv.getvalue(), id="scatter_plot_" + N).put() matplotlib.pyplot.close() rv.close()
def post(self): import numpy, matplotlib, matplotlib.pyplot heatmap_plot = ndb.Key(Plot, "d_plot").get() if heatmap_plot is not None: heatmap_plot.key.delete() words = GlobalDictionaryWord.query( GlobalDictionaryWord.used_times > 0).fetch() matplotlib.pyplot.title("heatmap") x = [] y = [] max_used = 0 for word in words: max_used = max(word.used_times, max_used) x.append(word.used_times) y.append(word.D) max_used = 8 heatmap, xedges, yedges = numpy.histogram2d(y, x, bins=[30, max_used], range=[[0, 30], [0, max_used]]) extent = [0, max_used, 0, 30] matplotlib.pyplot.clf() matplotlib.pyplot.axis(extent) matplotlib.pyplot.imshow(heatmap, extent=extent, aspect="auto", origin="lower") matplotlib.pyplot.title("heatmap for word used times to D") matplotlib.pyplot.xlabel("times used", fontsize=12) matplotlib.pyplot.ylabel("word D", fontsize=12) rv = StringIO.StringIO() matplotlib.pyplot.savefig(rv, format="png", dpi=100) Plot(plot=rv.getvalue(), id="d_plot").put() matplotlib.pyplot.close() rv.close()
def post(self): for dictionary in Dictionary.query(): data_object = [] key = str(int(time.time())) words = ( GlobalDictionaryWord.query( GlobalDictionaryWord.deleted == False, GlobalDictionaryWord.lang == dictionary.key.id() ) .order(GlobalDictionaryWord.E) .fetch() ) chunk_size = len(words) // 100 for i, word in enumerate(words): data_object.append( {"word": word.word, "diff": i // chunk_size, "used": word.used_times, "tags": word.tags} ) output_file = gcs.open(get_gcs_filename(key), "w", "application/json") json.dump(data_object, output_file) output_file.close() old_key = dictionary.gcs_key dictionary.gcs_key = key dictionary.put() if old_key: gcs.delete(get_gcs_filename(old_key))
def test_add(self): request = make_request("/internal/global_dictionary/add_words/task_queue", "POST", True, 'json=["a", "b", "c"]') response = request.get_response(main.app) self.assertEqual(response.status_int, 200) self.assertEqual(GlobalDictionaryWord.query().count(), 3) request = make_request("/admin/global_dictionary/add_words", "POST", True, 'json=["a", "b", "d"]') response = request.get_response(main.app) task_response = self.run_tasks(1) must_be = [{"E": 50.0, "U": 0, "word": "a", "tags": ""}, {"E": 50.0, "U": 0, "word": "b", "tags": ""}, {"E": 50.0, "U": 0, "word": "c", "tags": ""}, {"E": 50.0, "U": 0, "word": "d", "tags": ""}] self.assertEqual(task_response[0].status_int, 200) self.assertEqual(response.status_int, 200) self.assertEqual(GlobalDictionaryWord.query().count(), 4) self.assertEqual(GlobalDictionaryJson.query().count(), 0) request = make_request("/admin/global_dictionary/update_json", "POST", True, '0') response = request.get_response(main.app) task_response = self.run_tasks(1) self.assertEqual(response.status_int, 200) self.assertEqual(GlobalDictionaryWord.query().count(), 4) self.assertEqual(GlobalDictionaryJson.query().count(), 1) self.assertEqual(task_response[0].status_int, 200) server_json = json.loads(GlobalDictionaryJson.query().get().json) self.assertEqual(len(must_be), len(server_json)) for i in must_be: ok = False for j in server_json: if i["E"] == j["E"] and i["word"] == j["word"] and i["tags"] == j["tags"]: ok = True break self.assertTrue(ok) request = make_request("/api/global_dictionary/get_words/0", "GET", True, '0') response = request.get_response(main.app) server_json = json.loads(response.body) timestamp = server_json["timestamp"] words_time = datetime.fromtimestamp(0) for word in GlobalDictionaryWord.query().fetch(): words_time = max(words_time, word.timestamp) self.assertEqual(time.mktime(words_time.timetuple()) * 1000, timestamp) self.assertEqual(len(must_be), len(server_json["words"])) for i in must_be: ok = False for j in server_json["words"]: if i["E"] == j["E"] and i["word"] == j["word"] and i["tags"] == j["tags"]: ok = True break self.assertTrue(ok) request = make_request("/api/global_dictionary/get_words/{0}".format(timestamp), "GET", True, '0') response = request.get_response(main.app) self.assertEqual(json.loads(response.body)["words"], []) self.assertEqual(json.loads(response.body)["timestamp"], timestamp) time.sleep(1) request = make_request("/admin/global_dictionary/add_words", "POST", True, 'json=["f", "g", "h"]') request.get_response(main.app) task_response = self.run_tasks(1) self.assertEqual(task_response[0].status_int, 200) request = make_request("/admin/global_dictionary/update_json", "POST", True, '0') request.get_response(main.app) task_response = self.run_tasks(1) self.assertEqual(task_response[0].status_int, 200) self.assertEqual(GlobalDictionaryWord.query().count(), 7) self.assertEqual(GlobalDictionaryJson.query().count(), 2) request = make_request("/api/global_dictionary/get_words/0", "GET", True, '0') response = request.get_response(main.app) self.assertEqual(len(json.loads(response.body)["words"]), 7)
def post(self): game_key = ndb.Key(urlsafe=self.request.get('game_key')) logging.info("Handling log of game {}".format(game_key.id())) if game_key.kind() not in ('GameLog', 'GameHistory'): self.abort(200) log_db = game_key.get() if log_db is None: logging.error("Can't find game log") self.abort(200) is_legacy = game_key.kind() == 'GameHistory' try: words_orig, seen_words_time, words_outcome, explained_at_once, explained_pair, players_count, \ start_timestamp, finish_timestamp = self.parse_history(log_db) if is_legacy else self.parse_log(log_db) if start_timestamp and finish_timestamp: self.update_game_len_prediction( players_count, 'game', finish_timestamp - start_timestamp) bad_words_count = 0 if 2 * len(seen_words_time) < len(words_orig): raise BadGameError('suspect_too_little_words') for k, v in seen_words_time.items(): if v < 2: bad_words_count += 1 if 2 * bad_words_count > len(seen_words_time): raise BadGameError('suspect_too_quick_explanation') for word in words_orig: self.check_word(word) self.word_db = [ GlobalDictionaryWord.get(word) for word in words_orig ] self.ratings = [ TRUESKILL_ENVIRONMENT.create_rating(word.E, word.D) if word else None for word in self.word_db ] self.langs = get_langs() d = defaultdict(list) for word in seen_words_time: if explained_at_once[word] and words_outcome[word] == 'guessed': d[explained_pair[word]].append(word) for l in d.values(): l.sort(key=lambda item: -seen_words_time[item]) self.rate(l) d.clear() for word in seen_words_time: if words_outcome[word] == 'guessed': d[explained_pair[word][0]].append(word) for l in d.values(): l.sort(key=lambda item: -seen_words_time[item]) self.rate(l, coef=0.3) d.clear() for word in seen_words_time: if words_outcome[word] == 'guessed': d[explained_pair[word][1]].append(word) for l in d.values(): l.sort(key=lambda item: -seen_words_time[item]) self.rate(l, coef=0.8) words = [] for word in seen_words_time: if words_outcome[word] == 'guessed': words.append(word) words.sort(key=lambda item: -seen_words_time[item]) self.rate(words, coef=0.6) for i in range(len(words_orig)): if i in seen_words_time: self.update_word(words_orig[i], words_outcome[i], seen_words_time[i], self.ratings[i], game_key) if start_timestamp: start_timestamp //= 1000 log_db.time = datetime.datetime.fromtimestamp(start_timestamp) log_db.put() if finish_timestamp: finish_timestamp //= 1000 duration = finish_timestamp - start_timestamp else: duration = 0 game_date = get_date(start_timestamp) self.update_daily_statistics(game_date, len(seen_words_time), players_count, duration) self.update_total_statistics(len(seen_words_time), start_timestamp) if players_count: self.update_statistics_by_player_count(players_count) memcache.delete_multi([ "danger_top", "words_top", "words_bottom", "used_words_count" ]) except BadGameError as e: if isinstance(e, BadGameError): reason = e.reason else: reason = 'format-error' log_db.ignored = True log_db.reason = reason log_db.put() logging.warning( "Did not handle and marked this game as ignored: {}".format( log_db.reason)) self.abort(200)
def post(self): word = self.request.get("word") GlobalDictionaryWord(word=word, cnt=0, tags="", id=word).put() ignore_word(word)
def post(self): game_key = ndb.Key(urlsafe=self.request.get('game_key')) logging.info("Handling log of game {}".format(game_key.id())) if game_key.kind() not in ('GameLog', 'GameHistory'): self.abort(200) log_db = game_key.get() if log_db is None: logging.error("Can't find game log") self.abort(200) is_legacy = game_key.kind() == 'GameHistory' try: words_orig, seen_words_time, words_outcome, explained_at_once, explained_pair, players_count,\ start_timestamp, finish_timestamp = self.parse_history(log_db) if is_legacy else self.parse_log(log_db) if start_timestamp and finish_timestamp: self.update_game_len_prediction(players_count, 'game', finish_timestamp - start_timestamp) bad_words_count = 0 for k, v in seen_words_time.items(): if v < 2: bad_words_count += 1 if 2*len(seen_words_time) < len(words_orig): raise BadGameError('suspect_too_little_words') if 2*bad_words_count > len(seen_words_time): raise BadGameError('suspect_too_quick_explanation') for word in words_orig: self.check_word(word) word_db = [GlobalDictionaryWord.get(word) for word in words_orig] self.ratings = [TRUESKILL_ENVIRONMENT.create_rating(word.E, word.D) if word else None for word in word_db] d = defaultdict(list) for word in seen_words_time: if explained_at_once[word]: d[explained_pair[word]].append(word) for l in d.values(): l.sort(key=lambda item: (words_outcome[item] != 'failed', -seen_words_time[item])) self.rate(l) d.clear() d = defaultdict(list) for word in seen_words_time: if words_outcome[word] in ('guessed', 'failed'): d[explained_pair[word][0]].append(word) for l in d.values(): l.sort(key=lambda item: (words_outcome[item] != 'failed', -seen_words_time[item])) self.rate(l, coef=0.3) d.clear() for word in seen_words_time: if words_outcome[word] == 'guessed': d[explained_pair[word][1]].append(word) for l in d.values(): l.sort(key=lambda item: -seen_words_time[item]) self.rate(l, coef=0.8) words = [] for word in seen_words_time: if words_outcome[word] in ('guessed', 'failed'): words.append(word) words.sort(key=lambda item: (words_outcome[item] != 'failed', -seen_words_time[item])) self.rate(words, coef=0.6) for i in range(len(words_orig)): if i in seen_words_time: self.update_word(words_orig[i], words_outcome[i], seen_words_time[i], self.ratings[i], game_key) if start_timestamp: start_timestamp //= 1000 if finish_timestamp: finish_timestamp //= 1000 duration = finish_timestamp - start_timestamp else: duration = 0 game_date = get_date(start_timestamp) self.update_daily_statistics(game_date, len(seen_words_time), players_count, duration) self.update_total_statistics(len(seen_words_time), start_timestamp) if players_count: self.update_statistics_by_player_count(players_count) memcache.delete_multi(["danger_top", "words_top", "words_bottom", "used_words_count"]) except BadGameError as e: if isinstance(e, BadGameError): reason = e.reason else: reason = 'format-error' log_db.ignored = True log_db.reason = reason log_db.put() logging.warning("Did not handle and marked this game as ignored: {}".format(log_db.reason)) self.abort(200)
def test_add(self): request = make_request( "/internal/global_dictionary/add_words/task_queue", "POST", True, 'json=["a", "b", "c"]') response = request.get_response(main.app) self.assertEqual(response.status_int, 200) self.assertEqual(GlobalDictionaryWord.query().count(), 3) request = make_request("/admin/global_dictionary/add_words", "POST", True, 'json=["a", "b", "d"]') response = request.get_response(main.app) task_response = self.run_tasks(1) must_be = [{ "E": 50.0, "U": 0, "word": "a", "tags": "" }, { "E": 50.0, "U": 0, "word": "b", "tags": "" }, { "E": 50.0, "U": 0, "word": "c", "tags": "" }, { "E": 50.0, "U": 0, "word": "d", "tags": "" }] self.assertEqual(task_response[0].status_int, 200) self.assertEqual(response.status_int, 200) self.assertEqual(GlobalDictionaryWord.query().count(), 4) self.assertEqual(GlobalDictionaryJson.query().count(), 0) request = make_request("/admin/global_dictionary/update_json", "POST", True, '0') response = request.get_response(main.app) task_response = self.run_tasks(1) self.assertEqual(response.status_int, 200) self.assertEqual(GlobalDictionaryWord.query().count(), 4) self.assertEqual(GlobalDictionaryJson.query().count(), 1) self.assertEqual(task_response[0].status_int, 200) server_json = json.loads(GlobalDictionaryJson.query().get().json) self.assertEqual(len(must_be), len(server_json)) for i in must_be: ok = False for j in server_json: if i["E"] == j["E"] and i["word"] == j["word"] and i[ "tags"] == j["tags"]: ok = True break self.assertTrue(ok) request = make_request("/api/global_dictionary/get_words/0", "GET", True, '0') response = request.get_response(main.app) server_json = json.loads(response.body) timestamp = server_json["timestamp"] words_time = datetime.fromtimestamp(0) for word in GlobalDictionaryWord.query().fetch(): words_time = max(words_time, word.timestamp) self.assertEqual(time.mktime(words_time.timetuple()) * 1000, timestamp) self.assertEqual(len(must_be), len(server_json["words"])) for i in must_be: ok = False for j in server_json["words"]: if i["E"] == j["E"] and i["word"] == j["word"] and i[ "tags"] == j["tags"]: ok = True break self.assertTrue(ok) request = make_request( "/api/global_dictionary/get_words/{0}".format(timestamp), "GET", True, '0') response = request.get_response(main.app) self.assertEqual(json.loads(response.body)["words"], []) self.assertEqual(json.loads(response.body)["timestamp"], timestamp) time.sleep(1) request = make_request("/admin/global_dictionary/add_words", "POST", True, 'json=["f", "g", "h"]') request.get_response(main.app) task_response = self.run_tasks(1) self.assertEqual(task_response[0].status_int, 200) request = make_request("/admin/global_dictionary/update_json", "POST", True, '0') request.get_response(main.app) task_response = self.run_tasks(1) self.assertEqual(task_response[0].status_int, 200) self.assertEqual(GlobalDictionaryWord.query().count(), 7) self.assertEqual(GlobalDictionaryJson.query().count(), 2) request = make_request("/api/global_dictionary/get_words/0", "GET", True, '0') response = request.get_response(main.app) self.assertEqual(len(json.loads(response.body)["words"]), 7)
def post(self): new_words = json.loads(self.request.get("json")) for word in new_words: GlobalDictionaryWord(id=word, word=word, E=50.0, D=50.0/3, tags="").put()
def put_word(word): return GlobalDictionaryWord(id=word, word=word).put()
def post(self, *args, **kwargs): word_to_delete = self.request.get("word") entity = GlobalDictionaryWord.get_by_id(word_to_delete) if entity is not None and entity.tags.find("-deleted") == -1: entity.tags += "-deleted" entity.put()