def search_notification(param, size): try: rs = requests.session() query_json = {'query': {'match_all': {}}} must = [] keyword_fields = [ 'user_id', 'sender_id', 'notification_type', 'status' ] for f in param: if f in keyword_fields: must.append({'term': {f: param[f]}}) else: must.append({'match': {f: param[f]}}) if len(must) > 0: query_json = {'query': {'bool': {'must': must}}} query_json['size'] = size query_json['sort'] = [{'created_at': {'order': 'desc'}}] search_url = 'http://{}/{}/{}/_search'.format( app.config['ES_HOST'], _es_user_user_notification, _es_type) response = rs.post(url=search_url, json=query_json, headers=_http_headers).json() item_list = [] if 'hits' in response: for hit in response['hits']['hits']: data = hit['_source'] data['id'] = hit['_id'] user_details = get_user_details(data['user_id']) data['user_id_handle'] = user_details['username'] if data['sender_id'] != 'System': user_details = get_user_details(data['sender_id']) data['sender_id_handle'] = user_details['username'] else: data['sender_id_handle'] = 'System' if 'notification_text' in data: data['notification_text'] = data[ 'notification_text'] + ' ' + data['sender_id_handle'] if 'created_at' in data: data['created_at'] = time.strftime( '%Y-%m-%d %H:%M:%S', time.localtime(data['created_at'])) if 'status' in data and data['status'] == UNREAD: data[UNREAD] = True item_list.append(data) return item_list except Exception as e: raise e
def get_team_details(team_id): try: rs = requests.session() search_url = 'http://{}/{}/{}/{}'.format(app.config['ES_HOST'], _es_index_team, _es_type, team_id) response = rs.get(url=search_url, headers=_http_headers).json() if 'found' in response: if response['found']: data = response['_source'] data['id'] = response['_id'] data['member_list'] = get_all_users_from_team(team_id) data['rating_history'] = get_team_rating_history(team_id) data['task_list'] = search_task_lists( {'classroom_id': team_id}, 0, 3) data['class_list'] = search_class_lists( {'classroom_id': team_id}, 0, 3) user_details = get_user_details(data['team_leader_id']) data['team_leader_handle'] = user_details['username'] data['team_leader_skill_color'] = user_details['skill_color'] data['follow_stat'] = get_follow_stat(data['id']) data['last_synced_time'] = last_completed_job_time(team_id) data = reformat_team_data(data) return data raise Exception('Team not found') app.logger.error('Elasticsearch down, response: ' + str(response)) raise Exception('Internal server error') except Exception as e: raise e
def get_category_toppers(category_id): try: rs = requests.session() must = [{'term': {'category_id': category_id}}] query_json = {'query': {'bool': {'must': must}}} query_json['size'] = _es_size query_json['sort'] = [{'skill_value': {'order': 'desc'}}] search_url = 'http://{}/{}/{}/_search'.format(app.config['ES_HOST'], _es_index_user_category, _es_type) response = rs.post(url=search_url, json=query_json, headers=_http_headers).json() if 'hits' in response: item_list = [] rank = 1 for hit in response['hits']['hits']: edge = hit['_source'] edge['id'] = hit['_id'] try: user_details = get_user_details(edge['user_id']) except Exception as e: app.logger.error(f'User not found: {str(e)}') continue edge['user_handle'] = user_details['username'] edge['user_skill_color'] = user_details['skill_color'] edge['skill_value'] = float("{:.2f}".format( edge.get('skill_value', 0))) edge['rank'] = rank rank += 1 item_list.append(edge) return item_list except Exception as e: raise e
def dfs_comment_tree(cur_node_id): try: rs = requests.session() must = [{'term': {'comment_parent_id': cur_node_id}}] query_json = {'query': {'bool': {'must': must}}} query_json['size'] = _es_size search_url = 'http://{}/{}/{}/_search'.format(app.config['ES_HOST'], _es_index_comment, _es_type) response = rs.post(url=search_url, json=query_json, headers=_http_headers).json() print('response: ', json.dumps(response)) if 'hits' in response: comment_list = [] for hit in response['hits']['hits']: data = hit['_source'] data['comment_id'] = hit['_id'] data['updated_at'] = time.strftime( '%Y-%m-%d %H:%M:%S', time.localtime(data['updated_at'])) if 'comment_writer' in data: user_details = get_user_details(data['comment_writer']) data['comment_writer_handle'] = user_details['username'] data['comment_writer_skill_color'] = user_details[ 'skill_color'] data['comment_id'] = hit['_id'] data['vote_count'] = get_vote_count_list(data['comment_id']) child_list = dfs_comment_tree(data['comment_id']) data['comment_list'] = child_list comment_list.append(data) return comment_list app.logger.error('Elasticsearch down, response: ' + str(response)) raise Exception('Internal server error') except Exception as e: raise e
def get(self, resource_id): app.logger.info('Get resource_details method called') rs = requests.session() search_url = 'http://{}/{}/{}/{}'.format(app.config['ES_HOST'], _es_index, _es_type, resource_id) response = rs.get(url=search_url, headers=_http_headers).json() print(response) if 'found' in response: if response['found']: data = response['_source'] data['id'] = response['_id'] data['resource_id'] = response['_id'] data['updated_at'] = time.strftime( '%Y-%m-%d %H:%M:%S', time.localtime(data['updated_at'])) user_details = get_user_details(data['resource_writer']) data['resource_writer_handle'] = user_details['username'] data['resource_writer_skill_color'] = user_details[ 'skill_color'] data['vote_count'] = get_vote_count_list(data['resource_id']) app.logger.info('Get resource_details method completed') return data, 200 app.logger.warning('Resource not found') return {'found': response['found']}, 404 app.logger.error('Elasticsearch down, response: ' + str(response)) return response, 500
def get(self, user_id): try: app.logger.info('Get user API called, id: ' + str(user_id)) data = get_user_details(user_id) data['rating_history'] = get_user_rating_history(user_id) return data except Exception as e: return {'message': str(e)}, 500
def put(self, user_id): app.logger.info('Sync user API called, id: ' + str(user_id)) try: logged_in_user = get_jwt_identity().get('id') logged_in_user_details = get_user_details(logged_in_user) logged_in_user_role = logged_in_user_details.get('user_role', None) response = add_pending_job(user_id, logged_in_user_role, 'USER_SYNC') return response, 200 except Exception as e: return {'message': str(e)}, 500
def put(self, team_id): app.logger.info('Sync team API called, id: ' + str(team_id)) try: logged_in_user = get_jwt_identity().get('id') logged_in_user_details = get_user_details(logged_in_user) logged_in_user_role = logged_in_user_details.get('user_role', None) app.logger.info('team_training_model_sync') add_pending_job(team_id, logged_in_user_role, 'TEAM_SYNC') return {'message': 'success'}, 200 except Exception as e: return {'message': str(e)}, 500
def post(self): try: app.logger.info('generate-crypty-key service called') current_user = get_jwt_identity().get('id') user_details = get_user_details(current_user) if user_details['user_role'] != 'admin': return {'message': 'bad request'}, 400 flask_crypto.generate_key() return {'message': 'success'}, 200 except Exception as e: return {'message': str(e)}, 500
def post(self): try: current_user = get_jwt_identity().get('id') user_details = get_user_details(current_user) if user_details['user_role'] != 'admin': return {'message': 'bad request'}, 400 user_list_sync() app.logger.info('user_list_sync done') team_list_sync() app.logger.info('team_list_sync done') return {'message': 'success'}, 200 except Exception as e: return {'message': str(e)}, 500
def post(self): try: app.logger.info('Create job method called') logged_in_user = get_jwt_identity().get('id') logged_in_user_details = get_user_details(logged_in_user) logged_in_user_role = logged_in_user_details.get('user_role', None) data = request.get_json() job_ref_id = data['job_ref_id'] job_type = data['job_type'] response = add_pending_job(job_ref_id, logged_in_user_role, job_type) return response, 200 except Exception as e: return {'message': str(e)}, 500
def post(self, page=0): app.logger.info('Blog search method called') rs = requests.session() param = request.get_json() query_json = {'query': {'match_all': {}}} must = [] keyword_fields = [ 'blog_title', 'blog_root', 'blog_type', 'blog_ref_id' ] for f in param: if f in keyword_fields: must.append({'term': {f: param[f]}}) else: must.append({'match': {f: param[f]}}) if len(must) > 0: query_json = {'query': {'bool': {'must': must}}} query_json['from'] = page * _es_size query_json['size'] = _es_size query_json['sort'] = [{'updated_at': {'order': 'desc'}}] search_url = 'http://{}/{}/{}/_search'.format(app.config['ES_HOST'], _es_index, _es_type) response = rs.post(url=search_url, json=query_json, headers=_http_headers).json() print(response) if 'hits' in response: item_list = [] for hit in response['hits']['hits']: blog = hit['_source'] blog['id'] = hit['_id'] blog['blog_id'] = hit['_id'] blog['updated_at'] = time.strftime( '%Y-%m-%d %H:%M:%S', time.localtime(blog['updated_at'])) blog['comment_list'] = get_comment_list(blog['blog_id']) blog['vote_count'] = get_vote_count_list(blog['blog_id']) blog['comment_count'] = get_comment_count(blog['blog_id']) if 'blog_writer' in blog: user_details = get_user_details(blog['blog_writer']) blog['blog_writer_skill_color'] = user_details[ 'skill_color'] blog['blog_writer_handle'] = user_details['username'] item_list.append(blog) print(item_list) app.logger.info('Blog search method completed') return {'blog_list': item_list} app.logger.error('Elasticsearch down, response: ' + str(response)) return {'message': 'internal server error'}, 500
def search_resource(param, from_val, size): try: rs = requests.session() query_json = {'query': {'match_all': {}}} must = [] keyword_fields = [ 'resource_ref_id', 'resource_type', 'resource_title', 'resource_writer', 'resource_owner' ] for f in param: if f in keyword_fields: must.append({'term': {f: param[f]}}) else: must.append({'match': {f: param[f]}}) if len(must) > 0: query_json = {'query': {'bool': {'must': must}}} query_json['size'] = size query_json['from'] = from_val query_json['sort'] = [{'created_at': {'order': 'desc'}}] search_url = 'http://{}/{}/{}/_search'.format(app.config['ES_HOST'], _es_index_resource, _es_type) response = rs.post(url=search_url, json=query_json, headers=_http_headers).json() print('response: ', response) item_list = [] if 'hits' in response: for hit in response['hits']['hits']: data = hit['_source'] data['id'] = hit['_id'] user_details = get_user_details(data['resource_writer']) data['resource_writer_handle'] = user_details['username'] data['resource_writer_skill_color'] = user_details[ 'skill_color'] data['resource_id'] = hit['_id'] data['updated_at'] = time.strftime( '%Y-%m-%d %H:%M:%S', time.localtime(data['updated_at'])) data['vote_count'] = get_vote_count_list(data['resource_id']) item_list.append(data) print('item_list: ', item_list) return item_list except Exception as e: raise e
def get_user_problem_submission_history(user_id, start, size): try: rs = requests.session() must = [ { 'term': { 'user_id': user_id } }, { 'term': { 'status': SOLVED } }, ] query_json = {'query': {'bool': {'must': must}}} query_json['from'] = start query_json['size'] = size query_json['sort'] = [{'updated_at': {'order': 'desc'}}] search_url = 'http://{}/{}/{}/_search'.format(app.config['ES_HOST'], _es_index_problem_user, _es_type) response = rs.post(url=search_url, json=query_json, headers=_http_headers).json() submission_list = [] resp = {} if 'hits' in response: order = 1 resp['total'] = response['hits']['total']['value'] for hit in response['hits']['hits']: edge = hit['_source'] user_id = edge['user_id'] user_data = get_user_details(user_id) edge['user_handle'] = user_data['username'] edge['user_skill_color'] = user_data['skill_color'] edge['order'] = order + start edge['problem_details'] = get_problem_details( edge['problem_id']) submission_list.append(edge) order += 1 resp['submission_list'] = submission_list return resp raise Exception('Elasticsearch down') except Exception as e: raise e
def user_list_sync(): app.logger.info(f'user_list_sync called') user_list = search_user({}, 0, _es_size) for user in user_list: id = user['id'] user_problem_data_sync(id) app.logger.info(f'user_problem_data_sync done') # user_training_model_sync(id) # app.logger.info(f'user_training_model_sync done') details_info = get_user_details(id) skill_value = details_info.get('skill_value', 0) solve_count = details_info.get('solve_count', 0) app.logger.info( f'skill_value: {skill_value}, solve_count: {solve_count}') previous_problem_score = details_info['total_score'] app.logger.info(f'previous_problem_score: {previous_problem_score}') current_problem_score = get_total_problem_score_for_user([id]) app.logger.info(f'current_problem_score: {current_problem_score}') updated_data = {'total_score': current_problem_score} decrease_factor = 0 if current_problem_score < previous_problem_score + details_info[ 'target_score']: decrease_amount = previous_problem_score + details_info[ 'target_score'] - current_problem_score app.logger.info(f'decrease_amount: {decrease_amount}') decrease_factor = math.sqrt(decrease_amount) skill = Skill() updated_data['decreased_skill_value'] = details_info[ 'decreased_skill_value'] + decrease_factor current_skill = skill_value - updated_data['decreased_skill_value'] app.logger.info(f'current_skill: {current_skill}') current_skill_level = skill.get_skill_level_from_skill(current_skill) app.logger.info(f'decrease_amount: {current_skill_level}') updated_data['target_score'] = skill.generate_next_week_prediction( current_skill_level) app.logger.info(f'updated_data: {updated_data}') add_user_ratings(id, current_skill, solve_count) update_user_details(id, updated_data) app.logger.info(f'user_list_sync completed')
def get(self, user_id): try: rs = requests.session() must = [{'term': {'followed_by': user_id}}] query_json = {'query': {'bool': {'must': must}}} query_json['sort'] = [{'updated_at': {'order': 'asc'}}] query_json['size'] = _es_size url = 'http://{}/{}/{}/_search'.format(app.config['ES_HOST'], _es_index_followers, _es_type) response = rs.post(url=url, json=query_json, headers=_http_headers).json() if 'hits' in response: item_list = [] order = 1 for hit in response['hits']['hits']: data = hit['_source'] user_id = data['user_id'] print('Find User: '******'User not found {user_id}') continue print('user_details: ', user_details) user_data = { 'user_id': user_id, 'order': order, 'user_handle': user_details['username'], 'skill_value': user_details['skill_value'], 'skill_color': user_details['skill_color'], } item_list.append(user_data) order += 1 return {'following_list': item_list} return response, 500 except Exception as e: return {'message': str(e)}, 500
def search_class_lists(param, from_val, size_val): try: rs = requests.session() query_json = {'query': {'match_all': {}}} must = [] keyword_fields = ['classroom_id'] for f in param: if f in keyword_fields: must.append({'term': {f: param[f]}}) else: must.append({'match': {f: param[f]}}) if len(must) > 0: query_json = {'query': {'bool': {'must': must}}} query_json['from'] = from_val query_json['size'] = size_val query_json['sort'] = [{'updated_at': {'order': 'desc'}}] search_url = 'http://{}/{}/{}/_search'.format( app.config['ES_HOST'], _es_index_classroom_classes, _es_type) response = rs.post(url=search_url, json=query_json, headers=_http_headers).json() item_list = [] if 'hits' in response: for hit in response['hits']['hits']: data = hit['_source'] data['id'] = hit['_id'] user_details = get_user_details(data['class_moderator_id']) data['class_moderator_id_user_handle'] = user_details[ 'username'] data['class_moderator_skill_color'] = user_details[ 'skill_color'] item_list.append(data) return item_list except Exception as e: raise e
def get_contest_details(contest_id): try: rs = requests.session() search_url = 'http://{}/{}/{}/{}'.format(app.config['ES_HOST'], _es_index_contest, _es_type, contest_id) response = rs.get(url=search_url, headers=_http_headers).json() if 'found' in response: if response['found']: data = response['_source'] data['id'] = response['_id'] setter_data = get_user_details(data['setter_id']) data['setter_handle'] = setter_data['username'] data['setter_handle'] = setter_data['username'] data['setter_skill_color'] = setter_data['skill_color'] data['problem_set'] = find_problem_set_for_contest(contest_id) return data raise Exception('Contest not found') app.logger.error('Elasticsearch down, response: ' + str(response)) raise Exception('Elasticsearch Down') except Exception as e: raise e
def put(self, sync_type, team_id): app.logger.info('Sync-administration user API called, team_id: ' + str(team_id)) try: logged_in_user = get_jwt_identity().get('id') logged_in_user_details = get_user_details(logged_in_user) logged_in_user_role = logged_in_user_details.get('user_role', None) response = None if team_id == 'all': if sync_type == 'sync-restore': response = add_pending_job(logged_in_user, logged_in_user_role, 'RESTORE_ALL_TEAMS') elif sync_type == 'sync': response = add_pending_job(logged_in_user, logged_in_user_role, 'SYNC_ALL_TEAMS') else: if sync_type == 'sync-restore': response = add_pending_job(team_id, logged_in_user_role, 'TEAM_SYNC_RESTORE') elif sync_type == 'sync': response = add_pending_job(team_id, logged_in_user_role, 'TEAM_SYNC') return response, 200 except Exception as e: return {'message': str(e)}, 500
def synch_user_problem(user_id): try: uva = UvaScrapper() codeforces = CodeforcesScrapper() spoj = SpojScrapper() codechef = CodechefScrapper() lightoj = LightOJScrapper() user_info = get_user_details(user_id) print('user_info: ', user_info) allowed_judges = ['codeforces', 'uva', 'codechef', 'spoj', 'lightoj'] oj_problem_set = [] if 'codeforces' in allowed_judges: handle = user_info.get('codeforces_handle', None) print('Codeforces: ', handle) if handle: problem_stat = codeforces.get_user_info_heavy(handle) oj_problem_set.append({ 'problem_list': problem_stat['solved_problems'], 'oj_name': 'codeforces' }) app.logger.info(f'codeforces problem_stat: {problem_stat}') print('problem_stat: ',problem_stat) if 'codechef' in allowed_judges: handle = user_info.get('codechef_handle', None) print('codechef: ', handle) if handle: problem_stat = codechef.get_user_info_heavy(handle) oj_problem_set.append({ 'problem_list': problem_stat['solved_problems'], 'oj_name': 'codechef' }) if 'uva' in allowed_judges: handle = user_info.get('uva_handle', None) print('uva: ', handle) if handle: problem_stat = uva.get_user_info_heavy(handle) oj_problem_set.append({ 'problem_list': problem_stat['solved_problems'], 'oj_name': 'uva' }) if 'spoj' in allowed_judges: handle = user_info.get('spoj_handle', None) print('spoj: ', handle) if handle: problem_stat = spoj.get_user_info_heavy(handle) oj_problem_set.append({ 'problem_list': problem_stat['solved_problems'], 'oj_name': 'spoj' }) if 'lightoj' in allowed_judges: handle = user_info.get('lightoj_handle', None) app.logger.info(f'lightoj handle: {handle}') if handle: credentials = { 'username': app.config['LIGHTOJ_USERNAME'], 'password': app.config['LIGHTOJ_PASSWORD'] } problem_stat = lightoj.get_user_info_heavy(handle, credentials) oj_problem_set.append({ 'problem_list': problem_stat['solved_problems'], 'oj_name': 'lightoj' }) sync_problems(user_id, oj_problem_set) except Exception as e: raise e
def search_contests(param, from_value, size_value): try: query_json = {'query': {'match_all': {}}} rs = requests.session() must = [] keyword_fields = [ 'setter_id', 'contest_ref_id', 'contest_type', 'contest_level' ] minimum_level = 0 maximum_level = 100 if 'minimum_level' in param and param['minimum_level']: minimum_level = int(param['minimum_level']) if 'maximum_level' in param and param['maximum_level']: maximum_level = int(param['maximum_level']) param.pop('minimum_level', None) param.pop('maximum_level', None) for f in param: if f in keyword_fields: if param[f]: must.append({'term': {f: param[f]}}) else: if param[f]: must.append({'match': {f: param[f]}}) must.append({ "range": { "contest_level": { "gte": minimum_level, "lte": maximum_level } } }) if len(must) > 0: query_json = {'query': {'bool': {'must': must}}} if 'category_root' not in param: if 'query' in query_json and 'bool' in query_json['query']: query_json['query']['bool']['must_not'] = [{ 'term': { 'category_root': 'root' } }] else: query_json = { 'query': { 'bool': { 'must_not': [{ 'term': { 'category_root': 'root' } }] } } } query_json['from'] = from_value query_json['size'] = size_value print('query_json: ', json.dumps(query_json)) search_url = 'http://{}/{}/{}/_search'.format(app.config['ES_HOST'], _es_index_contest, _es_type) response = rs.post(url=search_url, json=query_json, headers=_http_headers).json() item_list = [] if 'hits' in response: for hit in response['hits']['hits']: data = hit['_source'] data['id'] = hit['_id'] setter_data = get_user_details(data['setter_id']) data['setter_handle'] = setter_data['username'] data['setter_skill_color'] = setter_data['skill_color'] item_list.append(data) return item_list app.logger.error('Elasticsearch down, response: ' + str(response)) return item_list except Exception as e: raise e