def remove_bad_source(request): params = request.data if 'source_id' not in params: raise EnvironmentError('source_id must be in params') with mongo.MongoConnection() as mongodb: mongodb.remove_bad_source(params['source_id']) return JsonResponse({}, encoder=JSONEncoderHttp)
def update_source_list_from_server(): mongodb = mongo.MongoConnection() try: print("Background task update_source_list_from_server started") inserted_ids, deleted_ids = mongodb.update_source_list_from_server() print("Background task update_source_list_from_server finished. Response: \n{} \n{}".format(inserted_ids, deleted_ids)) except Exception as ex: print(ex) pass
def update_country_list(): mongodb = mongo.MongoConnection() try: print("Background task update_country_list started") mongodb.update_country_list() print("Background task update_country_list finished") except Exception as ex: print(ex) pass
def dev_find_article_ids_with_tag_length_more_than_length(request): params = request.data with mongo.MongoConnection() as mongodb: article_ids = mongodb.dev_find_article_ids_with_tag_length_more_than_length(params) article_ids = [x['article_id'] for x in article_ids] dictinct_article_ids = [] for x in article_ids: if x not in dictinct_article_ids: dictinct_article_ids.append(x) return JsonResponse({'article_ids': dictinct_article_ids, 'total': len(dictinct_article_ids)}, encoder=JSONEncoderHttp)
def fix_article_content_and_retrain_entity_bt_article_ids(request): params = request.data article_ids = params['article_ids'] with mongo.MongoConnection() as mongodb: for article_id in article_ids: mongodb.fix_one_article_by_id({"_id": article_id}) model.get_tags_from_article({ "retrain": True, "article_id": article_id }) return JsonResponse({}, encoder=JSONEncoderHttp)
def fill_up_geolocation(request): with mongo.MongoConnection() as mongodb: mongodb.fill_up_geolocation(mongodb.location, 'name') return JsonResponse({}, encoder=JSONEncoderHttp)
def update_language_list(request): with mongo.MongoConnection() as mongodb: mongodb.update_language_list() return JsonResponse({}, encoder=JSONEncoderHttp)
def delete_permanent_phrase_list(request): params = request.data with mongo.MongoConnection() as mongodb: mongodb.delete_permanent_phrases(params=params) return JsonResponse({}, encoder=JSONEncoderHttp)
def download_articles_by_phrases(request): with mongo.MongoConnection() as mongodb: mongodb.download_articles_by_phrases() return JsonResponse({}, encoder=JSONEncoderHttp)
def get_article_by_id(request): params = request.data with mongo.MongoConnection() as mongodb: article = mongodb.get_article_by_id(params=params) return JsonResponse({'article': article}, encoder=JSONEncoderHttp)
def aggregate_articles_by_locations(request): params = request.data with mongo.MongoConnection() as mongodb: articles = mongodb.aggregate_articles_by_locations(params=params) return JsonResponse(articles, encoder=JSONEncoderHttp)
def get_location_info_by_id(request): params = request.data with mongo.MongoConnection() as mongodb: location = mongodb.get_location_info_by_id(params) return JsonResponse({'location': location}, encoder=JSONEncoderHttp)
def fix_article_content(request): with mongo.MongoConnection() as mongodb: mongodb.fix_article_content() return JsonResponse({}, encoder=JSONEncoderHttp)
def fix_sources_and_add_official_field(request): with mongo.MongoConnection() as mongodb: mongodb.fix_sources_and_add_official_field() return JsonResponse({}, encoder=JSONEncoderHttp)
def load_iso(request): with mongo.MongoConnection() as mongodb: mongodb.load_iso() return JsonResponse({}, encoder=JSONEncoderHttp)
def fix_entity_location(request): with mongo.MongoConnection() as mongodb: mongodb.entity.update({}, {"$unset": {"locations": 1}}, multi=True) model.add_locations_to_untrained_articles() return JsonResponse({}, encoder=JSONEncoderHttp)
def fix_original_fields(request): params = request.data with mongo.MongoConnection() as mongodb: mongodb.fix_original_fields(params) return JsonResponse({}, encoder=JSONEncoderHttp)
def fix_one_article_by_id(request): params = request.data with mongo.MongoConnection() as mongodb: mongodb.fix_one_article_by_id(params) return JsonResponse({}, encoder=JSONEncoderHttp)
def update_category(request): with mongo.MongoConnection() as mongodb: mongodb.update_category() return JsonResponse({}, encoder=JSONEncoderHttp)
def show_source_list(request): params = request.data with mongo.MongoConnection() as mongodb: sources, more = mongodb.show_source_list(params) return JsonResponse({'sources': sources, 'more': more}, encoder=JSONEncoderHttp)
def show_category(request): params = request.data with mongo.MongoConnection() as mongodb: # FUNCTION IN PROGRESS response, more = mongodb.show_category(params=params) return JsonResponse({'state': response, 'more': more}, encoder=JSONEncoderHttp)
def fix_article_source_with_null_id(request): with mongo.MongoConnection() as mongodb: mongodb.fix_article_source_with_null_id() return JsonResponse({}, encoder=JSONEncoderHttp)
def tag_stat_by_articles_list(request): params = request.data with mongo.MongoConnection() as mongodb: tags = mongodb.tag_stat_by_articles_list(params=params) return JsonResponse(tags, encoder=JSONEncoderHttp)
def delete_trash_from_article_content(request): params = request.data text = params['text'] with mongo.MongoConnection() as mongodb: ans = mongodb.delete_trash_from_article_content(text) return JsonResponse({'before': text, 'after': ans}, encoder=JSONEncoderHttp)
def remove_all_bad_source(request): with mongo.MongoConnection() as mongodb: mongodb.remove_all_bad_source() return JsonResponse({}, encoder=JSONEncoderHttp)
def dev_update_sources_by_one_article(request): params = request.data with mongo.MongoConnection() as mongodb: mongodb.dev_update_sources_by_one_article(params) return JsonResponse({}, encoder=JSONEncoderHttp)
def get_article_list(request): params = request.data with mongo.MongoConnection() as mongodb: articles, more = mongodb.get_q_article_list(params=params) return JsonResponse({'articles': articles, 'more': more}, encoder=JSONEncoderHttp)
def remove_dubles_articles_and_entities(request): with mongo.MongoConnection() as mongodb: mongodb.remove_dubles_articles_and_entities() return JsonResponse({}, encoder=JSONEncoderHttp)
import os import sys import django _PATH = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) if _PATH not in sys.path: sys.path.append(_PATH) os.environ.setdefault("DJANGO_SETTINGS_MODULE", "newsAPI.settings") import logging logger = logging.getLogger() logger.info(_PATH) django.setup() logger.info('DJANGO START\n **************************') from lib import mongo_connection as mongo mongodb = mongo.MongoConnection() logger.info("SCHEDULAR UPDATE SOURCES STARTED") inserted_ids, deleted_ids = mongodb.update_source_list_from_server() logger.info('\n {} \n {}'.format(inserted_ids, deleted_ids)) logger.info("SCHEDULAR UPDATE SOURCES FINISHED")
def get_default_entity(request): params = request.data with mongo.MongoConnection() as mongodb: entities = mongodb.get_default_entity(params=params) return JsonResponse({'entities': entities}, encoder=JSONEncoderHttp)