def remove_bad_source(request):
params = request.data
if 'source_id' not in params:
raise EnvironmentError('source_id must be in params')
with mongo.MongoConnection() as mongodb:
mongodb.remove_bad_source(params['source_id'])
return JsonResponse({}, encoder=JSONEncoderHttp)
def update_source_list_from_server():
mongodb = mongo.MongoConnection()
try:
print("Background task update_source_list_from_server started")
inserted_ids, deleted_ids = mongodb.update_source_list_from_server()
print("Background task update_source_list_from_server finished. Response: \n{} \n{}".format(inserted_ids, deleted_ids))
except Exception as ex:
print(ex)
pass
def update_country_list():
mongodb = mongo.MongoConnection()
try:
print("Background task update_country_list started")
mongodb.update_country_list()
print("Background task update_country_list finished")
except Exception as ex:
print(ex)
pass
def dev_find_article_ids_with_tag_length_more_than_length(request):
params = request.data
with mongo.MongoConnection() as mongodb:
article_ids = mongodb.dev_find_article_ids_with_tag_length_more_than_length(params)
article_ids = [x['article_id'] for x in article_ids]
dictinct_article_ids = []
for x in article_ids:
if x not in dictinct_article_ids:
dictinct_article_ids.append(x)
return JsonResponse({'article_ids': dictinct_article_ids, 'total': len(dictinct_article_ids)}, encoder=JSONEncoderHttp)
def fix_article_content_and_retrain_entity_bt_article_ids(request):
params = request.data
article_ids = params['article_ids']
with mongo.MongoConnection() as mongodb:
for article_id in article_ids:
mongodb.fix_one_article_by_id({"_id": article_id})
model.get_tags_from_article({
"retrain": True,
"article_id": article_id
})
return JsonResponse({}, encoder=JSONEncoderHttp)
def fill_up_geolocation(request):
with mongo.MongoConnection() as mongodb:
mongodb.fill_up_geolocation(mongodb.location, 'name')
return JsonResponse({}, encoder=JSONEncoderHttp)
def update_language_list(request):
with mongo.MongoConnection() as mongodb:
mongodb.update_language_list()
return JsonResponse({}, encoder=JSONEncoderHttp)
def delete_permanent_phrase_list(request):
params = request.data
with mongo.MongoConnection() as mongodb:
mongodb.delete_permanent_phrases(params=params)
return JsonResponse({}, encoder=JSONEncoderHttp)
def download_articles_by_phrases(request):
with mongo.MongoConnection() as mongodb:
mongodb.download_articles_by_phrases()
return JsonResponse({}, encoder=JSONEncoderHttp)
def get_article_by_id(request):
params = request.data
with mongo.MongoConnection() as mongodb:
article = mongodb.get_article_by_id(params=params)
return JsonResponse({'article': article}, encoder=JSONEncoderHttp)
def aggregate_articles_by_locations(request):
params = request.data
with mongo.MongoConnection() as mongodb:
articles = mongodb.aggregate_articles_by_locations(params=params)
return JsonResponse(articles, encoder=JSONEncoderHttp)
def get_location_info_by_id(request):
params = request.data
with mongo.MongoConnection() as mongodb:
location = mongodb.get_location_info_by_id(params)
return JsonResponse({'location': location}, encoder=JSONEncoderHttp)
def fix_article_content(request):
with mongo.MongoConnection() as mongodb:
mongodb.fix_article_content()
return JsonResponse({}, encoder=JSONEncoderHttp)
def fix_sources_and_add_official_field(request):
with mongo.MongoConnection() as mongodb:
mongodb.fix_sources_and_add_official_field()
return JsonResponse({}, encoder=JSONEncoderHttp)
def load_iso(request):
with mongo.MongoConnection() as mongodb:
mongodb.load_iso()
return JsonResponse({}, encoder=JSONEncoderHttp)
def fix_entity_location(request):
with mongo.MongoConnection() as mongodb:
mongodb.entity.update({}, {"$unset": {"locations": 1}}, multi=True)
model.add_locations_to_untrained_articles()
return JsonResponse({}, encoder=JSONEncoderHttp)
def fix_original_fields(request):
params = request.data
with mongo.MongoConnection() as mongodb:
mongodb.fix_original_fields(params)
return JsonResponse({}, encoder=JSONEncoderHttp)
def fix_one_article_by_id(request):
params = request.data
with mongo.MongoConnection() as mongodb:
mongodb.fix_one_article_by_id(params)
return JsonResponse({}, encoder=JSONEncoderHttp)
def update_category(request):
with mongo.MongoConnection() as mongodb:
mongodb.update_category()
return JsonResponse({}, encoder=JSONEncoderHttp)
def show_source_list(request):
params = request.data
with mongo.MongoConnection() as mongodb:
sources, more = mongodb.show_source_list(params)
return JsonResponse({'sources': sources, 'more': more}, encoder=JSONEncoderHttp)
def show_category(request):
params = request.data
with mongo.MongoConnection() as mongodb:
# FUNCTION IN PROGRESS
response, more = mongodb.show_category(params=params)
return JsonResponse({'state': response, 'more': more}, encoder=JSONEncoderHttp)
def fix_article_source_with_null_id(request):
with mongo.MongoConnection() as mongodb:
mongodb.fix_article_source_with_null_id()
return JsonResponse({}, encoder=JSONEncoderHttp)
def tag_stat_by_articles_list(request):
params = request.data
with mongo.MongoConnection() as mongodb:
tags = mongodb.tag_stat_by_articles_list(params=params)
return JsonResponse(tags, encoder=JSONEncoderHttp)
def delete_trash_from_article_content(request):
params = request.data
text = params['text']
with mongo.MongoConnection() as mongodb:
ans = mongodb.delete_trash_from_article_content(text)
return JsonResponse({'before': text, 'after': ans}, encoder=JSONEncoderHttp)
def remove_all_bad_source(request):
with mongo.MongoConnection() as mongodb:
mongodb.remove_all_bad_source()
return JsonResponse({}, encoder=JSONEncoderHttp)
def dev_update_sources_by_one_article(request):
params = request.data
with mongo.MongoConnection() as mongodb:
mongodb.dev_update_sources_by_one_article(params)
return JsonResponse({}, encoder=JSONEncoderHttp)
def get_article_list(request):
params = request.data
with mongo.MongoConnection() as mongodb:
articles, more = mongodb.get_q_article_list(params=params)
return JsonResponse({'articles': articles, 'more': more}, encoder=JSONEncoderHttp)
def remove_dubles_articles_and_entities(request):
with mongo.MongoConnection() as mongodb:
mongodb.remove_dubles_articles_and_entities()
return JsonResponse({}, encoder=JSONEncoderHttp)
import os
import sys
import django
_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
if _PATH not in sys.path:
sys.path.append(_PATH)
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "newsAPI.settings")
import logging
logger = logging.getLogger()
logger.info(_PATH)
django.setup()
logger.info('DJANGO START\n **************************')
from lib import mongo_connection as mongo
mongodb = mongo.MongoConnection()
logger.info("SCHEDULAR UPDATE SOURCES STARTED")
inserted_ids, deleted_ids = mongodb.update_source_list_from_server()
logger.info('\n {} \n {}'.format(inserted_ids, deleted_ids))
logger.info("SCHEDULAR UPDATE SOURCES FINISHED")
def get_default_entity(request):
params = request.data
with mongo.MongoConnection() as mongodb:
entities = mongodb.get_default_entity(params=params)
return JsonResponse({'entities': entities}, encoder=JSONEncoderHttp)
