def save_or_skip(self, result_article, main_search, parent, where):
try:
result = save_or_skip(result_article,
main_search,
parent,
skip_similarity_check=True)
if result and where not in WORKER_NAMES:
send_to_websocket(self.channel_layer,
where=where,
method='success',
message='')
if result and result.link != main_search.link and main_search.twitter_search:
statusUpdate.get(TWITTER_URL_SEARCHER_NAME).queued(
main_search.id)
send_to_worker(self.channel_layer,
sender=self.name,
where=TWITTER_URL_SEARCHER_NAME,
method='search',
body={
'link':
result.link,
'search_id':
main_search.id,
'parent':
Parent(id=result.link,
type=self.name).to_dict()
})
except Exception as e:
self.log(logging.WARNING,
'Object was not added to database: {}'.format(str(e)))
def process_link(self, msg):
main_search_id = msg['body']['search_id']
updater = statusUpdate.get(self.name)
updater.in_progress(main_search_id)
if search_cancelled(main_search_id):
self.log(logging.INFO, 'Search cancelled, finishing')
updater.success(main_search_id)
return
try:
asyncio.set_event_loop(asyncio.new_event_loop())
link = msg['body']['link']
date = datetime.fromtimestamp(int(msg['body'].get('date'))) if msg['body'].get('date') else None
title = msg['body'].get('title') or ''
snippet = msg['body'].get('snippet') or ''
main_search = get_main_search(main_search_id)
parent = Parent.from_dict(msg['body']['parent'])
sender = msg['sender']
if ImportedArticle.objects.filter(link=link).exists() and main_search.db_search:
statusUpdate.get(DB_URL_SEARCHER_NAME).queued(main_search_id)
send_to_worker(self.channel_layer, sender=sender, where=DB_URL_SEARCHER_NAME,
method='search', body={
'link': link,
'search_id': main_search.id,
'parent': parent.to_dict()
})
return
if is_valid(link) and main_search.link != link:
try:
with transaction.atomic():
domain_str = get_domain(link)
domain, _ = Domain.objects.get_or_create(link=domain_str)
result = get_or_create(link, date, domain_str, domain, title, snippet)
add_parent(result, parent)
if main_search.twitter_search:
statusUpdate.get(TWITTER_URL_SEARCHER_NAME).queued(main_search_id)
send_to_worker(self.channel_layer, sender=sender, where=TWITTER_URL_SEARCHER_NAME,
method='search', body={
'link': result.link,
'search_id': main_search.id,
'parent': Parent(id=result.link, type=self.name).to_dict()
})
except Exception as e:
self.log(logging.WARNING, 'Object was not added to database: {}'.format(str(e)))
if sender not in WORKER_NAMES:
send_to_websocket(self.channel_layer, where=sender, method='success', message='')
except Exception as e:
print(traceback.format_exc())
self.log(logging.ERROR, 'Failed: {0}'.format(str(e)))
def save_tweet(self, tweet, parent, where):
try:
save_tweet(tweet, parent)
if where not in WORKER_NAMES:
send_to_websocket(self.channel_layer,
where=where,
method='success',
message='')
except Exception as e:
self.log(logging.WARNING,
'Object was not added to database: {}'.format(str(e)))