def yield_actions(self, data):
try:
# Create indexes
categories = {c['id']: c for c in data['categories']}
topics = {t['id']: t for t in data['topics']}
posts = {p['id']: p for p in data['posts']}
def is_visible(obj):
if 'deleted_at' in obj and obj['deleted_at']:
return False
if 'hidden' in obj and obj['hidden']:
return False
if 'topic_id' in obj:
topic = topics[obj['topic_id']]
if topic['deleted_at']:
return False
elif topic['read_restricted']:
return False
def category_is_visible(category):
if category['read_restricted']:
return False
if category['name'] in {'Meta', 'Feedback', 'Uncategorized'}:
return False
parent_id = category['parent_category_id']
if parent_id:
try:
parent = categories[parent_id]
except KeyError:
try:
parent = self.es.get(index=self.index_name,
doc_type='category',
id=parent_id)['_source']
except NotFoundError:
return False
return category_is_visible(parent)
else:
return True
def topic_is_visible(topic):
if topic['deleted_at']:
return False
try:
category = categories[topic['category_id']]
except KeyError:
try:
category = self.es.get(
index=self.index_name,
doc_type='category',
id=topic['category_id'])['_source']
except NotFoundError:
return False
#category = categories[topic['category_id']]
return category_is_visible(category)
def post_is_visible(post):
if post['deleted_at'] or post['hidden']:
return False
try:
topic = topics[post['topic_id']]
except KeyError:
try:
topic = self.es.get(index=self.index_name,
doc_type='topic',
id=post['topic_id'])['_source']
except NotFoundError:
return False
return topic_is_visible(topic)
def delete_posts(topic_id):
query = {"query": {"term": {"topic_id": topic_id}}}
results = self.es.search(self.index_name,
'post',
body=query,
source=['id'])
for hit in results['hits']:
post_id = hit['_source']['id']
action = {'_type': 'post', '_id': post_id}
action['_op_type'] = 'delete'
yield action
for category in data['categories']:
action = {
'_type': 'category',
'_id': category['id'],
'_source': category
}
if not category_is_visible(category):
action['_op_type'] = 'delete'
yield action
for topic in data['topics']:
if topic['category_id'] is None:
topic['category_id'] = 1
action = {
'_type': 'topic',
'_id': topic['id'],
'_source': topic
}
if not topic_is_visible(topic):
action['_op_type'] = 'delete'
delete_posts(topic['id'])
yield action
for post in data['posts']:
try:
action = {
'_type': 'post',
'_id': post['id'],
'_source': post
}
if post_is_visible(post):
post['plain'] = self.to_plain_text(post['cooked'])
action['_parent'] = post['topic_id']
else:
action['_op_type'] = 'delete'
yield action
except Exception as e:
logger.exception(e)
continue
except:
self.locs = locals()
raise
def yield_actions(self, data):
try:
# Create indexes
categories = {c['id']: c for c in data['categories']}
topics = {t['id']: t for t in data['topics']}
posts = {p['id']: p for p in data['posts']}
def is_visible(obj):
if 'deleted_at' in obj and obj['deleted_at']:
return False
if 'hidden' in obj and obj['hidden']:
return False
if 'topic_id' in obj:
topic = topics[obj['topic_id']]
if topic['deleted_at']:
return False
elif topic['read_restricted']:
return False
def category_is_visible(category):
if category['read_restricted']:
return False
if category['name'] in {'Meta', 'Feedback'}:
return False
parent_id = category['parent_category_id']
if parent_id:
try:
parent = categories[parent_id]
except KeyError:
try:
parent = self.es.get(index=self.index_name, doc_type='category', id=parent_id)['_source']
except NotFoundError:
return False
return category_is_visible(parent)
else:
return True
def topic_is_visible(topic):
if topic['deleted_at']:
return False
try:
category = categories[topic['category_id']]
except KeyError:
try:
category = self.es.get(index=self.index_name, doc_type='category', id=topic['category_id'])['_source']
except NotFoundError:
return False
#category = categories[topic['category_id']]
return category_is_visible(category)
def post_is_visible(post):
if post['deleted_at'] or post['hidden']:
return False
try:
topic = topics[post['topic_id']]
except KeyError:
try:
topic = self.es.get(index=self.index_name, doc_type='topic', id=post['topic_id'])['_source']
except NotFoundError:
return False
return topic_is_visible(topic)
def delete_posts(topic_id):
query = {
"query": {
"term": {
"topic_id": topic_id
}
}
}
results = self.es.search(self.index_name, 'post', body=query, source=['id'])
for hit in results['hits']:
post_id = hit['_source']['id']
action = {'_type': 'post', '_id': post_id}
action['_op_type'] = 'delete'
yield action
for category in data['categories']:
action = {'_type': 'category',
'_id': category['id'],
'_source': category}
if not category_is_visible(category):
action['_op_type'] = 'delete'
yield action
for topic in data['topics']:
if topic['category_id'] is None:
topic['category_id'] = 1
action = {'_type': 'topic',
'_id': topic['id'],
'_source': topic}
if not topic_is_visible(topic):
action['_op_type'] = 'delete'
delete_posts(topic['id'])
yield action
for post in data['posts']:
try:
action = {'_type': 'post',
'_id': post['id'],
'_source': post}
if post_is_visible(post):
post['plain'] = self.to_plain_text(post['cooked'])
action['_parent'] = post['topic_id']
else:
action['_op_type'] = 'delete'
yield action
except Exception as e:
logger.exception(e)
continue
except:
self.locs = locals()
raise
def _green_text(text):
try:
from colorama import Fore
return Fore.GREEN + text + Fore.RESET
except ImportError:
return text
def _red_text(text):
try:
from colorama import Fore
return Fore.RED + text + Fore.RESET
except ImportError:
return text
try:
import sc
from sc import app, updater
import cherrypy
start_server()
t = threading.Thread(target=test_server)
t.start()
except Exception as e:
logger.exception(e)
stop(2, 'fails to start due to an exception')
def _green_text(text):
try:
from colorama import Fore
return Fore.GREEN + text + Fore.RESET
except ImportError:
return text
def _red_text(text):
try:
from colorama import Fore
return Fore.RED + text + Fore.RESET
except ImportError:
return text
try:
import sc
from sc import app, updater
import cherrypy
start_server()
t = threading.Thread(target=test_server)
t.start()
except Exception as e:
logger.exception(e)
stop(2, 'fails to start due to an exception')
