def yield_actions(self, data): try: # Create indexes categories = {c['id']: c for c in data['categories']} topics = {t['id']: t for t in data['topics']} posts = {p['id']: p for p in data['posts']} def is_visible(obj): if 'deleted_at' in obj and obj['deleted_at']: return False if 'hidden' in obj and obj['hidden']: return False if 'topic_id' in obj: topic = topics[obj['topic_id']] if topic['deleted_at']: return False elif topic['read_restricted']: return False def category_is_visible(category): if category['read_restricted']: return False if category['name'] in {'Meta', 'Feedback', 'Uncategorized'}: return False parent_id = category['parent_category_id'] if parent_id: try: parent = categories[parent_id] except KeyError: try: parent = self.es.get(index=self.index_name, doc_type='category', id=parent_id)['_source'] except NotFoundError: return False return category_is_visible(parent) else: return True def topic_is_visible(topic): if topic['deleted_at']: return False try: category = categories[topic['category_id']] except KeyError: try: category = self.es.get( index=self.index_name, doc_type='category', id=topic['category_id'])['_source'] except NotFoundError: return False #category = categories[topic['category_id']] return category_is_visible(category) def post_is_visible(post): if post['deleted_at'] or post['hidden']: return False try: topic = topics[post['topic_id']] except KeyError: try: topic = self.es.get(index=self.index_name, doc_type='topic', id=post['topic_id'])['_source'] except NotFoundError: return False return topic_is_visible(topic) def delete_posts(topic_id): query = {"query": {"term": {"topic_id": topic_id}}} results = self.es.search(self.index_name, 'post', body=query, source=['id']) for hit in results['hits']: post_id = hit['_source']['id'] action = {'_type': 'post', '_id': post_id} action['_op_type'] = 'delete' yield action for category in data['categories']: action = { '_type': 'category', '_id': category['id'], '_source': category } if not category_is_visible(category): action['_op_type'] = 'delete' yield action for topic in data['topics']: if topic['category_id'] is None: topic['category_id'] = 1 action = { '_type': 'topic', '_id': topic['id'], '_source': topic } if not topic_is_visible(topic): action['_op_type'] = 'delete' delete_posts(topic['id']) yield action for post in data['posts']: try: action = { '_type': 'post', '_id': post['id'], '_source': post } if post_is_visible(post): post['plain'] = self.to_plain_text(post['cooked']) action['_parent'] = post['topic_id'] else: action['_op_type'] = 'delete' yield action except Exception as e: logger.exception(e) continue except: self.locs = locals() raise
def yield_actions(self, data): try: # Create indexes categories = {c['id']: c for c in data['categories']} topics = {t['id']: t for t in data['topics']} posts = {p['id']: p for p in data['posts']} def is_visible(obj): if 'deleted_at' in obj and obj['deleted_at']: return False if 'hidden' in obj and obj['hidden']: return False if 'topic_id' in obj: topic = topics[obj['topic_id']] if topic['deleted_at']: return False elif topic['read_restricted']: return False def category_is_visible(category): if category['read_restricted']: return False if category['name'] in {'Meta', 'Feedback'}: return False parent_id = category['parent_category_id'] if parent_id: try: parent = categories[parent_id] except KeyError: try: parent = self.es.get(index=self.index_name, doc_type='category', id=parent_id)['_source'] except NotFoundError: return False return category_is_visible(parent) else: return True def topic_is_visible(topic): if topic['deleted_at']: return False try: category = categories[topic['category_id']] except KeyError: try: category = self.es.get(index=self.index_name, doc_type='category', id=topic['category_id'])['_source'] except NotFoundError: return False #category = categories[topic['category_id']] return category_is_visible(category) def post_is_visible(post): if post['deleted_at'] or post['hidden']: return False try: topic = topics[post['topic_id']] except KeyError: try: topic = self.es.get(index=self.index_name, doc_type='topic', id=post['topic_id'])['_source'] except NotFoundError: return False return topic_is_visible(topic) def delete_posts(topic_id): query = { "query": { "term": { "topic_id": topic_id } } } results = self.es.search(self.index_name, 'post', body=query, source=['id']) for hit in results['hits']: post_id = hit['_source']['id'] action = {'_type': 'post', '_id': post_id} action['_op_type'] = 'delete' yield action for category in data['categories']: action = {'_type': 'category', '_id': category['id'], '_source': category} if not category_is_visible(category): action['_op_type'] = 'delete' yield action for topic in data['topics']: if topic['category_id'] is None: topic['category_id'] = 1 action = {'_type': 'topic', '_id': topic['id'], '_source': topic} if not topic_is_visible(topic): action['_op_type'] = 'delete' delete_posts(topic['id']) yield action for post in data['posts']: try: action = {'_type': 'post', '_id': post['id'], '_source': post} if post_is_visible(post): post['plain'] = self.to_plain_text(post['cooked']) action['_parent'] = post['topic_id'] else: action['_op_type'] = 'delete' yield action except Exception as e: logger.exception(e) continue except: self.locs = locals() raise
def _green_text(text): try: from colorama import Fore return Fore.GREEN + text + Fore.RESET except ImportError: return text def _red_text(text): try: from colorama import Fore return Fore.RED + text + Fore.RESET except ImportError: return text try: import sc from sc import app, updater import cherrypy start_server() t = threading.Thread(target=test_server) t.start() except Exception as e: logger.exception(e) stop(2, 'fails to start due to an exception')