def extract_post(self, data, post_type): post = Post() if data['publish_date'] is None: publish_date_date = calendar.timegm(time.gmtime()) * 1000 else: publish_date_date = data['publish_date']['$date'] date_str = datetime.datetime.fromtimestamp( publish_date_date / 1000).strftime('%Y-%m-%d %H:%M:%S') post.post_id = compute_post_guid(data['url'], data['source'], date_str) post.guid = post.post_id post.author_guid = compute_author_guid_by_author_name(data['source']) post.author = str(data['source']) post.date = convert_str_to_unicode_datetime(date_str) post.title = str(data['title']) post.url = str(data['url']) post.source_url = str(data['source']) post.content = str(data['text']) post.tags = ','.join(data['keywords']) post.domain = self._domain post.post_type = post_type if 'description' not in data['meta_data']: post.description = "" else: post.description = str(data['meta_data']['description']) return post
def _create_post_by_row(self, record_dict): post = Post() post_id = self._convert_to_unicode_value(record_dict["post_id"]) post.post_osn_id = post_id post.post_id = post_id author_name = self._convert_to_unicode_value(record_dict["tumblog_id"]) post.author = author_name post_short_url = self._convert_to_unicode_value( record_dict["post_short_url"]) self._set_post_url(post_short_url, author_name, post) post_creation_date = self._convert_to_unicode_value( record_dict["created_time_epoch"]) post.created_at = post_creation_date if post_creation_date is not None: post_formatted_creation_date, str_post_formatted_creation_date = convert_epoch_timestamp_to_datetime( post_creation_date) post.date = post_formatted_creation_date else: str_post_formatted_creation_date = self._set_start_date() post.guid = compute_post_guid(post.url, author_name, str_post_formatted_creation_date) post.post_osn_guid = post.guid post.title = self._convert_to_unicode_value(record_dict["post_title"]) post_content = record_dict["post_content"] if post_content != 'NULL': content = json.loads(post_content.decode("utf-8")) #content = eval(record_dict["post_content"]) final_content = "" if 'title' in content.keys(): title = content['title'] final_content += title if 'text' in content.keys(): text = content['text'] final_content += text post.content = self._convert_to_unicode_value(final_content) post.domain = self._domain post.author_guid = compute_author_guid_by_author_name(author_name) post.post_type = self._convert_to_unicode_value( record_dict["post_type"]) post.post_format = self._convert_to_unicode_value( record_dict["post_format"]) post.reblog_key = self._convert_to_unicode_value( record_dict["post_reblog_key"]) post.tags = self._convert_to_unicode_value(record_dict["post_tags"]) post.state = self._convert_to_unicode_value(record_dict["post_state"]) if post.post_osn_id not in self._post_dict: self._post_dict[post.post_osn_id] = post return post
def _add_post(self, title, content): post = Post() post.author = self._author.author_full_name post.author_guid = self._author.author_guid post.content = content post.title = title post.domain = u'test' post.post_id = len(self._posts) post.guid = post.post_id self._db.addPost(post) self._posts.append(post)
def _add_post(self, title, content, _domain=u'Microblog'): post = Post() post.author = self._author.author_full_name post.author_guid = self._author.author_guid post.content = content post.title = title post.domain = _domain post.post_id = title post.guid = title self._db.addPost(post) self._posts.append(post)
def _add_post(self, author_guid, title, content, domain=u'Microblog'): post = Post() post.author = author_guid post.author_guid = author_guid post.content = content post.title = title post.domain = domain post.post_id = title post.guid = post.post_id post.is_detailed = True post.is_LB = False self._db.addPost(post) self._posts.append(post)
def _add_post(self, title, content, author_guid): post = Post() post.author = author_guid post.author_guid = author_guid post.content = content post.title = title post.domain = u'test' post.post_id = len(self._posts) post.guid = post.post_id post.date = date('2020-01-01 23:59:59') self._db.addPost(post) self._db.session.commit() self._posts.append(post)
def _add_post(self, post_id, content, url, _domain=u'Microblog'): post = Post() post.author = u'test_user' post.author_guid = u'test_user' post.content = content post.title = post_id post.domain = _domain post.post_id = post_id post.guid = post_id post.url = url post.source_url = url self._db.addPost(post) self._posts[post_id] = post
def _add_post(self, post_id, content, tags, date_str, domain=u'Microblog'): post = Post() post.author = self._author.author_guid post.author_guid = self._author.author_guid post.content = content post.title = post_id post.domain = domain post.post_id = post_id post.guid = post.post_id post.date = convert_str_to_unicode_datetime(date_str) post.created_at = post.date post.tags = tags self._db.addPost(post) self._author.statuses_count += 1
def _extract_result(result, keywords, domain=u"google_search"): post = Post() post.title = result['title'] post.url = result['link'] post.domain = domain post.content = result['snippet'] post.post_type = result['kind'] if 'cacheId' not in result: post.guid = compute_author_guid_by_author_name(u'{}_{}'.format( post.url, keywords)) else: post.guid = compute_author_guid_by_author_name(u'{}_{}_{}'.format( post.url, keywords, result['cacheId'])) post.post_id = post.guid return post
def setUp(self): self._db = DB() self._db.setUp() self.author_guid = u"author_guid" author = Author() author.author_guid = self.author_guid author.author_full_name = u'author' author.name = u'author_name' author.author_screen_name = u'author_screen_name' author.domain = u'Microblog' author.statuses_count = 10 author.friends_count = 5 author.followers_count = 6 author.favourites_count = 8 author.author_sub_type = u"bot" author.author_type = u"bad" author.created_at = u"2017-06-17 05:00:00" author.default_profile = True author.default_profile_image = True author.verified = True self._db.add_author(author) post = Post() post.author = self.author_guid post.author_guid = self.author_guid post.content = u"content" post.title = u"title" post.domain = u"domain" post.post_id = u"post_id" post.guid = post.post_id post.date = convert_str_to_unicode_datetime("2017-06-14 05:00:00") post.created_at = post.date self._db.addPost(post) self._db.session.commit() self.feature_prefix = u"AccountPropertiesFeatureGenerator_" self.account_properties_feature_generator = AccountPropertiesFeatureGenerator( self._db, **{ 'authors': [author], 'posts': { self.author_guid: [post] } }) self.account_properties_feature_generator.execute()
def _add_post(self, title, content, date_str, domain='Microblog', post_type=None): post = Post() post.author = self._author.author_guid post.author_guid = self._author.author_guid post.content = content post.title = title post.domain = domain post.post_id = title post.guid = post.post_id post.date = convert_str_to_unicode_datetime(date_str) post.created_at = post.date post.post_type = post_type self._db.addPost(post) self._posts.append(post)
def convert_comment_to_post(self, comment, submission, domain=u"Reddit"): post = Post() post.post_osn_id = unicode(comment.id) post.created_at = datetime.fromtimestamp(comment.created) post.date = datetime.fromtimestamp(comment.created) if hasattr(comment, 'author') and comment.author: post.author = unicode(comment.author.name) self._redditors.append(comment.author) else: self._deleted_redditors.append(str(post.date)) post.author = unicode('') post.author_guid = compute_author_guid_by_author_name(post.author) post.url = unicode('https://www.reddit.com' + '/'.join(getattr(comment, 'permalink', '').split('/')[3:7])) post.title = unicode(submission.title) post.content = unicode(getattr(comment, 'body', '').encode('utf-8').strip()) post.guid = compute_post_guid(post.url, post.post_osn_id, date_to_str(post.created_at)) post.domain = domain post.post_type = domain post.post_id = post.guid post.url = u'https://www.reddit.com{}'.format(comment.permalink) return post
def photo_xml_to_post(self, child): p = Post() p.title = str(child.find('title').text) p.url = str(child.find('urls').find('url').text) try: p.tags = ','.join(tag.text for tag in child.find('tags').findall('tag')) except: pass p.created_at = str(child.find('dates').get('posted')) p.date = datetime.datetime.fromtimestamp(int(p.created_at)) p.author = str(child.find('owner').get('nsid')) p.domain = 'flickr' p.author_guid = compute_author_guid_by_author_name(p.author) p.retweet_count = int(child.find('comments').text) p.post_id = compute_post_guid(p.url, p.author, date_to_str(p.date)) p.post_osn_id = str(child.get('id')) if child.find('labels') is not None: p.post_type = ','.join( tag.text for tag in child.find('labels').findall('label')) return p
def create_dummy_post(self): post = Post() post.post_id = unicode(self.post_id) post.author = u"author" post.guid = unicode(generate_random_guid()) post.title = u"title" post.url = u"http://google.com" post.date = str_to_date("2016-08-24 10:00:15") post.content = u"text" post.is_detailed = True post.is_LB = False post.is_valid = True post.domain = u"Google" post.author_guid = unicode(self.author_guid) post.post_osn_id = 123455678 post.retweet_count = 11 post.favorite_count = 10 post.created_at = u"2016-08-24 10:00:15" return post