def add_story(db: DatabaseHandler, story: dict, feeds_id: int, skip_checking_if_new: bool = False) -> Optional[dict]:
"""If the story is new, add story to the database with the feed of the download as story feed.
Returns created story or None if story wasn't created.
"""
story = decode_object_from_bytes_if_needed(story)
if isinstance(feeds_id, bytes):
feeds_id = decode_object_from_bytes_if_needed(feeds_id)
feeds_id = int(feeds_id)
if isinstance(skip_checking_if_new, bytes):
skip_checking_if_new = decode_object_from_bytes_if_needed(skip_checking_if_new)
skip_checking_if_new = bool(int(skip_checking_if_new))
if db.in_transaction():
raise McAddStoryException("add_story() can't be run from within transaction.")
db.begin()
db.query("LOCK TABLE stories IN ROW EXCLUSIVE MODE")
if not skip_checking_if_new:
if not is_new(db=db, story=story):
log.debug("Story '{}' is not new.".format(story['url']))
db.commit()
return None
medium = db.find_by_id(table='media', object_id=story['media_id'])
if story.get('full_text_rss', None) is None:
story['full_text_rss'] = medium.get('full_text_rss', False) or False
if len(story.get('description', '')) == 0:
story['full_text_rss'] = False
try:
story = db.create(table='stories', insert_hash=story)
except Exception as ex:
db.rollback()
# FIXME get rid of this, replace with native upsert on "stories_guid" unique constraint
if 'unique constraint \"stories_guid' in str(ex):
log.warning(
"Failed to add story for '{}' to GUID conflict (guid = '{}')".format(story['url'], story['guid'])
)
return None
else:
raise McAddStoryException("Error adding story: {}\nStory: {}".format(str(ex), str(story)))
db.find_or_create(
table='feeds_stories_map',
insert_hash={
'stories_id': story['stories_id'],
'feeds_id': feeds_id,
}
)
db.commit()
return story
def extract_and_process_story(db: DatabaseHandler,
story: dict,
extractor_args: PyExtractorArguments = PyExtractorArguments()) -> None:
"""Extract all of the downloads for the given story and then call process_extracted_story()."""
story = decode_object_from_bytes_if_needed(story)
stories_id = story['stories_id']
use_transaction = not db.in_transaction()
if use_transaction:
db.begin()
log.debug("Fetching downloads for story {}...".format(stories_id))
downloads = db.query("""
SELECT *
FROM downloads
WHERE stories_id = %(stories_id)s
AND type = 'content'
ORDER BY downloads_id ASC
""", {'stories_id': stories_id}).hashes()
# MC_REWRITE_TO_PYTHON: Perlism
if downloads is None:
downloads = []
for download in downloads:
log.debug("Extracting download {} for story {}...".format(download['downloads_id'], stories_id))
extract_and_create_download_text(db=db, download=download, extractor_args=extractor_args)
log.debug("Processing extracted story {}...".format(stories_id))
process_extracted_story(db=db, story=story, extractor_args=extractor_args)
if use_transaction:
db.commit()
def extract_and_process_story(
db: DatabaseHandler,
story: dict,
extractor_args: PyExtractorArguments = PyExtractorArguments()
) -> None:
"""Extract all of the downloads for the given story and then call process_extracted_story()."""
story = decode_object_from_bytes_if_needed(story)
stories_id = story['stories_id']
use_transaction = not db.in_transaction()
if use_transaction:
db.begin()
log.debug("Fetching downloads for story {}...".format(stories_id))
downloads = db.query(
"""
SELECT *
FROM downloads
WHERE stories_id = %(stories_id)s
AND type = 'content'
AND state = 'success'
ORDER BY downloads_id ASC
""", {
'stories_id': stories_id
}).hashes()
# MC_REWRITE_TO_PYTHON: Perlism
if downloads is None:
downloads = []
for download in downloads:
log.debug("Extracting download {} for story {}...".format(
download['downloads_id'], stories_id))
extract_and_create_download_text(db=db,
download=download,
extractor_args=extractor_args)
log.debug("Processing extracted story {}...".format(stories_id))
process_extracted_story(db=db, story=story, extractor_args=extractor_args)
if use_transaction:
db.commit()
def update_story_sentences_and_language(
db: DatabaseHandler,
story: dict,
extractor_args: PyExtractorArguments = PyExtractorArguments()
) -> None:
"""Update story vectors for the given story, updating "story_sentences".
If extractor_args.no_delete() is True, do not try to delete existing entries in the above table before creating new
ones (useful for optimization if you are very sure no story vectors exist for this story).
If extractor_args.no_dedup_sentences() is True, do not perform sentence deduplication (useful if you are
reprocessing a small set of stories).
"""
story = decode_object_from_bytes_if_needed(story)
use_transaction = not db.in_transaction()
if use_transaction:
db.begin()
stories_id = story['stories_id']
if not extractor_args.no_delete():
_delete_story_sentences(db=db, story=story)
story_text = story.get('story_text', None)
if not story_text:
story_text = get_text_for_word_counts(db=db, story=story)
if not story_text:
story_text = ''
story_lang = language_code_for_text(text=story_text)
sentences = _get_sentences_from_story_text(story_text=story_text,
story_lang=story_lang)
if (not story.get('language',
None)) or story.get('language', None) != story_lang:
db.query(
"""
UPDATE stories
SET language = %(story_lang)s
WHERE stories_id = %(stories_id)s
""", {
'stories_id': stories_id,
'story_lang': story_lang
})
story['language'] = story_lang
if sentences is None:
raise McUpdateStorySentencesAndLanguageException(
"Sentences for story {} are undefined.".format(stories_id))
if len(sentences) == 0:
log.debug("Story {} doesn't have any sentences.".format(stories_id))
return
sentences = _clean_sentences(sentences)
_insert_story_sentences(
db=db,
story=story,
sentences=sentences,
no_dedup_sentences=extractor_args.no_dedup_sentences(),
)
story['ap_syndicated'] = _update_ap_syndicated(
db=db,
stories_id=stories_id,
story_title=story['title'],
story_text=story_text,
story_language=story_lang,
)
if use_transaction:
db.commit()
def add_story(db: DatabaseHandler,
story: dict,
feeds_id: int,
skip_checking_if_new: bool = False) -> Optional[dict]:
"""If the story is new, add story to the database with the feed of the download as story feed.
Returns created story or None if story wasn't created.
"""
story = decode_object_from_bytes_if_needed(story)
if isinstance(feeds_id, bytes):
feeds_id = decode_object_from_bytes_if_needed(feeds_id)
feeds_id = int(feeds_id)
if isinstance(skip_checking_if_new, bytes):
skip_checking_if_new = decode_object_from_bytes_if_needed(
skip_checking_if_new)
skip_checking_if_new = bool(int(skip_checking_if_new))
if db.in_transaction():
raise McAddStoryException(
"add_story() can't be run from within transaction.")
db.begin()
db.query("LOCK TABLE stories IN ROW EXCLUSIVE MODE")
if not skip_checking_if_new:
if not is_new(db=db, story=story):
log.debug("Story '{}' is not new.".format(story['url']))
db.commit()
return None
medium = db.find_by_id(table='media', object_id=story['media_id'])
if story.get('full_text_rss', None) is None:
story['full_text_rss'] = medium.get('full_text_rss', False) or False
if len(story.get('description', '')) == 0:
story['full_text_rss'] = False
try:
story = db.create(table='stories', insert_hash=story)
except Exception as ex:
db.rollback()
# FIXME get rid of this, replace with native upsert on "stories_guid" unique constraint
if 'unique constraint \"stories_guid' in str(ex):
log.warning(
"Failed to add story for '{}' to GUID conflict (guid = '{}')".
format(story['url'], story['guid']))
return None
else:
raise McAddStoryException(
"Error adding story: {}\nStory: {}".format(
str(ex), str(story)))
db.find_or_create(table='feeds_stories_map',
insert_hash={
'stories_id': story['stories_id'],
'feeds_id': feeds_id,
})
db.commit()
return story
def add_story(db: DatabaseHandler, story: dict,
feeds_id: int) -> Optional[dict]:
"""Return an existing dup story if it matches the url, guid, or title; otherwise, add a new story and return it.
Returns found or created story. Adds an is_new = True story if the story was created by the call.
"""
story = decode_object_from_bytes_if_needed(story)
if isinstance(feeds_id, bytes):
feeds_id = decode_object_from_bytes_if_needed(feeds_id)
feeds_id = int(feeds_id)
if db.in_transaction():
raise McAddStoryException(
"add_story() can't be run from within transaction.")
db.begin()
db.query("LOCK TABLE stories IN ROW EXCLUSIVE MODE")
db_story = find_dup_story(db, story)
if db_story:
log.debug("found existing dup story: %s [%s]" %
(story['title'], story['url']))
db.commit()
return db_story
medium = db.find_by_id(table='media', object_id=story['media_id'])
if story.get('full_text_rss', None) is None:
story['full_text_rss'] = medium.get('full_text_rss', False) or False
if len(story.get('description', '')) == 0:
story['full_text_rss'] = False
try:
story = db.create(table='stories', insert_hash=story)
except Exception as ex:
db.rollback()
# FIXME get rid of this, replace with native upsert on "stories_guid" unique constraint
if 'unique constraint \"stories_guid' in str(ex):
log.warning(
"Failed to add story for '{}' to GUID conflict (guid = '{}')".
format(story['url'], story['guid']))
return None
else:
raise McAddStoryException(
"Error adding story: {}\nStory: {}".format(
str(ex), str(story)))
story['is_new'] = True
[insert_story_urls(db, story, u) for u in (story['url'], story['guid'])]
# on conflict does not work with partitioned feeds_stories_map
db.query(
"""
insert into feeds_stories_map_p ( feeds_id, stories_id )
select %(a)s, %(b)s where not exists (
select 1 from feeds_stories_map where feeds_id = %(a)s and stories_id = %(b)s )
""", {
'a': feeds_id,
'b': story['stories_id']
})
db.commit()
log.debug("added story: %s" % story['url'])
return story
