def fetch_user_history_rows(username, start_date, end_date, comment_keys,
only_authored):
num_posts = 0
num_comments = 0
current_date = None
# Iterate through the user history. Note that the comment operations returned from the history
# api call are not the same as the full comments, they are missing certain information such as
# root author and root permlink.
for comment_op in fetch_user_comment_ops(username):
op_date = parse_date(comment_op['timestamp'])
if op_date > end_date:
continue
if op_date < start_date:
break
if op_date != current_date:
current_date = op_date
print_with_timestamp(f'Searching history for {current_date}')
post_comments = fetch_comment_op_thread(comment_op, comment_keys,
username, only_authored)
num_posts += 1
num_comments += len(post_comments)
yield format_comment_rows(post_comments)
print_with_timestamp(
f'Fetched {num_posts} posts with {num_comments} comments')
def fetch_comment_op_thread(comment_op, comment_keys, username, only_authored):
# Skip comments we've already fetched
op_key = get_key(comment_op)
if op_key in comment_keys:
return []
post_comments = []
# We can't fetch deleted comments from the api, so try to use the parent
# to fill in the information missing from the operation
comment = fetch_comment(op_key)
if comment_is_not_found(comment):
parent_comment = fetch_comment(get_parent_key(comment_op))
if comment_is_not_found(parent_comment):
print_with_timestamp('Could not find \'{}\', skipping...'.format(get_link_string(*op_key)))
return []
post_comments.append(make_comment_from_parent(comment_op, parent_comment))
print_with_timestamp('Comment \'{}\' was deleted but was able to fill in information from parent'.format(get_link_string(*op_key)))
comment = parent_comment # Now that the deleted comment was added, find the root comment from the parent
if only_authored and comment['root_author'] != username:
return []
# If this isn't the root comment, fetch it
comment_key = get_key(comment)
root_key = get_root_key(comment)
root_comment = comment if root_key == comment_key else fetch_comment(root_key)
with timer('Fetching post \'{}\''.format(get_link_string(*root_key))):
post_comments.extend(fetch_thread(root_comment, comment_keys))
return post_comments
def insert_comments(rows):
result = db.insert_or_ignore(db.comment, rows)
num_inserted = result.rowcount
num_skipped = len(rows) - num_inserted
message = f'Inserted {num_inserted} rows into "{db.comment.name}"'
if num_skipped:
message += f' ({num_skipped} skipped)'
print_with_timestamp(message)
def insert_comments(rows):
result = db.insert_or_ignore(db.comment, rows)
num_inserted = result.rowcount
num_skipped = len(rows) - num_inserted
message = 'Inserted {} rows into \'{}\''.format(num_inserted, db.comment.name)
if num_skipped:
message += ' ({} skipped)'.format(num_skipped)
print_with_timestamp(message)