#!/usr/bin/python3 import logging.handlers import sys import signal import time import traceback import discord_logging import argparse log = discord_logging.init_logging(backup_count=80) import counters from database import Database import praw_wrapper import messages import comments import notifications import utils import static from praw_wrapper import PushshiftType database = None def signal_handler(signal, frame): log.info("Handling interrupt") database.close() discord_logging.flush_discord() sys.exit(0)
import discord_logging import requests import time import json import praw import sys from datetime import datetime, timezone, timedelta log = discord_logging.init_logging() from database import Comment, User, Submission, Database database = Database() url = "https://api.pushshift.io/reddit/comment/search?limit=1000&sort=desc&subreddit=Marriage&before=" start_time = datetime.utcnow() # datetime.strptime("21-11-19 15:34:07", '%y-%m-%d %H:%M:%S') end_time = datetime.strptime("22-08-20 01:43:35", '%y-%m-%d %H:%M:%S') # start_time - timedelta(days=300) count = 0 previous_epoch = int(start_time.replace(tzinfo=timezone.utc).timestamp()) - 1 while True: new_url = url + str(previous_epoch) json_text = requests.get(new_url, headers={'User-Agent': "Post downloader by /u/Watchful1"}) time.sleep(1) # pushshift has a rate limit, if we send requests too fast it will start returning error messages try: json_data = json_text.json() except json.decoder.JSONDecodeError: time.sleep(1) continue if 'data' not in json_data:
import traceback import sqlite3 import re from datetime import datetime from datetime import timedelta import discord_logging ### Config ### LOG_FOLDER_NAME = "logs" SUBREDDIT = "ListOfSubreddits" USER_AGENT = "ListOfSubreddits helper (by /u/Watchful1)" LOOP_TIME = 60 * 60 DATABASE_NAME = "database.db" LIMIT = 50000 log = discord_logging.init_logging(debug=True) dbConn = sqlite3.connect(DATABASE_NAME) c = dbConn.cursor() c.execute(''' CREATE TABLE IF NOT EXISTS subreddits ( ID INTEGER PRIMARY KEY AUTOINCREMENT, Subreddit VARCHAR(80) NOT NULL, CheckedDate TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, Subscribers INTEGER DEFAULT 0, UNIQUE (Subreddit) ) ''') dbConn.commit()