from common.lib.database import Database
from common.lib.logger import Logger
import psycopg2
import config
log = Logger(output=True)
db = Database(logger=log,
dbname=config.DB_NAME,
user=config.DB_USER,
password=config.DB_PASSWORD,
host=config.DB_HOST,
port=config.DB_PORT,
appname="4cat-migrate")
print(" Checking for 4chan database tables... ", end="")
try:
test = db.fetchone("SELECT * FROM posts_4chan LIMIT 1")
except psycopg2.ProgrammingError:
print("not available, nothing to upgrade!")
exit(0)
print(" Checking if required columns exist... ", end="")
columns = [
row["column_name"] for row in db.fetchall(
"SELECT column_name FROM information_schema.columns WHERE table_name = %s",
("posts_4chan", ))
]
if "board" in columns:
print("yes!")
else:
default=True)
args = cli.parse_args()
args.truncate = bool(args.truncate)
limit = int(args.limit)
sourcefile = Path(args.input)
if not sourcefile.exists():
print("The file %s does not exist" % sourcefile)
exit(1)
dbconn = sqlite3.connect(args.input)
dbconn.row_factory = sqlite3.Row
cursor = dbconn.cursor()
db = Database(logger=Logger())
db.execute(open("database.sql").read())
if args.truncate:
db.execute("TRUNCATE posts_usenet")
db.execute("TRUNCATE threads_usenet")
db.execute("TRUNCATE groups_usenet")
db.commit()
post_to_threads = {}
posts = cursor.execute("SELECT * FROM postsdata")
print("Loading posts....")
done = 0
while posts:
post = posts.fetchone()
if not post or (limit and done > limit):
type=str,
required=True,
help="Name of SQLite table containing threads.")
cli.add_argument("-p",
"--posts_table",
type=str,
required=True,
help="Name of the SQLite table containing posts.")
cli.add_argument("-b", "--board", type=str, required=True, help="Board name")
args = cli.parse_args()
if not Path(args.input).exists() or not Path(args.input).is_file():
print("%s is not a valid folder name." % args.input)
sys.exit(1)
logger = Logger()
db = Database(logger=logger, appname="queue-dump")
seen_post_ids = set()
# Columns from 4archive dumps
posts_columns = ["id", "chan_id", "threads_id", "chan_image_name", "image_size", \
"image_dimensions", "thumb_dimensions", "image_url", "original_image_name", "subject", \
"name", "chan_user_id", "tripcode", "capcode", "chan_post_date", "body", "available"]
threads_columns = ["id", "thread_id", "board", "archive_date", "update_date", "user_ips", \
"times_updated", "views", "admin_note", "secret", "available", "alive", "takedown_reason", \
"busy", "tweeted"]
conn = sqlite3.connect(args.input)
print("Connected to SQLite database.")
from flask_login import LoginManager
from flask_limiter import Limiter
from flask_limiter.util import get_remote_address
import config
from common.lib.database import Database
from common.lib.logger import Logger
from common.lib.queue import JobQueue
database_name = config.DB_NAME_TEST if hasattr(
config.FlaskConfig,
"DEBUG") and config.FlaskConfig.DEBUG == "Test" else config.DB_NAME
login_manager = LoginManager()
app = Flask(__name__)
log = Logger()
db = Database(logger=log, dbname=database_name, appname="frontend")
queue = JobQueue(logger=log, database=db)
# initialize openapi endpoint collector for later specification generation
from webtool.lib.openapi_collector import OpenAPICollector
openapi = OpenAPICollector(app)
# initialize rate limiter
limiter = Limiter(app, key_func=get_remote_address)
# make sure a secret key was set in the config file, for secure session cookies
if config.FlaskConfig.SECRET_KEY == "REPLACE_THIS":
raise Exception(
"You need to set a FLASK_SECRET in config.py before running the web tool."
)
def run(as_daemon=True):
if not as_daemon:
indent_spaces = round(shutil.get_terminal_size().columns / 2) - 33
indent = "".join([" " for i in range(0, indent_spaces)
]) if indent_spaces > 0 else ""
print("\n\n")
print(
indent +
"+---------------------------------------------------------------+"
)
print(
indent +
"| |"
)
print(
indent +
"| welcome to |"
)
print(
indent +
"| |"
)
print(
indent +
"| j88D .o88b. .d8b. d888888b |"
)
print(
indent +
"| j8~88 d8P Y8 d8' `8b `~~88~~' |"
)
print(
indent +
"| j8' 88 8P 88ooo88 88 |"
)
print(
indent +
"| V88888D 8b 88~~~88 88 |"
)
print(
indent +
"| 88 Y8b d8 88 88 88 |"
)
print(
indent +
"| VP `Y88P' YP YP YP |"
)
print(
indent +
"| |"
)
print(
indent +
"| 4CAT: Capture and Analysis Toolkit |"
)
print(
indent +
"| |"
)
print(
indent +
"| |"
)
print(
indent +
"+---------------------------------------------------------------+"
)
print(
indent +
"| press q + enter to shut down |"
)
print(
indent +
"| |"
)
print(
indent +
"| WARNING: Not running as a daemon. Quitting this process will |"
)
print(
indent +
"| shut down the backend as well. |"
)
print(
indent +
"+---------------------------------------------------------------+\n\n"
)
# load everything
log = Logger(output=not as_daemon)
db = Database(logger=log, appname="main")
queue = JobQueue(logger=log, database=db)
# clean up after ourselves
db.commit()
queue.release_all()
# make it happen
WorkerManager(logger=log, database=db, queue=queue, as_daemon=as_daemon)
log.info("4CAT Backend shut down.")
cli.add_argument(
"-f",
"--fast",
default=False,
type=bool,
help=
"Use batch queries instead of inserting posts individually. This is far faster than 'slow' mode, "
"but will crash if trying to insert a duplicate post, so it should only be used on an empty "
"database or when you're sure datasets don't overlap.")
args = cli.parse_args()
if not os.path.exists(args.input):
print("File not found: %s" % args.input)
sys.exit(1)
db = Database(logger=Logger(), appname="4chan-import")
print("Opening %s." % args.input)
if args.skip > 0:
print("Skipping %i posts." % args.skip)
if args.fast:
print("Fast mode enabled.")
with open(args.input, encoding="utf-8") as inputfile:
postscsv = csv.DictReader(inputfile,
fieldnames=FourPlebs.columns,
dialect=FourPlebs)
postbuffer = []
threads = {}