Create a new 4CAT user and send them a registration e-mail
"""
import argparse
import psycopg2
import time
import sys
import re
import os
sys.path.insert(0, os.path.abspath(os.path.dirname(__file__)) + "/..")
from backend.lib.database import Database
from backend.lib.logger import Logger
from webtool.lib.user import User
log = Logger()
db = Database(logger=log, appname="create-user")
cli = argparse.ArgumentParser()
cli.add_argument("-u", "--username", required=True, help="Name of user (must be unique)")
args = cli.parse_args()
if __name__ != "__main__":
sys.exit(1)
if not re.match(r"[^@]+\@.*?\.[a-zA-Z]+", args.username):
print("Please provide an e-mail address as username.")
sys.exit(1)
try:
type=str,
required=True,
help="Name of SQLite table containing threads.")
cli.add_argument("-p",
"--posts_table",
type=str,
required=True,
help="Name of the SQLite table containing posts.")
cli.add_argument("-b", "--board", type=str, required=True, help="Board name")
args = cli.parse_args()
if not Path(args.input).exists() or not Path(args.input).is_file():
print("%s is not a valid folder name." % args.input)
sys.exit(1)
logger = Logger()
db = Database(logger=logger, appname="queue-dump")
seen_post_ids = set()
# Columns from 4archive dumps
posts_columns = ["id", "chan_id", "threads_id", "chan_image_name", "image_size", \
"image_dimensions", "thumb_dimensions", "image_url", "original_image_name", "subject", \
"name", "chan_user_id", "tripcode", "capcode", "chan_post_date", "body", "available"]
threads_columns = ["id", "thread_id", "board", "archive_date", "update_date", "user_ips", \
"times_updated", "views", "admin_note", "secret", "available", "alive", "takedown_reason", \
"busy", "tweeted"]
conn = sqlite3.connect(args.input)
print("Connected to SQLite database.")
default=True)
args = cli.parse_args()
args.truncate = bool(args.truncate)
limit = int(args.limit)
sourcefile = Path(args.input)
if not sourcefile.exists():
print("The file %s does not exist" % sourcefile)
exit(1)
dbconn = sqlite3.connect(args.input)
dbconn.row_factory = sqlite3.Row
cursor = dbconn.cursor()
db = Database(logger=Logger())
db.execute(open("database.sql").read())
if args.truncate:
db.execute("TRUNCATE posts_usenet")
db.execute("TRUNCATE threads_usenet")
db.execute("TRUNCATE groups_usenet")
db.commit()
post_to_threads = {}
posts = cursor.execute("SELECT * FROM postsdata")
print("Loading posts....")
done = 0
while posts:
post = posts.fetchone()
if not post or (limit and done > limit):
def run(as_daemon=True):
if not as_daemon:
indent_spaces = round(shutil.get_terminal_size().columns / 2) - 33
indent = "".join([" " for i in range(0, indent_spaces)
]) if indent_spaces > 0 else ""
print("\n\n")
print(
indent +
"+---------------------------------------------------------------+"
)
print(
indent +
"| |"
)
print(
indent +
"| welcome to |"
)
print(
indent +
"| |"
)
print(
indent +
"| j88D .o88b. .d8b. d888888b |"
)
print(
indent +
"| j8~88 d8P Y8 d8' `8b `~~88~~' |"
)
print(
indent +
"| j8' 88 8P 88ooo88 88 |"
)
print(
indent +
"| V88888D 8b 88~~~88 88 |"
)
print(
indent +
"| 88 Y8b d8 88 88 88 |"
)
print(
indent +
"| VP `Y88P' YP YP YP |"
)
print(
indent +
"| |"
)
print(
indent +
"| 4CAT: Capture and Analysis Toolkit |"
)
print(
indent +
"| |"
)
print(
indent +
"| |"
)
print(
indent +
"+---------------------------------------------------------------+"
)
print(
indent +
"| press q + enter to shut down |"
)
print(
indent +
"| |"
)
print(
indent +
"| WARNING: Not running as a daemon. Quitting this process will |"
)
print(
indent +
"| shut down the backend as well. |"
)
print(
indent +
"+---------------------------------------------------------------+\n\n"
)
# load everything
log = Logger(output=not as_daemon)
db = Database(logger=log, appname="main")
queue = JobQueue(logger=log, database=db)
# clean up after ourselves
db.commit()
queue.release_all()
# make it happen
WorkerManager(logger=log, database=db, queue=queue, as_daemon=as_daemon)
log.info("4CAT Backend shut down.")
# this should have been done in the 1.9 -> 1.10 migration script, but alas...
from backend.lib.database import Database
from backend.lib.logger import Logger
import psycopg2
import config
log = Logger(output=True)
db = Database(logger=log, dbname=config.DB_NAME, user=config.DB_USER, password=config.DB_PASSWORD, host=config.DB_HOST, port=config.DB_PORT, appname="4cat-migrate")
for datasource in ("4chan", "8kun", "8chan"):
print(" Checking for %s database tables... " % datasource, end="")
test = db.fetchone("SELECT EXISTS ( SELECT FROM information_schema.tables WHERE table_schema = %s AND table_name = %s )", ("public", "posts_%s" % datasource))
if not test["exists"]:
print("not available, nothing to upgrade!")
continue
print(" Checking if required columns exist... ", end="")
columns = [row["column_name"] for row in db.fetchall("SELECT column_name FROM information_schema.columns WHERE table_name = %s", ("posts_%s" % datasource,))]
if "image_url" in columns:
print("yes!")
else:
print(" adding 'image_url' column to %s posts table" % datasource)
db.execute("ALTER TABLE posts_%s ADD COLUMN image_url TEXT DEFAULT NONE" % datasource)
cli.add_argument(
"-f",
"--fast",
default=False,
type=bool,
help=
"Use batch queries instead of inserting posts individually. This is far faster than 'slow' mode, "
"but will crash if trying to insert a duplicate post, so it should only be used on an empty "
"database or when you're sure datasets don't overlap.")
args = cli.parse_args()
if not os.path.exists(args.input):
print("File not found: %s" % args.input)
sys.exit(1)
db = Database(logger=Logger(), appname="4chan-import")
print("Opening %s." % args.input)
if args.skip > 0:
print("Skipping %i posts." % args.skip)
if args.fast:
print("Fast mode enabled.")
with open(args.input, encoding="utf-8") as inputfile:
postscsv = csv.DictReader(inputfile,
fieldnames=FourPlebs.columns,
dialect=FourPlebs)
postbuffer = []
threads = {}