def create_app(config="CTFd.config.Config"):
app = CTFdFlask(__name__)
with app.app_context():
app.config.from_object(config)
app.theme_loader = ThemeLoader(os.path.join(app.root_path, "themes"),
followlinks=True)
# Weird nested solution for accessing plugin templates
app.plugin_loader = jinja2.PrefixLoader({
"plugins":
jinja2.FileSystemLoader(searchpath=os.path.join(
app.root_path, "plugins"),
followlinks=True)
})
# Load from themes first but fallback to loading from the plugin folder
app.jinja_loader = jinja2.ChoiceLoader(
[app.theme_loader, app.plugin_loader])
from CTFd.models import ( # noqa: F401
db, Teams, Solves, Challenges, Fails, Flags, Tags, Files, Tracking,
)
url = create_database()
# This allows any changes to the SQLALCHEMY_DATABASE_URI to get pushed back in
# This is mostly so we can force MySQL's charset
app.config["SQLALCHEMY_DATABASE_URI"] = str(url)
# Register database
db.init_app(app)
# Register Flask-Migrate
migrations.init_app(app, db)
# Alembic sqlite support is lacking so we should just create_all anyway
if url.drivername.startswith("sqlite"):
# Enable foreign keys for SQLite. This must be before the
# db.create_all call because tests use the in-memory SQLite
# database (each connection, including db creation, is a new db).
# https://docs.sqlalchemy.org/en/13/dialects/sqlite.html#foreign-key-support
from sqlalchemy.engine import Engine
from sqlalchemy import event
@event.listens_for(Engine, "connect")
def set_sqlite_pragma(dbapi_connection, connection_record):
cursor = dbapi_connection.cursor()
cursor.execute("PRAGMA foreign_keys=ON")
cursor.close()
db.create_all()
stamp_latest_revision()
else:
# This creates tables instead of db.create_all()
# Allows migrations to happen properly
upgrade()
from CTFd.models import ma
ma.init_app(app)
app.db = db
app.VERSION = __version__
app.CHANNEL = __channel__
from CTFd.cache import cache
cache.init_app(app)
app.cache = cache
reverse_proxy = app.config.get("REVERSE_PROXY")
if reverse_proxy:
if type(reverse_proxy) is str and "," in reverse_proxy:
proxyfix_args = [int(i) for i in reverse_proxy.split(",")]
app.wsgi_app = ProxyFix(app.wsgi_app, *proxyfix_args)
else:
app.wsgi_app = ProxyFix(app.wsgi_app,
x_for=1,
x_proto=1,
x_host=1,
x_port=1,
x_prefix=1)
version = utils.get_config("ctf_version")
# Upgrading from an older version of CTFd
if version and (StrictVersion(version) < StrictVersion(__version__)):
if confirm_upgrade():
run_upgrade()
else:
exit()
if not version:
utils.set_config("ctf_version", __version__)
if not utils.get_config("ctf_theme"):
utils.set_config("ctf_theme", "core")
update_check(force=True)
init_request_processors(app)
init_template_filters(app)
init_template_globals(app)
# Importing here allows tests to use sensible names (e.g. api instead of api_bp)
from CTFd.views import views
from CTFd.teams import teams
from CTFd.users import users
from CTFd.challenges import challenges
from CTFd.scoreboard import scoreboard
from CTFd.auth import auth
from CTFd.admin import admin
from CTFd.api import api
from CTFd.events import events
from CTFd.errors import page_not_found, forbidden, general_error, gateway_error
app.register_blueprint(views)
app.register_blueprint(teams)
app.register_blueprint(users)
app.register_blueprint(challenges)
app.register_blueprint(scoreboard)
app.register_blueprint(auth)
app.register_blueprint(api)
app.register_blueprint(events)
app.register_blueprint(admin)
app.register_error_handler(404, page_not_found)
app.register_error_handler(403, forbidden)
app.register_error_handler(500, general_error)
app.register_error_handler(502, gateway_error)
init_logs(app)
init_events(app)
init_plugins(app)
return app
def import_ctf(backup, erase=True):
if not zipfile.is_zipfile(backup):
raise zipfile.BadZipfile
backup = zipfile.ZipFile(backup)
members = backup.namelist()
max_content_length = get_app_config("MAX_CONTENT_LENGTH")
for f in members:
if f.startswith("/") or ".." in f:
# Abort on malicious zip files
raise zipfile.BadZipfile
info = backup.getinfo(f)
if max_content_length:
if info.file_size > max_content_length:
raise zipfile.LargeZipFile
# Get list of directories in zipfile
member_dirs = [os.path.split(m)[0] for m in members if "/" in m]
if "db" not in member_dirs:
raise Exception(
'CTFd couldn\'t find the "db" folder in this backup. '
"The backup may be malformed or corrupted and the import process cannot continue."
)
try:
alembic_version = json.loads(
backup.open("db/alembic_version.json").read())
alembic_version = alembic_version["results"][0]["version_num"]
except Exception:
raise Exception(
"Could not determine appropriate database version. This backup cannot be automatically imported."
)
# Check if the alembic version is from CTFd 1.x
if alembic_version in (
"1ec4a28fe0ff",
"2539d8b5082e",
"7e9efd084c5a",
"87733981ca0e",
"a4e30c94c360",
"c12d2a1b0926",
"c7225db614c1",
"cb3cfcc47e2f",
"cbf5620f8e15",
"d5a224bf5862",
"d6514ec92738",
"dab615389702",
"e62fd69bd417",
):
raise Exception(
"The version of CTFd that this backup is from is too old to be automatically imported."
)
if erase:
# Clear out existing connections to release any locks
db.session.close()
db.engine.dispose()
# Drop database and recreate it to get to a clean state
if drop_database():
create_database()
else:
truncate_database()
create_database()
# We explicitly do not want to upgrade or stamp here.
# The import will have this information.
side_db = dataset.connect(get_app_config("SQLALCHEMY_DATABASE_URI"))
sqlite = get_app_config("SQLALCHEMY_DATABASE_URI").startswith("sqlite")
postgres = get_app_config("SQLALCHEMY_DATABASE_URI").startswith("postgres")
try:
if postgres:
side_db.query("SET session_replication_role=replica;")
else:
side_db.query("SET FOREIGN_KEY_CHECKS=0;")
except Exception:
print("Failed to disable foreign key checks. Continuing.")
first = [
"db/teams.json",
"db/users.json",
"db/challenges.json",
"db/dynamic_challenge.json",
"db/flags.json",
"db/hints.json",
"db/unlocks.json",
"db/awards.json",
"db/tags.json",
"db/submissions.json",
"db/solves.json",
"db/files.json",
"db/notifications.json",
"db/pages.json",
"db/tracking.json",
"db/config.json",
]
# We want to insert certain database tables first so we are specifying
# the order with a list. The leftover tables are tables that are from a
# plugin (more likely) or a table where we do not care about insertion order
for item in first:
if item in members:
members.remove(item)
# Upgrade the database to the point in time that the import was taken from
migration_upgrade(revision=alembic_version)
members.remove("db/alembic_version.json")
# Combine the database insertion code into a function so that we can pause
# insertion between official database tables and plugin tables
def insertion(table_filenames):
for member in table_filenames:
if member.startswith("db/"):
table_name = member[3:-5]
try:
# Try to open a file but skip if it doesn't exist.
data = backup.open(member).read()
except KeyError:
continue
if data:
table = side_db[table_name]
saved = json.loads(data)
for entry in saved["results"]:
# This is a hack to get SQLite to properly accept datetime values from dataset
# See Issue #246
if sqlite:
direct_table = get_class_by_tablename(table.name)
for k, v in entry.items():
if isinstance(v, string_types):
# We only want to apply this hack to columns that are expecting a datetime object
try:
is_dt_column = (type(
getattr(
direct_table,
k).type) == sqltypes.DateTime)
except AttributeError:
is_dt_column = False
# If the table is expecting a datetime, we should check if the string is one and convert it
if is_dt_column:
match = re.match(
r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d",
v,
)
if match:
entry[
k] = datetime.datetime.strptime(
v, "%Y-%m-%dT%H:%M:%S.%f")
continue
match = re.match(
r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}",
v)
if match:
entry[
k] = datetime.datetime.strptime(
v, "%Y-%m-%dT%H:%M:%S")
continue
# From v2.0.0 to v2.1.0 requirements could have been a string or JSON because of a SQLAlchemy issue
# This is a hack to ensure we can still accept older exports. See #867
if member in (
"db/challenges.json",
"db/hints.json",
"db/awards.json",
):
requirements = entry.get("requirements")
if requirements and isinstance(
requirements, string_types):
entry["requirements"] = json.loads(
requirements)
try:
table.insert(entry)
except ProgrammingError:
# MariaDB does not like JSON objects and prefers strings because it internally
# represents JSON with LONGTEXT.
# See Issue #973
requirements = entry.get("requirements")
if requirements and isinstance(requirements, dict):
entry["requirements"] = json.dumps(
requirements)
table.insert(entry)
db.session.commit()
if postgres:
# This command is to set the next primary key ID for the re-inserted tables in Postgres. However,
# this command is very difficult to translate into SQLAlchemy code. Because Postgres is not
# officially supported, no major work will go into this functionality.
# https://stackoverflow.com/a/37972960
if '"' not in table_name and "'" not in table_name:
query = "SELECT setval(pg_get_serial_sequence('{table_name}', 'id'), coalesce(max(id)+1,1), false) FROM \"{table_name}\"".format( # nosec
table_name=table_name)
side_db.engine.execute(query)
else:
raise Exception(
"Table name {table_name} contains quotes".
format(table_name=table_name))
# Insert data from official tables
insertion(first)
# Create tables created by plugins
try:
# Run plugin migrations
plugins = get_plugin_names()
try:
for plugin in plugins:
revision = plugin_current(plugin_name=plugin)
plugin_upgrade(plugin_name=plugin, revision=revision)
finally:
# Create tables that don't have migrations
app.db.create_all()
except OperationalError as e:
if not postgres:
raise e
else:
print("Allowing error during app.db.create_all() due to Postgres")
# Insert data for plugin tables
insertion(members)
# Bring plugin tables up to head revision
plugins = get_plugin_names()
for plugin in plugins:
plugin_upgrade(plugin_name=plugin)
# Extracting files
files = [f for f in backup.namelist() if f.startswith("uploads/")]
uploader = get_uploader()
for f in files:
filename = f.split(os.sep, 1)
if (
len(filename) < 2 or os.path.basename(filename[1]) == ""
): # just an empty uploads directory (e.g. uploads/) or any directory
continue
filename = filename[
1] # Get the second entry in the list (the actual filename)
source = backup.open(f)
uploader.store(fileobj=source, filename=filename)
# Alembic sqlite support is lacking so we should just create_all anyway
try:
migration_upgrade(revision="head")
except (OperationalError, CommandError, RuntimeError, SystemExit,
Exception):
app.db.create_all()
stamp_latest_revision()
try:
if postgres:
side_db.query("SET session_replication_role=DEFAULT;")
else:
side_db.query("SET FOREIGN_KEY_CHECKS=1;")
except Exception:
print("Failed to enable foreign key checks. Continuing.")
# Invalidate all cached data
cache.clear()
# Set default theme in case the current instance or the import does not provide it
set_config("ctf_theme", "core")
set_config("ctf_version", CTFD_VERSION)
def create_app(config="CTFd.config.Config"):
app = CTFdFlask(__name__)
with app.app_context():
app.config.from_object(config)
loaders = []
# We provide a `DictLoader` which may be used to override templates
app.overridden_templates = {}
loaders.append(jinja2.DictLoader(app.overridden_templates))
# A `ThemeLoader` with no `theme_name` will load from the current theme
loaders.append(ThemeLoader())
# If `THEME_FALLBACK` is set and true, we add another loader which will
# load from the `DEFAULT_THEME` - this mirrors the order implemented by
# `config.ctf_theme_candidates()`
if bool(app.config.get("THEME_FALLBACK")):
loaders.append(ThemeLoader(theme_name=DEFAULT_THEME))
# All themes including admin can be accessed by prefixing their name
prefix_loader_dict = {ADMIN_THEME: ThemeLoader(theme_name=ADMIN_THEME)}
for theme_name in CTFd.utils.config.get_themes():
prefix_loader_dict[theme_name] = ThemeLoader(theme_name=theme_name)
loaders.append(jinja2.PrefixLoader(prefix_loader_dict))
# Plugin templates are also accessed via prefix but we just point a
# normal `FileSystemLoader` at the plugin tree rather than validating
# each plugin here (that happens later in `init_plugins()`). We
# deliberately don't add this to `prefix_loader_dict` defined above
# because to do so would break template loading from a theme called
# `prefix` (even though that'd be weird).
plugin_loader = jinja2.FileSystemLoader(searchpath=os.path.join(
app.root_path, "plugins"),
followlinks=True)
loaders.append(jinja2.PrefixLoader({"plugins": plugin_loader}))
# Use a choice loader to find the first match from our list of loaders
app.jinja_loader = jinja2.ChoiceLoader(loaders)
from CTFd.models import ( # noqa: F401
db, Teams, Solves, Challenges, Fails, Flags, Tags, Files, Tracking,
)
url = create_database()
# This allows any changes to the SQLALCHEMY_DATABASE_URI to get pushed back in
# This is mostly so we can force MySQL's charset
app.config["SQLALCHEMY_DATABASE_URI"] = str(url)
# Register database
db.init_app(app)
# Register Flask-Migrate
migrations.init_app(app, db)
# Alembic sqlite support is lacking so we should just create_all anyway
if url.drivername.startswith("sqlite"):
# Enable foreign keys for SQLite. This must be before the
# db.create_all call because tests use the in-memory SQLite
# database (each connection, including db creation, is a new db).
# https://docs.sqlalchemy.org/en/13/dialects/sqlite.html#foreign-key-support
from sqlalchemy.engine import Engine
from sqlalchemy import event
@event.listens_for(Engine, "connect")
def set_sqlite_pragma(dbapi_connection, connection_record):
cursor = dbapi_connection.cursor()
cursor.execute("PRAGMA foreign_keys=ON")
cursor.close()
db.create_all()
stamp_latest_revision()
else:
# This creates tables instead of db.create_all()
# Allows migrations to happen properly
upgrade()
from CTFd.models import ma
ma.init_app(app)
app.db = db
app.VERSION = __version__
app.CHANNEL = __channel__
from CTFd.cache import cache
cache.init_app(app)
app.cache = cache
reverse_proxy = app.config.get("REVERSE_PROXY")
if reverse_proxy:
if type(reverse_proxy) is str and "," in reverse_proxy:
proxyfix_args = [int(i) for i in reverse_proxy.split(",")]
app.wsgi_app = ProxyFix(app.wsgi_app, *proxyfix_args)
else:
app.wsgi_app = ProxyFix(app.wsgi_app,
x_for=1,
x_proto=1,
x_host=1,
x_port=1,
x_prefix=1)
version = utils.get_config("ctf_version")
# Upgrading from an older version of CTFd
if version and (StrictVersion(version) < StrictVersion(__version__)):
if confirm_upgrade():
run_upgrade()
else:
exit()
if not version:
utils.set_config("ctf_version", __version__)
if not utils.get_config("ctf_theme"):
utils.set_config("ctf_theme", DEFAULT_THEME)
update_check(force=True)
init_request_processors(app)
init_template_filters(app)
init_template_globals(app)
# Importing here allows tests to use sensible names (e.g. api instead of api_bp)
from CTFd.views import views
from CTFd.teams import teams
from CTFd.users import users
from CTFd.challenges import challenges
from CTFd.scoreboard import scoreboard
from CTFd.auth import auth
from CTFd.admin import admin
from CTFd.api import api
from CTFd.events import events
from CTFd.errors import render_error
app.register_blueprint(views)
app.register_blueprint(teams)
app.register_blueprint(users)
app.register_blueprint(challenges)
app.register_blueprint(scoreboard)
app.register_blueprint(auth)
app.register_blueprint(api)
app.register_blueprint(events)
app.register_blueprint(admin)
for code in {403, 404, 500, 502}:
app.register_error_handler(code, render_error)
init_logs(app)
init_events(app)
init_plugins(app)
return app
def create_app(config="CTFd.config.Config"):
app = CTFdFlask(__name__)
with app.app_context():
app.config.from_object(config)
theme_loader = ThemeLoader(os.path.join(app.root_path, "themes"),
followlinks=True)
app.jinja_loader = theme_loader
from CTFd.models import ( # noqa: F401
db, Teams, Solves, Challenges, Fails, Flags, Tags, Files, Tracking,
)
url = create_database()
# This allows any changes to the SQLALCHEMY_DATABASE_URI to get pushed back in
# This is mostly so we can force MySQL's charset
app.config["SQLALCHEMY_DATABASE_URI"] = str(url)
# Register database
db.init_app(app)
# Register Flask-Migrate
migrations.init_app(app, db)
# Alembic sqlite support is lacking so we should just create_all anyway
if url.drivername.startswith("sqlite"):
db.create_all()
stamp_latest_revision()
else:
# This creates tables instead of db.create_all()
# Allows migrations to happen properly
upgrade()
from CTFd.models import ma
ma.init_app(app)
app.db = db
app.VERSION = __version__
from CTFd.cache import cache
cache.init_app(app)
app.cache = cache
reverse_proxy = app.config.get("REVERSE_PROXY")
if reverse_proxy:
if "," in reverse_proxy:
proxyfix_args = [int(i) for i in reverse_proxy.split(",")]
app.wsgi_app = ProxyFix(app.wsgi_app, None, *proxyfix_args)
else:
app.wsgi_app = ProxyFix(
app.wsgi_app,
num_proxies=None,
x_for=1,
x_proto=1,
x_host=1,
x_port=1,
x_prefix=1,
)
version = utils.get_config("ctf_version")
# Upgrading from an older version of CTFd
if version and (StrictVersion(version) < StrictVersion(__version__)):
if confirm_upgrade():
run_upgrade()
else:
exit()
if not version:
utils.set_config("ctf_version", __version__)
if not utils.get_config("ctf_theme"):
utils.set_config("ctf_theme", "core")
update_check(force=True)
init_request_processors(app)
init_template_filters(app)
init_template_globals(app)
# Importing here allows tests to use sensible names (e.g. api instead of api_bp)
from CTFd.views import views
from CTFd.teams import teams
from CTFd.users import users
from CTFd.challenges import challenges
from CTFd.scoreboard import scoreboard
from CTFd.auth import auth
from CTFd.admin import admin
from CTFd.api import api
from CTFd.events import events
from CTFd.errors import page_not_found, forbidden, general_error, gateway_error
app.register_blueprint(views)
app.register_blueprint(teams)
app.register_blueprint(users)
app.register_blueprint(challenges)
app.register_blueprint(scoreboard)
app.register_blueprint(auth)
app.register_blueprint(api)
app.register_blueprint(events)
app.register_blueprint(admin)
app.register_error_handler(404, page_not_found)
app.register_error_handler(403, forbidden)
app.register_error_handler(500, general_error)
app.register_error_handler(502, gateway_error)
init_logs(app)
init_events(app)
init_plugins(app)
return app
def import_ctf(backup, erase=True):
if not zipfile.is_zipfile(backup):
raise zipfile.BadZipfile
if erase:
drop_database()
create_database()
# We explicitly do not want to upgrade or stamp here.
# The import will have this information.
side_db = dataset.connect(get_app_config("SQLALCHEMY_DATABASE_URI"))
sqlite = get_app_config("SQLALCHEMY_DATABASE_URI").startswith("sqlite")
postgres = get_app_config("SQLALCHEMY_DATABASE_URI").startswith("postgres")
backup = zipfile.ZipFile(backup)
members = backup.namelist()
max_content_length = get_app_config("MAX_CONTENT_LENGTH")
for f in members:
if f.startswith("/") or ".." in f:
# Abort on malicious zip files
raise zipfile.BadZipfile
info = backup.getinfo(f)
if max_content_length:
if info.file_size > max_content_length:
raise zipfile.LargeZipFile
try:
if postgres:
side_db.query("SET session_replication_role=replica;")
else:
side_db.query("SET FOREIGN_KEY_CHECKS=0;")
except Exception:
print("Failed to disable foreign key checks. Continuing.")
first = [
"db/teams.json",
"db/users.json",
"db/challenges.json",
"db/dynamic_challenge.json",
"db/flags.json",
"db/hints.json",
"db/unlocks.json",
"db/awards.json",
"db/tags.json",
"db/submissions.json",
"db/solves.json",
"db/files.json",
"db/notifications.json",
"db/pages.json",
"db/tracking.json",
"db/config.json",
]
for item in first:
if item in members:
members.remove(item)
members = first + members
alembic_version = json.loads(
backup.open(
"db/alembic_version.json").read())["results"][0]["version_num"]
upgrade(revision=alembic_version)
# Create tables created by plugins
try:
app.db.create_all()
except OperationalError as e:
if not postgres:
raise e
else:
print("Allowing error during app.db.create_all() due to Postgres")
members.remove("db/alembic_version.json")
for member in members:
if member.startswith("db/"):
table_name = member[3:-5]
try:
# Try to open a file but skip if it doesn't exist.
data = backup.open(member).read()
except KeyError:
continue
if data:
table = side_db[table_name]
saved = json.loads(data)
for entry in saved["results"]:
# This is a hack to get SQLite to properly accept datetime values from dataset
# See Issue #246
if sqlite:
for k, v in list(entry.items()):
if isinstance(v, six.string_types):
match = re.match(
r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d",
v)
if match:
entry[k] = datetime.datetime.strptime(
v, "%Y-%m-%dT%H:%M:%S.%f")
continue
match = re.match(
r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}", v)
if match:
entry[k] = datetime.datetime.strptime(
v, "%Y-%m-%dT%H:%M:%S")
continue
# From v2.0.0 to v2.1.0 requirements could have been a string or JSON because of a SQLAlchemy issue
# This is a hack to ensure we can still accept older exports. See #867
if member in (
"db/challenges.json",
"db/hints.json",
"db/awards.json",
):
requirements = entry.get("requirements")
if requirements and isinstance(requirements,
six.string_types):
entry["requirements"] = json.loads(requirements)
try:
table.insert(entry)
except ProgrammingError:
# MariaDB does not like JSON objects and prefers strings because it internally
# represents JSON with LONGTEXT.
# See Issue #973
requirements = entry.get("requirements")
if requirements and isinstance(requirements, dict):
entry["requirements"] = json.dumps(requirements)
table.insert(entry)
db.session.commit()
if postgres:
# This command is to set the next primary key ID for the re-inserted tables in Postgres. However,
# this command is very difficult to translate into SQLAlchemy code. Because Postgres is not
# officially supported, no major work will go into this functionality.
# https://stackoverflow.com/a/37972960
if '"' not in table_name and "'" not in table_name:
query = "SELECT setval(pg_get_serial_sequence('{table_name}', 'id'), coalesce(max(id)+1,1), false) FROM \"{table_name}\"".format( # nosec
table_name=table_name)
side_db.engine.execute(query)
else:
raise Exception(
"Table name {table_name} contains quotes".format(
table_name=table_name))
# Extracting files
files = [f for f in backup.namelist() if f.startswith("uploads/")]
uploader = get_uploader()
for f in files:
filename = f.split(os.sep, 1)
if len(filename
) < 2: # just an empty uploads directory (e.g. uploads/)
continue
filename = filename[
1] # Get the second entry in the list (the actual filename)
source = backup.open(f)
uploader.store(fileobj=source, filename=filename)
# Alembic sqlite support is lacking so we should just create_all anyway
try:
upgrade(revision="head")
except (CommandError, RuntimeError, SystemExit):
app.db.create_all()
stamp_latest_revision()
try:
if postgres:
side_db.query("SET session_replication_role=DEFAULT;")
else:
side_db.query("SET FOREIGN_KEY_CHECKS=1;")
except Exception:
print("Failed to enable foreign key checks. Continuing.")
# Invalidate all cached data
cache.clear()
# Set default theme in case the current instance or the import does not provide it
set_config("ctf_theme", "core")
def import_ctf(backup, erase=True):
cache_timeout = 604800 # 604800 is 1 week in seconds
def set_error(val):
cache.set(key="import_error", value=val, timeout=cache_timeout)
print(val)
def set_status(val):
cache.set(key="import_status", value=val, timeout=cache_timeout)
print(val)
# Reset import cache keys and don't print these values
cache.set(key="import_error", value=None, timeout=cache_timeout)
cache.set(key="import_status", value=None, timeout=cache_timeout)
if not zipfile.is_zipfile(backup):
set_error("zipfile.BadZipfile: zipfile is invalid")
raise zipfile.BadZipfile
backup = zipfile.ZipFile(backup)
members = backup.namelist()
max_content_length = get_app_config("MAX_CONTENT_LENGTH")
for f in members:
if f.startswith("/") or ".." in f:
# Abort on malicious zip files
set_error("zipfile.BadZipfile: zipfile is malicious")
raise zipfile.BadZipfile
info = backup.getinfo(f)
if max_content_length:
if info.file_size > max_content_length:
set_error("zipfile.LargeZipFile: zipfile is too large")
raise zipfile.LargeZipFile
# Get list of directories in zipfile
member_dirs = [os.path.split(m)[0] for m in members if "/" in m]
if "db" not in member_dirs:
set_error("Exception: db folder is missing")
raise Exception(
'CTFd couldn\'t find the "db" folder in this backup. '
"The backup may be malformed or corrupted and the import process cannot continue."
)
try:
alembic_version = json.loads(
backup.open("db/alembic_version.json").read())
alembic_version = alembic_version["results"][0]["version_num"]
except Exception:
set_error(
"Exception: Could not determine appropriate database version")
raise Exception(
"Could not determine appropriate database version. This backup cannot be automatically imported."
)
# Check if the alembic version is from CTFd 1.x
if alembic_version in (
"1ec4a28fe0ff",
"2539d8b5082e",
"7e9efd084c5a",
"87733981ca0e",
"a4e30c94c360",
"c12d2a1b0926",
"c7225db614c1",
"cb3cfcc47e2f",
"cbf5620f8e15",
"d5a224bf5862",
"d6514ec92738",
"dab615389702",
"e62fd69bd417",
):
set_error(
"Exception: The version of CTFd that this backup is from is too old to be automatically imported."
)
raise Exception(
"The version of CTFd that this backup is from is too old to be automatically imported."
)
start_time = unix_time(datetime.datetime.utcnow())
cache.set(key="import_start_time", value=start_time, timeout=cache_timeout)
cache.set(key="import_end_time", value=None, timeout=cache_timeout)
set_status("started")
sqlite = get_app_config("SQLALCHEMY_DATABASE_URI").startswith("sqlite")
postgres = get_app_config("SQLALCHEMY_DATABASE_URI").startswith("postgres")
mysql = get_app_config("SQLALCHEMY_DATABASE_URI").startswith("mysql")
mariadb = is_database_mariadb()
if erase:
set_status("erasing")
# Clear out existing connections to release any locks
db.session.close()
db.engine.dispose()
# Kill sleeping processes on MySQL so we don't get a metadata lock
# In my testing I didn't find that Postgres or SQLite needed the same treatment
# Only run this when not in tests as we can't isolate the queries out
# This is a very dirty hack. Don't try this at home kids.
if mysql and get_app_config("TESTING", default=False) is False:
url = make_url(get_app_config("SQLALCHEMY_DATABASE_URI"))
r = db.session.execute("SHOW PROCESSLIST")
processes = r.fetchall()
for proc in processes:
if (proc.Command == "Sleep" and proc.User == url.username
and proc.db == url.database):
proc_id = proc.Id
db.session.execute(f"KILL {proc_id}")
# Drop database and recreate it to get to a clean state
drop_database()
create_database()
# We explicitly do not want to upgrade or stamp here.
# The import will have this information.
set_status("erased")
side_db = dataset.connect(get_app_config("SQLALCHEMY_DATABASE_URI"))
try:
set_status("disabling foreign key checks")
if postgres:
side_db.query("SET session_replication_role=replica;")
else:
side_db.query("SET FOREIGN_KEY_CHECKS=0;")
except Exception:
print("Failed to disable foreign key checks. Continuing.")
first = [
"db/teams.json",
"db/users.json",
"db/challenges.json",
"db/dynamic_challenge.json",
"db/flags.json",
"db/hints.json",
"db/unlocks.json",
"db/awards.json",
"db/tags.json",
"db/topics.json",
"db/submissions.json",
"db/solves.json",
"db/files.json",
"db/notifications.json",
"db/pages.json",
"db/tracking.json",
"db/config.json",
"db/fields.json",
]
# We want to insert certain database tables first so we are specifying
# the order with a list. The leftover tables are tables that are from a
# plugin (more likely) or a table where we do not care about insertion order
for item in first:
if item in members:
members.remove(item)
# Upgrade the database to the point in time that the import was taken from
migration_upgrade(revision=alembic_version)
members.remove("db/alembic_version.json")
# Combine the database insertion code into a function so that we can pause
# insertion between official database tables and plugin tables
def insertion(table_filenames):
for member in table_filenames:
set_status(f"inserting {member}")
if member.startswith("db/"):
table_name = member[3:-5]
try:
# Try to open a file but skip if it doesn't exist.
data = backup.open(member).read()
except KeyError:
continue
if data:
table = side_db[table_name]
saved = json.loads(data)
count = len(saved["results"])
for i, entry in enumerate(saved["results"]):
set_status(f"inserting {member} {i}/{count}")
# This is a hack to get SQLite to properly accept datetime values from dataset
# See Issue #246
if sqlite:
direct_table = get_class_by_tablename(table.name)
for k, v in entry.items():
if isinstance(v, string_types):
# We only want to apply this hack to columns that are expecting a datetime object
try:
is_dt_column = (type(
getattr(
direct_table,
k).type) == sqltypes.DateTime)
except AttributeError:
is_dt_column = False
# If the table is expecting a datetime, we should check if the string is one and convert it
if is_dt_column:
match = re.match(
r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d",
v,
)
if match:
entry[
k] = datetime.datetime.strptime(
v, "%Y-%m-%dT%H:%M:%S.%f")
continue
match = re.match(
r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}",
v)
if match:
entry[
k] = datetime.datetime.strptime(
v, "%Y-%m-%dT%H:%M:%S")
continue
# From v2.0.0 to v2.1.0 requirements could have been a string or JSON because of a SQLAlchemy issue
# This is a hack to ensure we can still accept older exports. See #867
if member in (
"db/challenges.json",
"db/hints.json",
"db/awards.json",
):
requirements = entry.get("requirements")
if requirements and isinstance(
requirements, string_types):
entry["requirements"] = json.loads(
requirements)
# From v3.1.0 to v3.5.0 FieldEntries could have been varying levels of JSON'ified strings.
# For example "\"test\"" vs "test". This results in issues with importing backups between
# databases. Specifically between MySQL and MariaDB. Because CTFd standardizes against MySQL
# we need to have an edge case here.
if member == "db/field_entries.json":
value = entry.get("value")
if value:
try:
# Attempt to convert anything to its original Python value
entry["value"] = str(json.loads(value))
except (json.JSONDecodeError, TypeError):
pass
finally:
# Dump the value into JSON if its mariadb or skip the conversion if not mariadb
if mariadb:
entry["value"] = json.dumps(
entry["value"])
try:
table.insert(entry)
except ProgrammingError:
# MariaDB does not like JSON objects and prefers strings because it internally
# represents JSON with LONGTEXT.
# See Issue #973
requirements = entry.get("requirements")
if requirements and isinstance(requirements, dict):
entry["requirements"] = json.dumps(
requirements)
table.insert(entry)
db.session.commit()
if postgres:
# This command is to set the next primary key ID for the re-inserted tables in Postgres. However,
# this command is very difficult to translate into SQLAlchemy code. Because Postgres is not
# officially supported, no major work will go into this functionality.
# https://stackoverflow.com/a/37972960
if '"' not in table_name and "'" not in table_name:
query = "SELECT setval(pg_get_serial_sequence('{table_name}', 'id'), coalesce(max(id)+1,1), false) FROM \"{table_name}\"".format( # nosec
table_name=table_name)
side_db.engine.execute(query)
else:
set_error(
f"Exception: Table name {table_name} contains quotes"
)
raise Exception(
"Table name {table_name} contains quotes".
format(table_name=table_name))
# Insert data from official tables
set_status("inserting tables")
insertion(first)
# Create tables created by plugins
# Run plugin migrations
set_status("inserting plugins")
plugins = get_plugin_names()
for plugin in plugins:
set_status(f"inserting plugin {plugin}")
revision = plugin_current(plugin_name=plugin)
plugin_upgrade(plugin_name=plugin, revision=revision, lower=None)
# Insert data for plugin tables
insertion(members)
# Bring plugin tables up to head revision
plugins = get_plugin_names()
for plugin in plugins:
plugin_upgrade(plugin_name=plugin)
# Extracting files
set_status("uploading files")
files = [f for f in backup.namelist() if f.startswith("uploads/")]
uploader = get_uploader()
for f in files:
filename = f.split(os.sep, 1)
if (
len(filename) < 2 or os.path.basename(filename[1]) == ""
): # just an empty uploads directory (e.g. uploads/) or any directory
continue
filename = filename[
1] # Get the second entry in the list (the actual filename)
source = backup.open(f)
uploader.store(fileobj=source, filename=filename)
# Alembic sqlite support is lacking so we should just create_all anyway
set_status("running head migrations")
if sqlite:
app.db.create_all()
stamp_latest_revision()
else:
# Run migrations to bring to latest version
migration_upgrade(revision="head")
# Create any leftover tables, perhaps from old plugins
app.db.create_all()
try:
set_status("reenabling foreign key checks")
if postgres:
side_db.query("SET session_replication_role=DEFAULT;")
else:
side_db.query("SET FOREIGN_KEY_CHECKS=1;")
except Exception:
print("Failed to enable foreign key checks. Continuing.")
# Invalidate all cached data
set_status("clearing caches")
cache.clear()
# Set default theme in case the current instance or the import does not provide it
set_config("ctf_theme", DEFAULT_THEME)
set_config("ctf_version", CTFD_VERSION)
# Set config variables to mark import completed
cache.set(key="import_start_time", value=start_time, timeout=cache_timeout)
cache.set(
key="import_end_time",
value=unix_time(datetime.datetime.utcnow()),
timeout=cache_timeout,
)