def run_task(*args, **kwargs):
"""Function to import all repositories from input list to the DB."""
try:
products = kwargs.get("products", None)
repos = kwargs.get("repos", None)
init_logging()
init_db()
if products:
product_store = ProductStore()
product_store.store(products)
if repos:
repository_controller = RepositoryController()
# Sync repos from input
for repo_url, content_set, basearch, releasever, cert_name, ca_cert, cert, key in repos:
repository_controller.add_repository(repo_url,
content_set,
basearch,
releasever,
cert_name=cert_name,
ca_cert=ca_cert,
cert=cert,
key=key)
repository_controller.import_repositories()
except Exception as err: # pylint: disable=broad-except
msg = "Internal server error <%s>" % err.__hash__()
LOGGER.exception(msg)
DatabaseHandler.rollback()
return "ERROR"
return "OK"
def run_task(*args, **kwargs):
"""Function to import all repositories from input list to the DB."""
try:
products = kwargs.get("products", None)
repos = kwargs.get("repos", None)
git_sync = kwargs.get("git_sync", False)
init_logging()
init_db()
if products:
product_store = ProductStore()
product_store.store(products)
if repos:
repository_controller = RepositoryController()
repos_in_db = repository_controller.repo_store.list_repositories()
# Sync repos from input
for repo_url, content_set, basearch, releasever, cert_name, ca_cert, cert, key in repos:
repository_controller.add_repository(repo_url, content_set, basearch, releasever,
cert_name=cert_name, ca_cert=ca_cert,
cert=cert, key=key)
repos_in_db.pop((content_set, basearch, releasever), None)
if git_sync: # Warn about extra repos in DB when syncing main repolist from git
for content_set, basearch, releasever in repos_in_db:
LOGGER.warning("Repository in DB but not in git repolist: %s", ", ".join(
filter(None, (content_set, basearch, releasever))))
repository_controller.import_repositories()
except Exception as err: # pylint: disable=broad-except
msg = "Internal server error <%s>" % err.__hash__()
LOGGER.exception(msg)
DatabaseHandler.rollback()
return "ERROR"
finally:
DatabaseHandler.close_connection()
return "OK"
def repo_sync_task(products=None, repos=None):
"""Function to start syncing all repositories from input list or from database."""
try:
init_db()
repository_controller = RepositoryController()
if products:
product_store = ProductStore()
product_store.store(products)
# Reference imported content set to associate with repositories
repository_controller.repo_store.set_content_set_db_mapping(
product_store.cs_to_dbid)
if repos:
# Sync repos from input
for repo in repos:
repo_label, repo_url, content_set, cert_name, ca_cert, cert, key = repo
repository_controller.add_repository(repo_label,
repo_url,
content_set=content_set,
cert_name=cert_name,
ca_cert=ca_cert,
cert=cert,
key=key)
else:
# Re-sync repos in DB
repository_controller.add_synced_repositories()
repository_controller.store()
except: # pylint: disable=bare-except
LOGGER.log(traceback.format_exc())
DatabaseHandler.rollback()
return "ERROR"
return "OK"
def exporter_db_conn():
"""Fixture for db connection."""
def _handler(postgresql):
"""Init DB with data."""
conn = psycopg2.connect(**postgresql.dsn())
cursor = conn.cursor()
with open("../database/vmaas_user_create_postgresql.sql",
"r") as psql_user:
cursor.execute(psql_user.read())
with open("../database/vmaas_db_postgresql.sql", "r") as vmaas_db:
cursor.execute(vmaas_db.read())
with open("test_data/exporter/exporter_test_data.sql",
"r") as test_data:
cursor.execute(test_data.read())
cursor.close()
conn.commit()
conn.close()
# Create temporary posgresql server
# pylint: disable=invalid-name
Postgresql = testing.postgresql.PostgresqlFactory(
cache_initialized_db=True, on_initialized=_handler)
postgresql = Postgresql()
os.environ["POSTGRESQL_PORT"] = str(postgresql.dsn()["port"])
DatabaseHandler.close_connection()
init_db()
conn = psycopg2.connect(**postgresql.dsn())
yield conn
# teardown - close connection, stop postgresql
conn.close()
postgresql.stop()
def run_task(*args, **kwargs):
"""Function to start exporting disk dump."""
try:
export_pkgtree(PKGTREE_FILE)
except Exception as err: # pylint: disable=broad-except
msg = "Internal server error <%s>" % err.__hash__()
LOGGER.exception(msg)
DatabaseHandler.rollback()
return "ERROR"
return "OK"
def __init__(self, app):
"""Initialises the connection between RabbitMQ queue and Flask server,
in order to get the objects waiting in Rabbit queue and put them in
the database."""
Thread.__init__(self)
self.connection_channel = False
self.app = app
self.handle_connection()
self.database_handler = DatabaseHandler(self.app)
def run_task(*args, **kwargs):
"""Function to start exporting disk dump."""
try:
export_data(DUMP)
except Exception as err: # pylint: disable=broad-except
msg = "Internal server error <%s>" % err.__hash__()
LOGGER.exception(msg)
DatabaseHandler.rollback()
return "ERROR"
finally:
DatabaseHandler.close_connection()
return "OK"
def cve_sync_task():
"""Function to start syncing all CVEs."""
try:
init_db()
controller = CveRepoController()
controller.add_repos()
controller.store()
except: # pylint: disable=bare-except
LOGGER.log(traceback.format_exc())
DatabaseHandler.rollback()
return "ERROR"
return "OK"
def run_task(*args, **kwargs):
"""Function to start syncing all CVEs."""
try:
init_logging()
init_db()
controller = CvemapController()
controller.store()
except Exception as err: # pylint: disable=broad-except
msg = "Internal server error <%s>" % err.__hash__()
LOGGER.exception(msg)
DatabaseHandler.rollback()
return "ERROR"
return "OK"
def run_task(*args, **kwargs):
"""Function to start syncing all repositories available from database."""
try:
init_logging()
init_db()
repository_controller = RepositoryController()
repository_controller.add_db_repositories()
repository_controller.store()
except Exception as err: # pylint: disable=broad-except
msg = "Internal server error <%s>" % err.__hash__()
LOGGER.exception(msg)
DatabaseHandler.rollback()
return "ERROR"
return "OK"
def run_task(*args, **kwargs):
"""Function to start deleting repos."""
try:
repo = kwargs.get("repo", None)
init_logging()
init_db()
repository_controller = RepositoryController()
repository_controller.delete_content_set(repo)
except Exception as err: # pylint: disable=broad-except
msg = "Internal server error <%s>" % err.__hash__()
LOGGER.exception(msg)
DatabaseHandler.rollback()
return "ERROR"
return "OK"
def __init__(self):
self.logger = get_logger(__name__)
self.conn = DatabaseHandler.get_connection()
self.module_store = ModulesStore()
self.package_store = PackageStore()
self.update_store = UpdateStore()
self.content_set_to_db_id = self._prepare_content_set_map()
def run_task(*args, **kwargs):
"""Function to start syncing OVALs."""
try:
init_logging()
init_db()
controller = OvalController()
controller.store()
except Exception as err: # pylint: disable=broad-except
msg = "Internal server error <%s>" % err.__hash__()
LOGGER.exception(msg)
FAILED_IMPORT_OVAL.inc()
DatabaseHandler.rollback()
return "ERROR"
finally:
DatabaseHandler.close_connection()
return "OK"
def upgrade(self):
"""perform database upgrade"""
conn = DatabaseHandler.get_connection()
try:
self._get_db_lock(conn)
db_version = self._get_current_db_version(conn)
if db_version == self.version_max:
LOGGER.info('Database is up to date at version: %d',
db_version)
return
if db_version > self.version_max:
msg = 'Database version %d is greater than upgrade version %d' % (
db_version, self.version_max)
LOGGER.warning(msg)
return
LOGGER.info('Database requires upgrade from version %d to %d',
db_version, self.version_max)
upgrades_to_apply = self._get_upgrades_to_apply(
db_version, self.version_max)
for upgrade in upgrades_to_apply:
self._apply_upgrade(upgrade['ver'], upgrade['script'], conn)
finally:
self._release_db_lock(conn)
def main(filename):
""" Main loop."""
init_logging()
init_db()
db_instance = DatabaseHandler.get_connection()
data = JsonPkgTree(db_instance, filename)
data.dump()
def __init__(self):
LOGGER.info('DatabaseUpgrade initializing.')
DatabaseHandler.close_connection()
init_db()
# get upgrade sql scripts directory
self.scripts_dir = os.getenv('DB_UPGRADE_SCRIPTS_DIR',
DB_UPGRADES_PATH)
if not self.scripts_dir.endswith('/'):
self.scripts_dir += '/'
# load the version2file_map and version_max
self.version2file_map, self.version_max = self._load_upgrade_file_list(
self.scripts_dir)
def main(filename):
""" Main loop."""
init_logging()
init_db()
db_instance = DatabaseHandler.get_connection()
#data = DataDump(db.cursor(), filename)
data = DataDump(db_instance, filename)
data.dump()
class RabbitObjectHandler(Thread):
"""Class which handles the RabbitMQ connection."""
def __init__(self, app):
"""Initialises the connection between RabbitMQ queue and Flask server,
in order to get the objects waiting in Rabbit queue and put them in
the database."""
Thread.__init__(self)
self.connection_channel = False
self.app = app
self.handle_connection()
self.database_handler = DatabaseHandler(self.app)
def handle_connection(self):
"""Handles the connection."""
reader = ReadHandler()
connection = pika.BlockingConnection(
pika.ConnectionParameters(reader.get_rabbitmq_address(),
reader.get_rabbitmq_port()))
self.connection_channel = connection.channel()
self.connection_channel.queue_declare(queue=get_rabbit_queue())
def collect_packet(self, channel, method, properties, body):
"""Adds the packet collected from the Rabbit queue to the database."""
self.database_handler.add_packet_to_database(json.loads(body))
print(body)
def run(self):
"""Starts the thread which consumes the objects
from the RabbitMQ queue."""
self.connection_channel.basic_consume(self.collect_packet,
queue=get_rabbit_queue(),
no_ack=True)
self.connection_channel.start_consuming()
def process(self):
"""Processes the dbchange get request. """
init_db()
self.db_instance = DatabaseHandler.get_connection()
result = {}
with self.db_instance.cursor() as crs:
crs.execute("select pkgtree_change from dbchange")
timestamp = crs.fetchone()
result["pkgtree_change"] = str(timestamp[0])
return result
def __init__(self):
self.logger = get_logger(__name__)
self.conn = DatabaseHandler.get_connection()
self.cpe_label_to_id = {}
self.cpe_label_to_name = {}
cur = self.conn.cursor()
cur.execute("select id, label, name from cpe")
for cpe_id, label, name in cur.fetchall():
self.cpe_label_to_id[label] = cpe_id
self.cpe_label_to_name[label] = name
cur.close()
def init_schema(self):
"""Initialize database schema."""
cfg = Config()
conn = DatabaseHandler.get_connection()
if self._is_initialized(conn):
LOGGER.info("DB schema is already initialized.")
return
try:
self._get_db_lock(conn)
LOGGER.info("Empty database, initializing...")
with conn.cursor() as cur:
cur.execute(open(USER_CREATE_SQL_PATH, "r").read())
cur.execute(open(DB_CREATE_SQL_PATH, "r").read())
cur.execute(
f"ALTER USER vmaas_writer WITH PASSWORD '{cfg.postgresql_writer_password}'"
)
cur.execute(
f"ALTER USER vmaas_reader WITH PASSWORD '{cfg.postgresql_writer_password}'"
)
conn.commit()
finally:
self._release_db_lock(conn)
def __init__(self):
self.logger = get_logger(__name__)
self.repo = []
self.conn = DatabaseHandler.get_connection()
self.cve_store = CveStore()
def __init__(self):
self.logger = SimpleLogger()
self.repo = []
self.conn = DatabaseHandler.get_connection()
self.cve_store = CveStore()
def __init__(self):
self.logger = SimpleLogger()
self.conn = DatabaseHandler.get_connection()
def __init__(self):
self.content_set_to_db_id = {}
self.logger = SimpleLogger()
self.conn = DatabaseHandler.get_connection()
self.package_store = PackageStore()
self.update_store = UpdateStore()
def __init__(self, user_admin: User, database_handler=None):
if AccountType(user_admin.account_type) is not AccountType.Admin:
raise CalendarAdmin.NotAdminUserException()
self.user_admin = user_admin
self.database_handler: DatabaseHandler = database_handler if database_handler is not None else DatabaseHandler(
)
# check connection to database here
if not self.database_handler.check_connected():
print("Application not connected to database")
def __init__(self):
self.logger = SimpleLogger()
self.conn = DatabaseHandler.get_connection()
# Access this dictionary from repository_store to reference content set table.
self.cs_to_dbid = {}
def __init__(self):
self.logger = get_logger(__name__)
self.conn = DatabaseHandler.get_connection()
def __init__(self, user: User, database_handler=None):
# to set another user it can be accessed as calendar_user object attribute
self.user = user
self.database_handler: DatabaseHandler = database_handler if database_handler is not None else DatabaseHandler()
# check connection to database here
if not self.database_handler.check_connected():
print("Application not connected to database")
"""Handles the connection of flask and database"""
from flask import Flask
from flask import render_template
from flask import request
from threads.get_objects_from_rabbit import RabbitObjectHandler
from files.configuration_handler import CreateConfiguration
from files.read_configuration_handler import ReadHandler
from database.database_handler import DatabaseHandler
APP = Flask(__name__, template_folder="templates")
CONFIG = CreateConfiguration()
DATABASE_HANDLER = DatabaseHandler(APP)
@APP.route('/')
def main_page_route():
"""Displays the main page and the types of information
you can get the server to display."""
return render_template("main_page.html")
@APP.route('/current_supported_metrics')
def current_supported_metrics_route():
"""Displays the currently supported metrics."""
return render_template("current_supported_metrics.html",
metrics=READER.get_supported_metrics())
@APP.route('/packets')
