def complete(self, task_id, success=True):
"""Mark a task as completed.
@param task_id: task id.
@param success: completed with status.
@return: operation status.
"""
try:
task = s.query(Task).filter_by(id=task_id).first()
except SQLAlchemyError as e:
raise CuckooDatabaseError(
"Unable to find locked task, reason: %s" % e)
if task:
if success:
task.status = 2
else:
task.status = 1
try:
s.commit()
except SQLAlchemyError as e:
raise CuckooDatabaseError("Unable to complete, reason: %s" % e)
else:
return False
return True
def __init__(self, dsn=None):
"""@param dsn: database connection string."""
cfg = Config()
if dsn:
self.engine = create_engine(dsn, poolclass=NullPool)
elif cfg.database.connection:
self.engine = create_engine(cfg.database.connection, poolclass=NullPool)
else:
db_file = os.path.join(CUCKOO_ROOT, "db", "cuckoo.db")
if not os.path.exists(db_file):
db_dir = os.path.dirname(db_file)
if not os.path.exists(db_dir):
try:
create_folder(folder=db_dir)
except CuckooOperationalError as e:
raise CuckooDatabaseError("Unable to create database directory: {0}".format(e))
self.engine = create_engine("sqlite:///{0}".format(db_file), poolclass=NullPool)
# Disable SQL logging. Turn it on for debugging.
self.engine.echo = False
# Connection timeout.
if cfg.database.timeout:
self.engine.pool_timeout = cfg.database.timeout
else:
self.engine.pool_timeout = 60
# Create schema.
try:
Base.metadata.create_all(self.engine)
except SQLAlchemyError as e:
raise CuckooDatabaseError("Unable to create or connect to database: {0}".format(e))
# Get db session.
self.Session = sessionmaker(bind=self.engine)
def __init__(self, dsn=None):
"""@param dsn: database connection string."""
cfg = Config()
if dsn:
self.engine = create_engine(dsn, poolclass=NullPool)
elif cfg.database.connection:
self.engine = create_engine(cfg.database.connection,
poolclass=NullPool)
else:
db_file = os.path.join(CUCKOO_ROOT, "db", "cuckoo.db")
if not os.path.exists(db_file):
db_dir = os.path.dirname(db_file)
if not os.path.exists(db_dir):
try:
create_folder(folder=db_dir)
except CuckooOperationalError as e:
raise CuckooDatabaseError(
"Unable to create database directory: {0}".format(
e))
self.engine = create_engine("sqlite:///{0}".format(db_file),
poolclass=NullPool)
# Disable SQL logging. Turn it on for debugging.
self.engine.echo = False
# Connection timeout.
if cfg.database.timeout:
self.engine.pool_timeout = cfg.database.timeout
else:
self.engine.pool_timeout = 60
# Create schema.
try:
Base.metadata.create_all(self.engine)
except SQLAlchemyError as e:
raise CuckooDatabaseError(
"Unable to create or connect to database: {0}".format(e))
# Get db session.
self.Session = sessionmaker(bind=self.engine)
# Set database schema version.
# TODO: it's a little bit dirty, needs refactoring.
tmp_session = self.Session()
if not tmp_session.query(AlembicVersion).count():
tmp_session.add(AlembicVersion(version_num=SCHEMA_VERSION))
try:
tmp_session.commit()
except SQLAlchemyError as e:
raise CuckooDatabaseError(
"Unable to set schema version: {0}".format(e))
tmp_session.rollback()
finally:
tmp_session.close()
else:
tmp_session.close()
def drop(self):
"""Drop all tables."""
try:
Base.metadata.drop_all(self.engine)
except SQLAlchemyError as e:
raise CuckooDatabaseError(
"Unable to create or connect to database: {0}".format(e))
def generate(self):
"""Create database.
@return: operation status.
"""
if os.path.exists(self.db_file):
return False
db_dir = os.path.dirname(self.db_file)
if not os.path.exists(db_dir):
try:
create_folder(folder=db_dir)
except CuckooOperationalError as e:
raise CuckooDatabaseError(
"Unable to create database directory: %s" % e)
conn = sqlite3.connect(self.db_file)
cursor = conn.cursor()
try:
cursor.execute("CREATE TABLE tasks (\n" \
" id INTEGER PRIMARY KEY,\n" \
" md5 TEXT DEFAULT NULL,\n" \
" file_path TEXT NOT NULL,\n" \
" timeout INTEGER DEFAULT NULL,\n" \
" priority INTEGER DEFAULT 0,\n" \
" custom TEXT DEFAULT NULL,\n" \
" machine TEXT DEFAULT NULL,\n" \
" package TEXT DEFAULT NULL,\n" \
" options TEXT DEFAULT NULL,\n" \
" platform TEXT DEFAULT NULL,\n" \
" added_on DATE DEFAULT CURRENT_TIMESTAMP,\n" \
" completed_on DATE DEFAULT NULL,\n" \
" lock INTEGER DEFAULT 0,\n" \
# Status possible values:
# 0 = not completed
# 1 = error occurred
# 2 = completed successfully.
" status INTEGER DEFAULT 0\n" \
");")
except sqlite3.OperationalError as e:
raise CuckooDatabaseError("Unable to create database: %s" % e)
return True
def get_task(self, task_id):
try:
s = Session()
task = s.query(Analysis).filter_by(task_id=task_id).first()
return task
except SQLAlchemyError as e:
raise CuckooDatabaseError(
"Unable to get all tasks for analysis, reason: %s" % e)
def unlock(self, task_id):
"""Unlock a task.
@param task_id: task id.
@return: operation status.
"""
try:
task = s.query(Task).filter_by(id=task_id).first()
except SQLAlchemyError as e:
raise CuckooDatabaseError("Unable to find lock, reason: %s" % e)
if row:
try:
task.lock = 0
s.commit()
except SQLAlchemyError as e:
raise CuckooDatabaseError("Unable to unlock, reason: %s" % e)
else:
return False
return True
def fetch(self):
try:
s = Session()
task = s.query(Task).filter_by(lock=0, status=0).order_by(
desc(Task.priority)).first()
#print("fetching task %s" % task)
s.close()
return task
except SQLAlchemyError as e:
raise CuckooDatabaseError("Unable to fetch, reason: %s" % e)
def get_all_analysis(self):
""" Get all analysis
@return ALL analysis
"""
try:
analysis = s.query(Analysis).order_by(desc(
Analysis.status), desc(Analysis.created_on)).all()
return analysis
except SQLAlchemyError as e:
raise CuckooDatabaseError("Unable to get all analysis, reason:" %
e)
def generate(self):
"""Create database.
@return: operation status.
NOTE: you need to create database and user before generate db tables
"""
try:
Base.metadata.create_all(engine)
except SQLAlchemyError as e:
raise CuckooDatabaseError("Unable to create database: %s" % e)
return True
'''
def get_analysis(self, a_id):
""" Retrive all tasks for given analysis id
@param: a_id: analysis id
@returns: all tasks for that analysis
"""
try:
s = Session()
tasks = s.query(Task).filter_by(a_id=a_id).all()
return tasks
except SQLAlchemyError as e:
raise CuckooDatabaseError(
"Unable to get all tasks for analysis, reason: %s" % e)
def add_analysis(self, desc, exe_id):
""" Add an analysis on database
@param desc: description
@param exe_id: id of executable to test
@return: cursor or None
"""
a = Analysis(desc, exe_id)
try:
s.add(a)
s.commit()
return a.id
except SQLAlchemyError as e:
raise CuckooDatabaseError("Unable to create analysis: %s" % e)
def add_exe(self, file_path, md5):
""" Add an exe to db
@param file_path: path to file
@return: cursor or None
"""
if not file_path or not os.path.exists(file_path):
return None
row = s.query(Exe).filter_by(md5=md5).first()
if row is not None:
return row.id
try:
exe = Exe(file_path, md5)
s.add(exe)
s.commit()
return exe.id
except SQLAlchemyError as e:
raise CuckooDatabaseError("Unable to add executable, reason: %s" %
e)
def add(self,
file_path,
a_id,
md5="",
timeout=0,
package="",
options="",
priority=1,
custom="",
machine="",
platform=""):
"""Add a task to database.
@param file_path: sample path.
@param anal_id: analysis id referenced to table.
@param md5: sample MD5.
@param timeout: selected timeout.
@param options: analysis options.
@param priority: analysis priority.
@param custom: custom options.
@param machine: selected machine.
@param platform: platform
@return: cursor or None.
"""
if not file_path or not os.path.exists(file_path):
return None
if not timeout:
timeout = 0
try:
task = Task(file_path, a_id, md5, timeout, package, options,
priority, custom, machine, platform)
s.add(task)
s.commit()
return task.id
except MySQLdb.Error as e:
raise CuckooDatabaseError("Unable to add task: %s" % e)
def __init__(self, dsn=None, schema_check=True):
"""@param dsn: database connection string.
@param schema_check: disable or enable the db schema version check
"""
cfg = Config()
if dsn:
self._connect_database(dsn)
elif cfg.database.connection:
self._connect_database(cfg.database.connection)
else:
db_file = os.path.join(CUCKOO_ROOT, "db", "cuckoo.db")
if not os.path.exists(db_file):
db_dir = os.path.dirname(db_file)
if not os.path.exists(db_dir):
try:
create_folder(folder=db_dir)
except CuckooOperationalError as e:
raise CuckooDatabaseError(
"Unable to create database directory: {0}".format(
e))
self._connect_database("sqlite:///%s" % db_file)
# Disable SQL logging. Turn it on for debugging.
self.engine.echo = False
# Connection timeout.
if cfg.database.timeout:
self.engine.pool_timeout = cfg.database.timeout
else:
self.engine.pool_timeout = 60
# Create schema.
try:
Base.metadata.create_all(self.engine)
except SQLAlchemyError as e:
raise CuckooDatabaseError(
"Unable to create or connect to database: {0}".format(e))
# Get db session.
self.Session = sessionmaker(bind=self.engine)
# Deal with schema versioning.
# TODO: it's a little bit dirty, needs refactoring.
tmp_session = self.Session()
if not tmp_session.query(AlembicVersion).count():
# Set database schema version.
tmp_session.add(AlembicVersion(version_num=SCHEMA_VERSION))
try:
tmp_session.commit()
except SQLAlchemyError as e:
raise CuckooDatabaseError(
"Unable to set schema version: {0}".format(e))
tmp_session.rollback()
finally:
tmp_session.close()
else:
# Check if db version is the expected one.
last = tmp_session.query(AlembicVersion).first()
tmp_session.close()
if last.version_num != SCHEMA_VERSION and schema_check:
raise CuckooDatabaseError(
"DB schema version mismatch: found "
"{0}, expected {1}. Try to apply all "
"migrations.".format(last.version_num, SCHEMA_VERSION))
def __init__(self, dsn=None, schema_check=True, echo=False):
"""
@param dsn: database connection string.
@param schema_check: disable or enable the db schema version check.
@param echo: echo sql queries.
"""
self._lock = SuperLock()
cfg = Config()
if dsn:
self._connect_database(dsn)
elif hasattr(cfg, "database") and cfg.database.connection:
self._connect_database(cfg.database.connection)
else:
db_file = os.path.join(CUCKOO_ROOT, "db", "cuckoo.db")
if not os.path.exists(db_file):
db_dir = os.path.dirname(db_file)
if not os.path.exists(db_dir):
try:
create_folder(folder=db_dir)
except CuckooOperationalError as e:
raise CuckooDatabaseError(
"Unable to create database directory: {0}".format(
e))
self._connect_database("sqlite:///%s" % db_file)
# Disable SQL logging. Turn it on for debugging.
self.engine.echo = echo
# Connection timeout.
if hasattr(cfg, "database") and cfg.database.timeout:
self.engine.pool_timeout = cfg.database.timeout
else:
self.engine.pool_timeout = 60
# Let's emit a warning just in case.
if not hasattr(cfg, "database"):
log.warning("It appears you don't have a valid `database` "
"section in conf/cuckoo.conf, using sqlite3 instead.")
# Create schema.
try:
Base.metadata.create_all(self.engine)
except SQLAlchemyError as e:
raise CuckooDatabaseError(
"Unable to create or connect to database: {0}".format(e))
# Get db session.
self.Session = sessionmaker(bind=self.engine)
# Deal with schema versioning.
# TODO: it's a little bit dirty, needs refactoring.
tmp_session = self.Session()
if not tmp_session.query(AlembicVersion).count():
# Set database schema version.
tmp_session.add(AlembicVersion(version_num=SCHEMA_VERSION))
try:
tmp_session.commit()
except SQLAlchemyError as e:
raise CuckooDatabaseError(
"Unable to set schema version: {0}".format(e))
tmp_session.rollback()
finally:
tmp_session.close()
else:
# Check if db version is the expected one.
last = tmp_session.query(AlembicVersion).first()
tmp_session.close()
if last.version_num != SCHEMA_VERSION and schema_check:
raise CuckooDatabaseError(
"DB schema version mismatch: found {0}, expected {1}. "
"Try to apply all migrations (cd utils/db_migration/ && "
"alembic upgrade head).".format(last.version_num,
SCHEMA_VERSION))
def __init__(self, dsn=None, schema_check=True):
"""@param dsn: database connection string.
@param schema_check: disable or enable the db schema version check
"""
self._lock = SuperLock()
self.cfg = Config()
if dsn:
self._connect_database(dsn)
elif self.cfg.database.connection:
self._connect_database(self.cfg.database.connection)
else:
db_file = os.path.join(CUCKOO_ROOT, "db", "cuckoo.db")
if not os.path.exists(db_file):
db_dir = os.path.dirname(db_file)
if not os.path.exists(db_dir):
try:
create_folder(folder=db_dir)
except CuckooOperationalError as e:
raise CuckooDatabaseError(
"Unable to create database directory: {0}".format(
e))
self._connect_database("sqlite:///%s" % db_file)
# Disable SQL logging. Turn it on for debugging.
self.engine.echo = False
# Connection timeout.
if self.cfg.database.timeout:
self.engine.pool_timeout = self.cfg.database.timeout
else:
self.engine.pool_timeout = 60
# Create schema.
try:
Base.metadata.create_all(self.engine)
except SQLAlchemyError as e:
raise CuckooDatabaseError(
"Unable to create or connect to database: {0}".format(e))
# Get db session.
self.Session = sessionmaker(bind=self.engine)
@event.listens_for(self.Session, 'after_flush')
def delete_tag_orphans(session, ctx):
session.query(Tag).filter(~Tag.tasks.any()).filter(
~Tag.machines.any()).delete(synchronize_session=False)
# Deal with schema versioning.
# TODO: it's a little bit dirty, needs refactoring.
tmp_session = self.Session()
if not tmp_session.query(AlembicVersion).count():
# Set database schema version.
tmp_session.add(AlembicVersion(version_num=SCHEMA_VERSION))
try:
tmp_session.commit()
except SQLAlchemyError as e:
tmp_session.rollback()
raise CuckooDatabaseError(
"Unable to set schema version: {0}".format(e))
finally:
tmp_session.close()
else:
# Check if db version is the expected one.
last = tmp_session.query(AlembicVersion).first()
tmp_session.close()
if last.version_num != SCHEMA_VERSION and schema_check:
raise CuckooDatabaseError(
"DB schema version mismatch: found {0}, expected {1}. "
"Try to apply all migrations (cd utils/db_migration/ && "
"alembic upgrade head).".format(last.version_num,
SCHEMA_VERSION))