def __enter__(self):
''' called when entering via a context manager '''
Storage._LOCK.acquire()
self.database = AbstractDatabase(self.DATABASE_FILE,
commitOnClose=True)
self.database.open()
self.windowsSymbolStore = WindowsSymbolStore(WINDOWS_SYMBOL_STORE)
self._setupStorage()
return self
def test_ensure_commit_on_close_works(self):
''' ensure commitOnClose works '''
TEST_DB_FILE = os.path.join(os.path.dirname(__file__),
'test_db.sqlite')
if os.path.isfile(TEST_DB_FILE):
os.remove(TEST_DB_FILE)
self.database = AbstractDatabase(TEST_DB_FILE, commitOnClose=False)
self.database.open()
assert self.database.createTable('MyTable',
[Column("TextColumn1", "TEXT")])
assert self.database.tableExists('MyTable')
self.database.close()
# now the table shouldn't exist since we did not commit
self.database = AbstractDatabase(TEST_DB_FILE, commitOnClose=True)
self.database.open()
assert not self.database.tableExists('MyTable')
assert self.database.createTable('MyTable',
[Column("TextColumn1", "TEXT")])
self.database.close()
# now the table should exist since we did commit
self.database = AbstractDatabase(TEST_DB_FILE)
self.database.open()
assert self.database.tableExists('MyTable')
self.database.addRow('MyTable', {'TextColumn1': 'MyText'})
assert len(
self.database.execute("SELECT * FROM MyTable").fetchall()) == 1
self.database.commitOnClose = False
self.database.close()
# Now make sure that row is gone
self.database = AbstractDatabase(TEST_DB_FILE)
self.database.open()
assert self.database.tableExists('MyTable')
assert len(
self.database.execute("SELECT * FROM MyTable").fetchall()) == 0
self.database.close()
os.remove(TEST_DB_FILE)
def test_from_cursor():
''' ensures we can create a table from a cursor '''
with AbstractDatabase() as db:
assert db.createTable('OurTable', [Column("Column1", 'TEXT')])
assert db.addRow('OurTable', {'Column1': 'MyValue'})
cursor = db.execute("SELECT * FROM OurTable")
assert cursor
t = HtmlTable.fromCursor(cursor)
html = t.__html__()
assert html.count('MyValue') == 1
assert html.count('Column1') == 1
class Storage(object):
''' object that keeps track of the various storage needed by this object '''
DATABASE_FILE = os.path.join(ROOT_STORAGE_LOCATION, 'database.sqlite')
_LOCK = threading.Lock()
def __enter__(self):
''' called when entering via a context manager '''
Storage._LOCK.acquire()
self.database = AbstractDatabase(self.DATABASE_FILE,
commitOnClose=True)
self.database.open()
self.windowsSymbolStore = WindowsSymbolStore(WINDOWS_SYMBOL_STORE)
self._setupStorage()
return self
def __exit__(self, type, value, traceback):
''' called when exiting via a context manager '''
self.database.close()
Storage._LOCK.release()
def _setupStorage(self):
''' called to ensure the environment is ready for storage, etc. '''
# Ensure our storage folder exists
try:
os.makedirs(ROOT_STORAGE_LOCATION)
except:
pass
# ensure we have required table(s)
for requiredTablesName, requiredTableColumns in REQUIRED_TABLES.items(
):
if not self.database.tableExists(requiredTablesName):
assert self.database.createTable(requiredTablesName, [])
if not self.database.ensureTableHasAtLeastTheseColumns(
requiredTablesName, requiredTableColumns):
raise RuntimeError(
"Unable to ensure we have needed table: %s" %
requiredTablesName)
for tableRow in self.database.execute(
"SELECT * FROM Applications").fetchall():
if not self.database.tableExists(tableRow.ApplicationTable):
assert self.database.createTable(tableRow.ApplicationTable, [])
assert self.database.ensureTableHasAtLeastTheseColumns(
tableRow.ApplicationTable, APPLICATION_UPLOADS_COLUMNS)
def applicationExists(self, name):
''' called to check if an application exists in our tables '''
return bool(
self.database.execute(
'SELECT * FROM Applications WHERE Name="%s"' %
name).fetchone())
def applicationAdd(self, name):
''' called to add a table for this application '''
applicationTableName = getUniqueTableName()
if not self.applicationExists(name) and self.database.createTable(
applicationTableName, APPLICATION_UPLOADS_COLUMNS):
if not self.database.addRow(
'Applications', {
'Name': name,
'ApplicationTable': applicationTableName
}):
logger.error("failed to add application row: %s" % name)
return False
return True
else:
logger.error(
"failed to create a table for: %s with a table name of %s" %
(name, applicationTableName))
return False
def getApplicationTableName(self, applicationName):
''' gets an application's table name '''
result = self.database.execute(
"SELECT * FROM Applications WHERE Name=\"%s\"" %
applicationName).fetchone()
if result:
return result.ApplicationTable
return False
def getApplicationNameFromTable(self, tableName):
''' gets the name of an app from the table name '''
result = self.database.execute(
"SELECT * FROM Applications WHERE ApplicationTable=\"%s\"" %
tableName).fetchone()
if result:
return result.Name
return False
def getApplicationCell(self, applicationName, rowUid, column):
''' finds the application database, then goes to a specific row and returns the given column '''
tableName = self.getApplicationTableName(applicationName)
if not tableName:
logger.warning("Application doesn't exist")
return False
result = self.database.execute("SELECT * FROM `%s` WHERE UID=\"%s\"" %
(tableName, rowUid)).fetchone()
if not result:
logger.warning("UID didn't exist: %s" % rowUid)
return False
if not hasattr(result, column):
logger.warning("Column %s doesn't exist" % column)
return False
return getattr(result, column)
def setApplicationCell(self, applicationName, rowUid, column, value):
''' finds the application database, then goes to a specific row and modifies the given column to the given value '''
tableName = self.getApplicationTableName(applicationName)
if not tableName:
logger.warning("Application doesn't exist")
return False
result = self.database.execute(
"UPDATE `%s` SET `%s` = ? WHERE UID = ?" % (tableName, column),
[value, rowUid])
if not result:
logger.warning("Unable to update cell with row UID: %s" % rowUid)
return False
return True
def getApplicationTable(self, applicationName):
''' used to get back a view of the given application's table '''
tableName = self.getApplicationTableName(applicationName)
if not tableName:
logger.warning(
"User requested application (%s) which doesn't have a matching table"
% applicationName)
flask.abort(404)
cursor = self.database.execute("SELECT * FROM %s" % tableName)
table = _html.HtmlTable.fromCursor(cursor,
classes='content',
name=applicationName)
table.addColumn('Actions')
def getLinks(row):
''' helper to get links to files in the current table '''
rowUid = table.getCellFromRow(row, 'UID')
for columnName in 'SymbolsFile', 'ExecutableFile', 'CrashDumpFile':
url = flask.url_for("getFile",
applicationName=applicationName,
rowUid=rowUid,
column=columnName)
index = table.tableHeaders.index(columnName)
cellValue = self.getApplicationCell(applicationName, rowUid,
columnName + "Name")
if cellValue:
row[index] = _html.getHtmlLinkString(url, cellValue)
actionRowIdx = table.tableHeaders.index('Actions')
row[actionRowIdx] = _html.getDropLeft(
'...', [('Show Analysis',
flask.url_for('getAnalysis',
applicationName=applicationName,
rowUid=rowUid,
useCache=True)),
('Show Analysis (No Cache)',
flask.url_for('getAnalysis',
applicationName=applicationName,
rowUid=rowUid,
useCache=False))])
return row
table.modifyAllRows(getLinks)
table.removeColumns([
'SymbolsFileName', 'ExecutableFileName', 'CrashDumpFileName',
'CrashDumpAnalysis'
])
return table
def getAnalysis(self, applicationName, rowUid, useCache=True):
''' internal function called to get the Analysis object for the given rowUid '''
if useCache:
dataBlob = self.getApplicationCell(applicationName, rowUid,
'CrashDumpAnalysis')
if dataBlob:
logger.debug("Returning from cache, analysis: %s / %s" %
(applicationName, rowUid))
try:
return pickle.loads(dataBlob)
except Exception as ex:
logger.error("Failed to de-serialize pickle data: %s" %
str(ex))
logger.info("Attempting to generate Analysis for: %s / %s" %
(applicationName, rowUid))
crashDumpBinary = self.getApplicationCell(applicationName, rowUid,
'CrashDumpFile')
if not crashDumpBinary:
logger.error(
"Crash dump file was not available with the given uid")
flask.abort(404)
operatingSystem = self.getApplicationCell(applicationName, rowUid,
'OperatingSystem')
if not operatingSystem:
logger.error(
"Operating system was not available with the given uid... this should not be possible!"
)
flask.abort(404)
with temporaryFilePath() as crashDumpBinaryFilePath:
with open(crashDumpBinaryFilePath, 'wb') as f:
f.write(crashDumpBinary)
if operatingSystem == SupportedOperatingSystems.WINDOWS.value:
debugger = WinDbg(crashDumpBinaryFilePath,
WINDOWS_SYMBOL_STORE)
else:
logger.error("Unsupported OS is somehow in the database")
logger.abort(500)
analysis = debugger.getAnalysis()
analysisAsPickledData = pickle.dumps(analysis)
if not self.setApplicationCell(applicationName, rowUid,
'CrashDumpAnalysis',
analysisAsPickledData):
logger.warning(
"Failed to save off crash dump analysis pickle data.. uid=%s" %
rowUid)
flask.abort(500)
return analysis
def getWindowsSymbolFilePath(self, path):
''' internal function used to serve back a Windows Symbol Store path '''
fullPath = os.path.abspath(os.path.join(WINDOWS_SYMBOL_STORE, path))
# the right side of the and is to make sure that somehow we aren't out of the symbols directory
if os.path.isfile(fullPath) and os.path.normpath(
WINDOWS_SYMBOL_STORE) in os.path.normpath(fullPath):
return fullPath
flask.abort(404)
def addFromAddRequest(self, request):
''' called by the flask app to add something for the given request to addHandler
Note that this will return the status that will be returned by addHandler() '''
def failAnd401(msg):
''' helper to at this moment and log to the logger with the given message '''
logger.error(msg)
flask.abort(flask.Response(msg, 401))
# verify valid request 1st.
# need to have the operating system set
operatingSystem = request.form.get("OperatingSystem")
if not operatingSystem:
failAnd401("request was missing required field: OperatingSystem")
if not SupportedOperatingSystems.isValidValue(operatingSystem):
failAnd401(
"request's OperatingSystem is not supported: %s. Valid options: %s"
%
(operatingSystem, str(SupportedOperatingSystems.getValues())))
# need to have the application set
application = request.form.get("Application")
if not application:
failAnd401("request was missing required field: Application")
# need at least one of: SymbolsFile, ExecutableFile, CrashDumpFile
symbolsFile = request.files.get("SymbolsFile")
executableFile = request.files.get("ExecutableFile")
crashDumpFile = request.files.get("CrashDumpFile")
if not (symbolsFile or executableFile or crashDumpFile):
failAnd401(
"request needed to include at least one of the following: SymbolsFile, ExecutableFile, CrashDumpFile"
)
# if we made it here, the request is valid
# ensure we have a table for this application
if not self.applicationExists(application):
if not self.applicationAdd(application):
failAnd401("unable to add application with name: %s" %
application)
applicationTableName = self.getApplicationTableName(application)
# get binary data
symbolsFileBinary = symbolsFile.read() if symbolsFile else None
executableFileBinary = executableFile.read(
) if executableFile else None
crashDumpFileBinary = crashDumpFile.read() if crashDumpFile else None
# get binary file names
symbolsFileName = symbolsFile.filename if symbolsFile else None
executableFileName = executableFile.filename if executableFile else None
crashDumpFileName = crashDumpFile.filename if crashDumpFile else None
# add objects to symbol store
if operatingSystem == SupportedOperatingSystems.WINDOWS.value:
if symbolsFileBinary:
# additions must have original name (to work in symbol store)
with temporaryFilePath(fileName=symbolsFileName) as temp:
with open(temp, 'wb') as f:
f.write(symbolsFileBinary)
try:
self.windowsSymbolStore.add(temp, compressed=True)
except Exception as ex:
failAnd401("Failed to add symbols file to store: %s" %
str(ex))
if executableFileBinary:
# additions must have original name (to work in symbol store)
with temporaryFilePath(fileName=executableFileName) as temp:
with open(temp, 'wb') as f:
f.write(executableFileBinary)
try:
self.windowsSymbolStore.add(temp, compressed=True)
except Exception as ex:
failAnd401(
"Failed to add executable file to store: %s" %
str(ex))
# add to database
uid = getUniqueId()
if not self.database.addRow(
applicationTableName,
{
'UID': uid,
'Timestamp': str(datetime.datetime.now()),
'UploaderIP': request.remote_addr,
'OperatingSystem': operatingSystem,
'Tag': request.form.get('Tag'),
'ApplicationVersion': request.form.get('ApplicationVersion'),
'SymbolsFile': symbolsFileBinary,
'SymbolsFileName': symbolsFileName,
'ExecutableFile': executableFileBinary,
'ExecutableFileName': executableFileName,
'CrashDumpFile': crashDumpFileBinary,
'CrashDumpFileName': crashDumpFileName,
# CrashDumpAnalysis is not given here, it can be generated later.
}):
failAnd401("Unable to add to database")
# success!
return "Successfully added! UID: %s" % uid
def setUp(self):
''' called at the start of each test case '''
logger.info("starting test case: %s" % self.id())
self.database = AbstractDatabase(':memory:')
self.database.open()
def test_database_as_contextmanager(self):
''' makes sure the context manager usage works '''
with AbstractDatabase(':memory:') as m:
assert m.createTable('Table2', [])
assert m.tableExists('Table2')
class TestAbstractDatabase(unittest.TestCase):
''' quick unit tests for the abstract database class '''
def setUp(self):
''' called at the start of each test case '''
logger.info("starting test case: %s" % self.id())
self.database = AbstractDatabase(':memory:')
self.database.open()
def tearDown(self):
''' called at the end of each test case '''
self.database.close()
logger.info("ending test case: %s" % self.id())
def test_create_table(self):
''' ensure we can create tables '''
assert not self.database.tableExists('MyTable')
assert self.database.createTable('MyTable',
[Column("ColumnName", "TEXT")])
assert not self.database.createTable('MyTable',
[Column("ColumnName", "TEXT")])
assert self.database.tableExists('MyTable')
assert not self.database.tableExists('MyTable2')
def test_add_row(self):
''' ensure we can add rows to a given table '''
assert not self.database.addRow('MyTable',
{"ColumnName": "ColumnData"})
assert self.database.createTable('MyTable',
[Column("ColumnName", "TEXT")])
assert self.database.addRow('MyTable', {"ColumnName": "ColumnData"})
assert self.database.execute(
'SELECT * FROM MYTABLE').fetchall()[0].ColumnName == 'ColumnData'
def test_ensure_has_columns(self):
''' ensure we can add columns if needed '''
assert self.database.createTable('MyTable',
[Column("TextColumn1", "TEXT")])
assert self.database.ensureTableHasAtLeastTheseColumns(
'MyTable', [
Column('TextColumn1', "TEXT"),
Column('TextColumn2', "INT"),
Column('TextColumn3', "TEXT"),
])
assert self.database.addRow('MyTable', {
'TextColumn1': 'Hello',
'TextColumn2': 123,
})
lastInsert = self.database.execute(
'SELECT * from MYTABLE').fetchall()[-1]
assert lastInsert.TextColumn2 == 123
assert lastInsert.TextColumn1 == 'Hello'
assert not self.database.ensureTableHasAtLeastTheseColumns(
'MyTable',
[
Column('TextColumn1', "TEXT"),
Column('TextColumn2', "INT"),
Column('TextColumn3', "INT"), # wrong type!
])
def test_ensure_commit_on_close_works(self):
''' ensure commitOnClose works '''
TEST_DB_FILE = os.path.join(os.path.dirname(__file__),
'test_db.sqlite')
if os.path.isfile(TEST_DB_FILE):
os.remove(TEST_DB_FILE)
self.database = AbstractDatabase(TEST_DB_FILE, commitOnClose=False)
self.database.open()
assert self.database.createTable('MyTable',
[Column("TextColumn1", "TEXT")])
assert self.database.tableExists('MyTable')
self.database.close()
# now the table shouldn't exist since we did not commit
self.database = AbstractDatabase(TEST_DB_FILE, commitOnClose=True)
self.database.open()
assert not self.database.tableExists('MyTable')
assert self.database.createTable('MyTable',
[Column("TextColumn1", "TEXT")])
self.database.close()
# now the table should exist since we did commit
self.database = AbstractDatabase(TEST_DB_FILE)
self.database.open()
assert self.database.tableExists('MyTable')
self.database.addRow('MyTable', {'TextColumn1': 'MyText'})
assert len(
self.database.execute("SELECT * FROM MyTable").fetchall()) == 1
self.database.commitOnClose = False
self.database.close()
# Now make sure that row is gone
self.database = AbstractDatabase(TEST_DB_FILE)
self.database.open()
assert self.database.tableExists('MyTable')
assert len(
self.database.execute("SELECT * FROM MyTable").fetchall()) == 0
self.database.close()
os.remove(TEST_DB_FILE)
def test_database_as_contextmanager(self):
''' makes sure the context manager usage works '''
with AbstractDatabase(':memory:') as m:
assert m.createTable('Table2', [])
assert m.tableExists('Table2')
def test_no_sql_chaining(self):
''' ensures we can't chain together sql commands '''
self.database.open()
with pytest.raises(SqlStatementUnsafeException):
self.database.execute('SELECT * FROM Apps;DROP TABLE *')