def setup_class(self) -> None:
#pylint: disable=W0201
self.appInfo = AppInfo()
settingsFilenameOrig = os.path.join(self.appInfo.path, "misc/setup/situationboard_default.conf")
settingsFilename = os.path.join(self.appInfo.path, ".temp/situationboard.conf")
databaseFilename = os.path.join(self.appInfo.path, ".temp/situationboard.sqlite")
shutil.copy(settingsFilenameOrig, settingsFilename)
self.database = Database(databaseFilename, reset = True)
settings = Settings(settingsFilename, self.appInfo.path)
settings.setFrontendHeader("header")
settings.setFrontendNews("news")
settings.setBoolean(Settings.SECTION_BACKEND, "web_api", True)
displayPowerManager = DisplayPowerManager(settings)
self.webSocket = WebSocket(self.appInfo, settings, self.database)
pluginManager = PluginManager(settings, self.database, self.webSocket, displayPowerManager)
self.webSocket.init(pluginManager)
self.appClient = self.webSocket.app_test_client()
self.socketClient = self.webSocket.socket_test_client(self.appClient)
def main():
# init
db_handler = Database()
scraper = Scraper()
annotator_instance = Annotator()
evaluator = Evaluator()
# 1: scrape
base_urls = db_handler.read("SELECT url FROM articles.source;")
base_urls = [base_url[0] for base_url in base_urls]
for base_url in base_urls:
if "seekingalpha.com" in base_url:
scraper.scrape(db_handler, base_url, True)
else:
scraper.scrape(db_handler, base_url)
scraper.quit_driver()
# 2: annotate
articles = db_handler.read(
"SELECT content FROM articles.article WHERE url LIKE '%fool.com%';")
articles = [article[0] for article in articles]
# run either
# - manual (either)
# -- internal annotation
# annotator_instance.manual_annotation(articles[:50], "auto_annotated")
# -- csv export
annotator_instance.unannotated_csv_export(articles[:50], "auto_annotated")
# - automatic
# annotator_instance.automatic_annotation(articles, "auto_annotated")
# 3: evaluate
model_file = "auto_annotated"
data = None
with open("./backend/data/" + model_file + ".csv", "r") as csvfile:
data = list(csv.reader(csvfile))
data = [tuple((datum[0], datum[1])) for datum in data]
classifier = NaiveBayesClassifier(data)
for i in range(40, 50):
evaluator.evaluate("article_" + str(i), classifier)
def test_csv(self) -> None:
appInfo = AppInfo()
dbFilename = os.path.join(appInfo.path, ".temp/situationboard.sqlite")
inFilename = os.path.join(appInfo.path, "docs/dummy.csv")
outFilename = os.path.join(appInfo.path, ".temp/dummy.csv")
csvFilename = os.path.join(appInfo.path, ".temp/corner.csv")
###### TEST 1 (round trip) ######
# create a new (empty) database
d = Database(dbFilename, reset = True)
assert(d.getEventCount(textOnly = False) == 0)
# import data from CSV
i = CSVImporter(d)
result = i.importEvents(inFilename)
assert(result == 0)
assert(d.getEventCount(textOnly = False) > 0)
# export data to CSV
e = CSVExporter(d)
result = e.exportEvents(outFilename)
assert(result == 0)
# compare CSV files
assert(filecmp.cmp(inFilename, outFilename, shallow=False))
# close database
d.close()
###### TEST 2 (corner cases) ######
# create a new (empty) database
d = Database(dbFilename, reset = True)
assert(d.getEventCount(textOnly = False) == 0)
# add new event with pathologic content
ts = datetime.datetime.now().strftime(AlarmEvent.TIMESTAMP_FORMAT)
content = "Test\n;\\Test\nTest\"äöüß\"\"äöüß'äöüß''äöüß\näöüß"
aeAdded = AlarmEvent()
aeAdded.timestamp = ts
aeAdded.event = content
aeAdded.eventDetails = content
aeAdded.location = content
aeAdded.locationDetails = content
aeAdded.comment = content
aeAdded.alarmTimestamp = ts
aeAdded.locationLatitude = 1.12
aeAdded.locationLongitude = -13.2
eventID = d.addEvent(aeAdded)
# export data to CSV
e = CSVExporter(d)
result = e.exportEvents(csvFilename)
assert(result == 0)
# close database
d.close()
# create a new (empty) database
d = Database(dbFilename, reset = True)
# import data from CSV
i = CSVImporter(d)
result = i.importEvents(csvFilename)
assert(result == 0)
assert(d.getEventCount(textOnly = False) > 0)
# load event
aeLoaded = d.getEvent(eventID)
assert(aeLoaded is not None)
# compare event data
assert(aeAdded.comment == aeLoaded.comment)
# close database
d.close()
def test_database(self) -> None:
appInfo = AppInfo()
databaseFilename = os.path.join(appInfo.path,
".temp/situationboard.sqlite")
maxLastEvents = 10
# check that creating a database works and yields an empty database
db = Database(databaseFilename, reset=True)
assert (db.getEventCount(textOnly=False) == 0)
assert (db.getEventCount(textOnly=True) == 0)
assert (db.getEvent(1302) is None)
assert (len(db.getEvents(textOnly=False)) == 0)
assert (len(db.getEvents(textOnly=True)) == 0)
assert (len(db.getLastEvents(maxLastEvents, textOnly=False)) == 0)
assert (len(db.getLastEvents(maxLastEvents, textOnly=True)) == 0)
# check for valid stats
assert (db.getEventStats(DatabaseTimespan.TOTAL, textOnly=False) == 0)
assert (db.getEventStats(DatabaseTimespan.YEAR, textOnly=False) == 0)
assert (db.getEventStats(DatabaseTimespan.MONTH, textOnly=False) == 0)
assert (db.getEventStats(DatabaseTimespan.TODAY, textOnly=False) == 0)
assert (db.getEventStats(DatabaseTimespan.TOTAL, textOnly=True) == 0)
assert (db.getEventStats(DatabaseTimespan.YEAR, textOnly=True) == 0)
assert (db.getEventStats(DatabaseTimespan.MONTH, textOnly=True) == 0)
assert (db.getEventStats(DatabaseTimespan.TODAY, textOnly=True) == 0)
# check that inserting a binary event succeeds
newEvent1 = self.__createBinaryEvent()
newEventTemp1 = copy.deepcopy(newEvent1)
assert (db.updateEvent(newEventTemp1) != 0)
eventID1 = db.addEvent(newEventTemp1)
assert (eventID1 != AlarmEvent.NO_ID)
assert (newEventTemp1.eventID == eventID1)
assert (db.getEventCount(textOnly=False) == 1)
assert (db.getEventCount(textOnly=True) == 0)
assert (len(db.getEvents(textOnly=False)) == 1)
assert (len(db.getEvents(textOnly=True)) == 0)
assert (len(db.getLastEvents(maxLastEvents, textOnly=False)) == 1)
assert (len(db.getLastEvents(maxLastEvents, textOnly=True)) == 0)
assert (db.getEvent(eventID1) is not None)
# check for valid stats
assert (db.getEventStats(DatabaseTimespan.TOTAL, textOnly=False) == 1)
assert (db.getEventStats(DatabaseTimespan.YEAR, textOnly=False) == 1)
assert (db.getEventStats(DatabaseTimespan.MONTH, textOnly=False) == 1)
assert (db.getEventStats(DatabaseTimespan.TODAY, textOnly=False) == 1)
assert (db.getEventStats(DatabaseTimespan.TOTAL, textOnly=True) == 0)
assert (db.getEventStats(DatabaseTimespan.YEAR, textOnly=True) == 0)
assert (db.getEventStats(DatabaseTimespan.MONTH, textOnly=True) == 0)
assert (db.getEventStats(DatabaseTimespan.TODAY, textOnly=True) == 0)
# check that inserting a text event succeeds
newEvent2 = self.__createTextEvent()
newEvent2Temp = copy.deepcopy(newEvent2)
eventID2 = db.addEvent(newEvent2Temp)
assert (eventID2 != AlarmEvent.NO_ID)
assert (newEvent2Temp.eventID == eventID2)
assert (db.getEventCount(textOnly=False) == 2)
assert (db.getEventCount(textOnly=True) == 1)
assert (len(db.getEvents(textOnly=False)) == 2)
assert (len(db.getEvents(textOnly=True)) == 1)
assert (len(db.getLastEvents(maxLastEvents, textOnly=False)) == 2)
assert (len(db.getLastEvents(maxLastEvents, textOnly=True)) == 1)
assert (db.getEvent(eventID2) is not None)
# check for valid stats
assert (db.getEventStats(DatabaseTimespan.TOTAL, textOnly=False) == 2)
assert (db.getEventStats(DatabaseTimespan.YEAR, textOnly=False) == 2)
assert (db.getEventStats(DatabaseTimespan.MONTH, textOnly=False) == 2)
assert (db.getEventStats(DatabaseTimespan.TODAY, textOnly=False) == 2)
assert (db.getEventStats(DatabaseTimespan.TOTAL, textOnly=True) == 1)
assert (db.getEventStats(DatabaseTimespan.YEAR, textOnly=True) == 1)
assert (db.getEventStats(DatabaseTimespan.MONTH, textOnly=True) == 1)
assert (db.getEventStats(DatabaseTimespan.TODAY, textOnly=True) == 1)
db.commitAndClose()
# check that reloading a database succeeds and yields existing data
dbr = Database(databaseFilename, reset=False)
assert (dbr.getEventCount(textOnly=False) == 2)
assert (dbr.getEventCount(textOnly=True) == 1)
assert (len(dbr.getEvents(textOnly=False)) == 2)
assert (len(dbr.getEvents(textOnly=True)) == 1)
assert (len(dbr.getLastEvents(maxLastEvents, textOnly=False)) == 2)
assert (len(dbr.getLastEvents(maxLastEvents, textOnly=True)) == 1)
# check that retrieving an event succeeds after reload succeeds and yields the correct data
event1 = dbr.getEvent(eventID1)
assert (event1 is not None)
assert (isinstance(event1, AlarmEvent))
assert (event1.eventID == eventID1)
self.__compareEvents(event1, newEvent1)
# check that retrieving an event succeeds after reload succeeds and yields the correct data
event2 = dbr.getEvent(eventID2)
assert (event2 is not None)
assert (isinstance(event2, AlarmEvent))
assert (event2.eventID == eventID2)
self.__compareEvents(event2, newEvent2)
# check that updating an event succeeds
newEvent2Updated = copy.deepcopy(newEvent2Temp)
self.__updateTextEvent(newEvent2Updated)
newEvent2UpdatedTemp = copy.deepcopy(newEvent2Updated)
assert (dbr.updateEvent(newEvent2UpdatedTemp) == 0)
event2Updated = dbr.getEvent(eventID2)
assert (event2Updated is not None)
assert (isinstance(event2Updated, AlarmEvent))
assert (event2Updated.eventID == eventID2)
self.__compareEvents(event2Updated, newEvent2Updated)
# check that deleting an event by ID succeeds
assert (dbr.removeEventID(eventID1) == 0)
assert (dbr.getEventCount(textOnly=False) == 1)
assert (dbr.getEventCount(textOnly=True) == 1)
assert (dbr.getEvent(eventID1) is None)
assert (len(dbr.getEvents(textOnly=False)) == 1)
assert (len(dbr.getEvents(textOnly=True)) == 1)
assert (len(dbr.getLastEvents(maxLastEvents, textOnly=False)) == 1)
assert (len(dbr.getLastEvents(maxLastEvents, textOnly=True)) == 1)
# check that deleting an event succeeds
assert (dbr.removeEvent(newEvent2Updated) == 0)
assert (dbr.getEventCount(textOnly=False) == 0)
assert (dbr.getEventCount(textOnly=True) == 0)
assert (dbr.getEvent(eventID2) is None)
assert (len(dbr.getEvents(textOnly=False)) == 0)
assert (len(dbr.getEvents(textOnly=True)) == 0)
assert (len(dbr.getLastEvents(maxLastEvents, textOnly=False)) == 0)
assert (len(dbr.getLastEvents(maxLastEvents, textOnly=True)) == 0)
# check that deleting non-existent events fails
assert (dbr.removeEvent(event1) != 0)
assert (dbr.removeEventID(eventID2) != 0)
dbr.commitAndClose()
def test_handle_event(self) -> None:
#pylint: disable=W0201
appInfo = AppInfo()
settingsFilenameOrig = os.path.join(
appInfo.path, "misc/setup/situationboard_default.conf")
settingsFilename = os.path.join(appInfo.path,
".temp/situationboard.conf")
databaseFilename = os.path.join(appInfo.path,
".temp/situationboard.sqlite")
shutil.copy(settingsFilenameOrig, settingsFilename)
settings = Settings(settingsFilename, appInfo.path)
maxLastEvents = 10
displayPowerManager = DisplayPowerManager(settings)
db = Database(databaseFilename, reset=True)
webSocket = WebSocket(appInfo, settings, db)
pluginManager = PluginManager(settings, db, webSocket,
displayPowerManager)
webSocket.init(pluginManager)
# check that creating a database works and yields an empty database
assert (db.getEventCount(textOnly=False) == 0)
assert (db.getEventCount(textOnly=True) == 0)
assert (db.getEvent(1302) is None)
assert (len(db.getEvents(textOnly=False)) == 0)
assert (len(db.getEvents(textOnly=True)) == 0)
assert (len(db.getLastEvents(maxLastEvents, textOnly=False)) == 0)
assert (len(db.getLastEvents(maxLastEvents, textOnly=True)) == 0)
# check for valid stats
assert (db.getEventStats(DatabaseTimespan.TOTAL, textOnly=False) == 0)
assert (db.getEventStats(DatabaseTimespan.YEAR, textOnly=False) == 0)
assert (db.getEventStats(DatabaseTimespan.MONTH, textOnly=False) == 0)
assert (db.getEventStats(DatabaseTimespan.TODAY, textOnly=False) == 0)
assert (db.getEventStats(DatabaseTimespan.TOTAL, textOnly=True) == 0)
assert (db.getEventStats(DatabaseTimespan.YEAR, textOnly=True) == 0)
assert (db.getEventStats(DatabaseTimespan.MONTH, textOnly=True) == 0)
assert (db.getEventStats(DatabaseTimespan.TODAY, textOnly=True) == 0)
action = ActionUpdateDatabase("", settings, db, webSocket)
# check that inserting an alarm event succeeds
newEvent = self.__createEvent()
action.handleEvent(newEvent)
assert (not newEvent.noID)
assert (db.getEventCount(textOnly=False) == 1)
assert (db.getEventCount(textOnly=True) == 1)
assert (len(db.getEvents(textOnly=False)) == 1)
assert (len(db.getEvents(textOnly=True)) == 1)
assert (len(db.getLastEvents(maxLastEvents, textOnly=False)) == 1)
assert (len(db.getLastEvents(maxLastEvents, textOnly=True)) == 1)
newEventID = newEvent.eventID
assert (db.getEvent(newEventID) is not None)
# check for valid stats
assert (db.getEventStats(DatabaseTimespan.TOTAL, textOnly=False) == 1)
assert (db.getEventStats(DatabaseTimespan.YEAR, textOnly=False) == 1)
assert (db.getEventStats(DatabaseTimespan.MONTH, textOnly=False) == 1)
assert (db.getEventStats(DatabaseTimespan.TODAY, textOnly=False) == 1)
assert (db.getEventStats(DatabaseTimespan.TOTAL, textOnly=True) == 1)
assert (db.getEventStats(DatabaseTimespan.YEAR, textOnly=True) == 1)
assert (db.getEventStats(DatabaseTimespan.MONTH, textOnly=True) == 1)
assert (db.getEventStats(DatabaseTimespan.TODAY, textOnly=True) == 1)
# check that updating an alarm event succeeds
self.__updateEvent(newEvent)
action.handleEvent(newEvent)
assert (newEvent.eventID == newEventID)
assert (db.getEventCount(textOnly=False) == 1)
assert (db.getEventCount(textOnly=True) == 1)
assert (len(db.getEvents(textOnly=False)) == 1)
assert (len(db.getEvents(textOnly=True)) == 1)
assert (len(db.getLastEvents(maxLastEvents, textOnly=False)) == 1)
assert (len(db.getLastEvents(maxLastEvents, textOnly=True)) == 1)
retrievedEvent = db.getEvent(newEventID)
assert (retrievedEvent is not None)
assert (retrievedEvent.comment ==
Test_ActionUpdateDatabase.UPDATED_COMMENT)
# check for valid stats
assert (db.getEventStats(DatabaseTimespan.TOTAL, textOnly=False) == 1)
assert (db.getEventStats(DatabaseTimespan.YEAR, textOnly=False) == 1)
assert (db.getEventStats(DatabaseTimespan.MONTH, textOnly=False) == 1)
assert (db.getEventStats(DatabaseTimespan.TODAY, textOnly=False) == 1)
assert (db.getEventStats(DatabaseTimespan.TOTAL, textOnly=True) == 1)
assert (db.getEventStats(DatabaseTimespan.YEAR, textOnly=True) == 1)
assert (db.getEventStats(DatabaseTimespan.MONTH, textOnly=True) == 1)
assert (db.getEventStats(DatabaseTimespan.TODAY, textOnly=True) == 1)
db.commitAndClose()
def run(self, argv: List[str]) -> None:
"""This is the main entry point of the backend application."""
defaultConfigPath = os.path.join(
self.appInfo.path, SituationBoard.DEFAULT_CONFIG_FILENAME)
defaultConfigPath = os.getenv("SB_CONFIG", defaultConfigPath)
defaultDatabasePath = os.path.join(
self.appInfo.path, SituationBoard.DEFAULT_DATABASE_FILENAME)
defaultDatabasePath = os.getenv("SB_DATABASE", defaultDatabasePath)
parser = argparse.ArgumentParser(
description=f"{self.appInfo.name} v{self.appInfo.version}",
add_help=False)
igroup = parser.add_argument_group("Important Commands and Parameters")
mmutex = igroup.add_mutually_exclusive_group()
mmutex.add_argument("-s",
"--server",
help="start backend server (default)",
default=True,
action='store_true')
mmutex.add_argument("-i",
"--import",
help="import data from CSV",
default=None,
dest='importFile')
mmutex.add_argument("-e",
"--export",
help="export data to CSV",
default=None,
dest='exportFile')
mmutex.add_argument("-n",
"--version",
help="show version number and exit",
default=False,
action='store_true')
mmutex.add_argument("-h",
"--help",
help="show this help message and exit",
action='help')
ogroup = parser.add_argument_group("Other Parameters and Options")
ogroup.add_argument("-c",
"--config",
help="alternate path to config file",
default=defaultConfigPath,
dest='configPath')
ogroup.add_argument("-d",
"--database",
help="alternate path to database file",
default=defaultDatabasePath,
dest='databasePath')
ogroup.add_argument("-r",
"--reset",
help="reset database before import",
default=False,
action='store_true')
ogroup.add_argument("-v",
"--verbose",
help="force debug output",
default=False,
action='store_true')
# args = parser.parse_args(argv)
args = parser.parse_args()
# Check command line parameters
if (args.reset is True) and (args.importFile is None):
parser.error("reset is only allowed for the import command")
sys.exit(1)
# Show version number and exit (if requested)
if args.version:
print(f"{self.appInfo.name} {self.appInfo.version}")
sys.exit(0)
# Print application header
if args.importFile is not None:
self.print("Starting import from CSV...")
elif args.exportFile is not None:
self.print("Starting export to CSV...")
else:
self.clrPrint(
f"Starting {self.appInfo.name} backend v{self.appInfo.version} (PID {self.appInfo.pid})"
)
# Load configuration and database
settings = Settings(args.configPath, self.appInfo.path, args.verbose)
database = Database(args.databasePath, args.reset)
self.lateInit(settings)
# Handle CSV import/export (if required)
if args.importFile is not None:
csvImporter = CSVImporter(database)
result = csvImporter.importEvents(args.importFile)
sys.exit(result)
elif args.exportFile is not None:
csvExporter = CSVExporter(database)
result = csvExporter.exportEvents(args.exportFile)
sys.exit(result)
# Start SituationBoard backend service
backendService = SituationBoardBackend(self.appInfo, settings,
database)
backendService.run()