def setupDB():
config = Config()
dbConn = Database(config)
# read SQL skeleton
with open(os.path.join(os.getcwd(), 'setup/db_create.sql'), 'r') as f:
sql = f.read()
# fill in placeholders
sql = sql.replace('&user', config.getProperty('Database', 'user'))
# run SQL
dbConn.execute(sql, None, None)
# add admin user
sql = '''
INSERT INTO aide_admin.user (name, email, hash, issuperuser)
VALUES (%s, %s, %s, %s)
ON CONFLICT (name) DO NOTHING;
'''
adminPass = config.getProperty('Project', 'adminPassword')
uHandler = UserHandling.backend.middleware.UserMiddleware(config)
adminPass = uHandler._create_hash(adminPass.encode('utf8'))
values = (config.getProperty('Project', 'adminName'), config.getProperty('Project', 'adminEmail'), adminPass, True,)
dbConn.execute(sql, values, None)
# setup
print('Setup...')
if not 'AIDE_CONFIG_PATH' in os.environ:
os.environ['AIDE_CONFIG_PATH'] = str(args.settings_filepath)
from util.configDef import Config
from modules.Database.app import Database
config = Config()
dbConn = Database(config)
# load model class function
print('Load and verify state dict...')
from util.helpers import get_class_executable, current_time
modelClass = getattr(
get_class_executable(
config.getProperty('AIController', 'model_lib_path')),
'model_class')
# load state dict
stateDict = torch.load(open(args.modelPath, 'rb'))
# verify model state
model = modelClass.loadFromStateDict(stateDict)
# load class definitions from database
classdef_db = {}
labelClasses = dbConn.execute(
'SELECT * FROM {schema}.labelclass;'.format(
schema=config.getProperty('Database', 'schema')), None, 'all')
for lc in labelClasses:
classdef_db[lc['id']] = lc
# setup
print('Setup...\n')
if not 'AIDE_CONFIG_PATH' in os.environ:
os.environ['AIDE_CONFIG_PATH'] = str(args.settings_filepath)
from tqdm import tqdm
from util.configDef import Config
from modules import Database
config = Config()
# setup DB connection
dbConn = Database(config)
if dbConn.connectionPool is None:
raise Exception('Error connecting to database.')
dbSchema = config.getProperty('Database', 'schema')
# check if correct type of annotations
annotationType = dbConn.execute(
'SELECT annotationtype FROM aide_admin.project WHERE shortname = %s;',
(args.project, ), 1)
if not len(annotationType):
raise Exception(
f'Project with name "{args.project}" could not be found in database.'
)
annotationType = annotationType[0]['annotationtype']
exportAnnotations = args.export_annotations
if exportAnnotations and not (annotationType == 'boundingBoxes'):
print(
'Warning: project annotations are not bounding boxes; skipping annotation export...'
)
import glob
from tqdm import tqdm
import datetime
import numpy as np
from PIL import Image
import base64
from io import BytesIO
from util.configDef import Config
from modules import Database
config = Config()
# check if correct type of annotations
exportAnnotations = args.export_annotations
if exportAnnotations and not config.getProperty('Project', 'annotationType') == 'segmentationMasks':
print('Warning: project annotations are not segmentation masks; skipping annotation export...')
exportAnnotations = False
# setup DB connection
dbConn = Database(config)
if dbConn.connectionPool is None:
raise Exception('Error connecting to database.')
dbSchema = config.getProperty('Database', 'schema')
# check if valid file format provided
valid_file_formats = (
'jpg',
'jpeg',
'png',
args = parser.parse_args()
# setup
print('Setup...\n')
if not 'AIDE_CONFIG_PATH' in os.environ:
os.environ['AIDE_CONFIG_PATH'] = str(args.settings_filepath)
from tqdm import tqdm
from util.configDef import Config
from modules import Database
config = Config()
# check if correct type of annotations
exportAnnotations = args.export_annotations
if exportAnnotations and not config.getProperty(
'Project', 'annotationType') == 'boundingBoxes':
print(
'Warning: project annotations are not bounding boxes; skipping annotation export...'
)
exportAnnotations = False
# setup DB connection
dbConn = Database(config)
if dbConn.connectionPool is None:
raise Exception('Error connecting to database.')
dbSchema = config.getProperty('Database', 'schema')
# query label definition
labeldef = {} # label UUID : (name, index,)
labeldef_inv = [] # list of label names in order (for export)
labelQuery = dbConn.execute(
from tqdm import tqdm
import datetime
from PIL import Image
from util.configDef import Config
from modules import Database
currentDT = datetime.datetime.now()
currentDT = '{}-{}-{} {}:{}:{}'.format(currentDT.year, currentDT.month,
currentDT.day, currentDT.hour,
currentDT.minute, currentDT.second)
config = Config()
dbConn = Database(config)
if dbConn.connectionPool is None:
raise Exception('Error connecting to database.')
dbSchema = config.getProperty('Database', 'schema')
valid_extensions = ('.jpg', '.jpeg', '.png', '.gif', '.tif', '.tiff',
'.bmp', '.ico', '.jfif', '.pjpeg', '.pjp')
# check if running on file server
imgBaseDir = config.getProperty('FileServer', 'staticfiles_dir')
if not os.path.isdir(imgBaseDir):
raise Exception(
'"{}" is not a valid directory on this machine. Are you running the script from the file server?'
.format(imgBaseDir))
if not imgBaseDir.endswith('/'):
imgBaseDir += '/'
# locate all images and their base names
help='Target filename for the model.')
args = parser.parse_args()
# setup
print('Setup...')
if not 'AIDE_CONFIG_PATH' in os.environ:
os.environ['AIDE_CONFIG_PATH'] = str(args.settings_filepath)
from util.configDef import Config
from modules import Database
config = Config()
# setup DB connection
dbConn = Database(config)
if dbConn.connectionPool is None:
raise Exception('Error connecting to database.')
dbSchema = config.getProperty('Database', 'schema')
# get state dict
print('Retrieving model state...')
stateDict_raw = dbConn.execute(
'SELECT statedict FROM {schema}.cnnstate WHERE partial IS FALSE ORDER BY timecreated DESC LIMIT 1;'
.format(schema=config.getProperty('Database', 'schema')), None, 1)
# convert from bytes and save to disk
print('Saving model state...')
stateDict_parsed = io.BytesIO(stateDict_raw[0]['statedict'])
stateDict_parsed = torch.load(stateDict_parsed,
map_location=lambda storage, loc: storage)
torch.save(stateDict_parsed, open(args.target_file, 'wb'))
args = parser.parse_args()
from util.configDef import Config
from modules import Database
# v2 config file
config = Config()
dbConn = Database(config)
if dbConn.connectionPool is None:
raise Exception('Error connecting to database.')
# v1 config file
v1Config = Config(args.settings_filepath)
# db schema of v1 project
dbSchema = v1Config.getProperty('Database', 'schema')
projectName = v1Config.getProperty('Project', 'projectName')
# verify we're running a database on v2 standards
isV2 = dbConn.execute(
'''
SELECT 1
FROM information_schema.tables
WHERE table_schema = 'aide_admin'
AND table_name = 'project';
''', None, 'all')
if isV2 is None or not len(isV2):
# upgrade to v2
dbName = config.getProperty('Database', 'name')
print(
f'WARNING: Target database "{dbName}" has not (yet) been upgraded to the AIDE v2 schema; we will attempt to do this now...'
parser.add_argument(
'--force_migrate',
type=int,
default=0,
help=
'If set to 1, database upgrade will be enforced even if AIDE versions already match.'
)
parser.add_argument(
'--verbose',
type=int,
default=1,
help='Set to 1 to print launch information to console.')
args = parser.parse_args()
try:
app = assemble_server(args.verbose, args.check_v1, args.migrate_db,
args.force_migrate, not bool(args.launch))
except Exception as e:
print(e)
sys.exit(1)
if bool(args.launch):
if args.verbose:
print('Launching server...')
config = Config(False)
host = config.getProperty('Server', 'host')
port = config.getProperty('Server', 'port')
app.run(host=host, port=port)
else:
sys.exit(0)
parser.add_argument('--settings_filepath', type=str, default='config/settings.ini', const=1, nargs='?',
help='Manual specification of the directory of the settings.ini file; only considered if environment variable unset (default: "config/settings.ini").')
args = parser.parse_args()
if not 'AIDE_CONFIG_PATH' in os.environ:
os.environ['AIDE_CONFIG_PATH'] = args.settings_filepath
config = Config()
dbConn = Database(config)
# read SQL skeleton
with open(os.path.join(os.getcwd(), 'setup/db_create.sql'), 'r') as f:
sql = f.read()
# fill in placeholders
sql = sql.replace('&user', config.getProperty('Database', 'user'))
# run SQL
dbConn.execute(sql, None, None)
# add admin user
sql = '''
INSERT INTO aide_admin.user (name, email, hash, issuperuser)
VALUES (%s, %s, %s, %s)
'''
adminPass = config.getProperty('Project', 'adminPassword')
uHandler = UserHandling.backend.middleware.UserMiddleware(config)
adminPass = uHandler._create_hash(adminPass.encode('utf8'))
values = (config.getProperty('Project', 'adminName'), config.getProperty('Project', 'adminEmail'), adminPass, True,)
dbConn.execute(sql, values, None)
elif module == 'fileserver':
queues.append(Queue('FileServer'))
elif module == 'aiworker':
queues.append(Queue('AIWorker'))
elif module == 'labelui':
queues.append(Queue('ModelMarketplace')) #TODO: allow outsourcing? Move to FileServer?
queues.extend([
Broadcast('aide_broadcast'),
Queue('aide@'+celery.utils.nodenames.gethostname())
])
app = Celery('AIDE',
broker=config.getProperty('AIController', 'broker_URL'), #TODO
backend=config.getProperty('AIController', 'result_backend')) #TODO
app.conf.update(
result_backend=config.getProperty('AIController', 'result_backend'), #TODO
task_ignore_result=False,
result_persistent=True,
accept_content = ['json'],
task_serializer = 'json',
result_serializer = 'json',
task_track_started = True,
broker_pool_limit=None, # required to avoid peer connection resets
broker_heartbeat = 0, # required to avoid peer connection resets
worker_max_tasks_per_child = 1, # required to free memory (also CUDA) after each process
# task_default_rate_limit = 3, #TODO
worker_prefetch_multiplier = 1, #TODO
task_acks_late = True,
)
args = parser.parse_args()
if not 'AIDE_CONFIG_PATH' in os.environ:
os.environ['AIDE_CONFIG_PATH'] = args.settings_filepath
config = Config()
dbConn = Database(config)
# read SQL skeleton
with open(os.path.join(os.getcwd(), 'projectCreation/db_create.sql'),
'r') as f:
sql = f.read()
# fill in placeholders
annoType_frontend = config.getProperty('Project', 'annotationType')
annoFields_frontend, additionalTables_frontend = _constructAnnotationFields(
annoType_frontend, 'annotation')
if additionalTables_frontend is not None:
sql += additionalTables_frontend
annoType_backend = config.getProperty('Project', 'predictionType')
annoFields_backend, additionalTables_backend = _constructAnnotationFields(
annoType_backend, 'prediction')
if additionalTables_backend is not None:
sql += additionalTables_backend
sql = sql.replace('&annotationFields', annoFields_frontend)
sql = sql.replace('&predictionFields', annoFields_backend)
sql = sql.replace('&dbName', config.getProperty('Database', 'name'))
print(f'''
The current installation of AIDE:
database host: {config.getProperty('Database', 'host')}
database name: {config.getProperty('Database', 'name')}
schema: {config.getProperty('Database', 'schema', str, '(not specified)')}
points to an installation of the legacy AIDE v1.
If you wish to continue using AIDE v2, you have to upgrade the project accordingly.
For instructions to do so, see here:
https://github.com/microsoft/aerial_wildlife_detection/blob/multiProject/doc/upgrade_from_v1.md
''')
import sys
sys.exit(1)
# check if project has been migrated
dbSchema = config.getProperty('Database', 'schema', str, None)
if dbSchema is not None:
isMigrated = dbConnector.execute(
'''
SELECT COUNT(*) AS cnt
FROM aide_admin.project
WHERE shortname = %s;
''', (dbSchema, ), 1)
if isMigrated is not None and len(
isMigrated) and isMigrated[0]['cnt'] == 0:
print(f'''
WARNING: the selected configuration .ini file
("{os.environ['AIDE_CONFIG_PATH']}")
points to a project that has not yet been migrated to AIDE v2.
Details:
database host: {config.getProperty('Database', 'host')}
description='Update AIde database structure.')
parser.add_argument(
'--settings_filepath',
type=str,
default='config/settings.ini',
const=1,
nargs='?',
help=
'Manual specification of the directory of the settings.ini file; only considered if environment variable unset (default: "config/settings.ini").'
)
args = parser.parse_args()
if not 'AIDE_CONFIG_PATH' in os.environ:
os.environ['AIDE_CONFIG_PATH'] = str(args.settings_filepath)
from util.configDef import Config
from modules import Database
config = Config()
dbConn = Database(config)
if dbConn.connectionPool is None:
raise Exception('Error connecting to database.')
dbSchema = config.getProperty('Database', 'schema')
# make modifications one at a time
for mod in MODIFICATIONS_sql:
dbConn.execute(mod.format(schema=dbSchema), None, None)
print(
'Project {} is now up-to-date for the latest changes in AIde.'.format(
config.getProperty('Project', 'projectName')))
from modules import Database
if args.label_folder == '':
args.label_folder = None
if args.label_folder is not None and not args.label_folder.endswith('/'):
args.label_folder += '/'
currentDT = datetime.datetime.now()
currentDT = '{}-{}-{} {}:{}:{}'.format(currentDT.year, currentDT.month, currentDT.day, currentDT.hour, currentDT.minute, currentDT.second)
config = Config()
dbConn = Database(config)
if dbConn.connectionPool is None:
raise Exception('Error connecting to database.')
dbSchema = config.getProperty('Database', 'schema')
valid_image_extensions = (
'.jpg',
'.jpeg',
'.png',
'.gif',
'.tif',
'.tiff',
'.bmp',
'.ico',
'.jfif',
'.pjpeg',
'.pjp'
)
def assemble_server(verbose_start=True,
check_v1_config=True,
migrate_database=True,
force_migrate=False,
passive_mode=False):
# force verbosity if any of the pre-flight checks is enabled
verbose_start = any((verbose_start, check_v1_config, migrate_database))
instance_args = os.environ['AIDE_MODULES'].split(',')
if verbose_start:
configPath = os.environ['AIDE_CONFIG_PATH']
aideModules = ', '.join(instance_args)
print(f'''\033[96m
#################################
version {AIDE_VERSION}
### #### ######## ########
## ## ## ## ## ## {platform.platform()}
## ## ## ## ## ##
## ## ## ## ## ###### [config]
######### ## ## ## ## .> {configPath}
## ## ## ## ## ##
## ## #### ######## ######## [modules]
.> {aideModules}
#################################\033[0m
''')
statusOffset = LogDecorator.get_ljust_offset()
# load configuration
config = Config(None, verbose_start)
bottle.BaseRequest.MEMFILE_MAX = 1024**3 #TODO: make hyperparameter in config?
# connect to database
dbConnector = Database(config, verbose_start)
if check_v1_config:
# check if config file points to unmigrated v1 project
print('Checking database...'.ljust(statusOffset), end='')
hasAdminTable = dbConnector.execute(
'''
SELECT EXISTS (
SELECT FROM information_schema.tables
WHERE table_schema = 'aide_admin'
AND table_name = 'project'
);
''', None, 1)
if not hasAdminTable[0]['exists']:
# not (yet) migrated, raise Exception with instructions to ensure compatibility
LogDecorator.print_status('fail')
print(f'''
The current installation of AIDE:
database host: {config.getProperty('Database', 'host')}
database name: {config.getProperty('Database', 'name')}
schema: {config.getProperty('Database', 'schema', str, '(not specified)')}
points to an installation of the legacy AIDE v1.
If you wish to continue using AIDE v2, you have to upgrade the project accordingly.
For instructions to do so, see here:
https://github.com/microsoft/aerial_wildlife_detection/blob/multiProject/doc/upgrade_from_v1.md
''')
sys.exit(1)
else:
LogDecorator.print_status('ok')
# check if projects have been migrated
print('Checking projects...'.ljust(statusOffset), end='')
dbSchema = config.getProperty('Database', 'schema', str, None)
if dbSchema is not None:
isMigrated = dbConnector.execute(
'''
SELECT COUNT(*) AS cnt
FROM aide_admin.project
WHERE shortname = %s;
''', (dbSchema, ), 1)
if isMigrated is not None and len(
isMigrated) and isMigrated[0]['cnt'] == 0:
LogDecorator.print_status('warn')
print(f'''
WARNING: the selected configuration .ini file
("{os.environ['AIDE_CONFIG_PATH']}")
points to a project that has not yet been migrated to AIDE v2.
Details:
database host: {config.getProperty('Database', 'host')}
database name: {config.getProperty('Database', 'name')}
schema: {dbSchema}
If you wish to continue using AIDE v2 for this project, you have to upgrade it
to v2 accordingly.
For instructions to do so, see here:
https://github.com/microsoft/aerial_wildlife_detection/blob/multiProject/doc/upgrade_from_v1.md
''')
else:
LogDecorator.print_status('ok')
else:
LogDecorator.print_status('ok')
if migrate_database:
# bring AIDE up-to-date
print('Updating database...'.ljust(statusOffset), end='')
warnings, errors = migrate_aide(force_migrate)
if len(warnings) or len(errors):
if len(errors):
LogDecorator.print_status('fail')
else:
LogDecorator.print_status('warn')
print(
f'Warnings and/or errors occurred while updating AIDE to the latest version ({AIDE_VERSION}):'
)
print('\nWarnings:')
for w in warnings:
print(f'\t"{w}"')
print('\nErrors:')
for e in errors:
print(f'\t"{e}"')
if len(errors):
sys.exit(2)
else:
LogDecorator.print_status('ok')
# prepare bottle
app = Bottle()
# parse requested instances
instances = {}
# "singletons"
dbConnector = REGISTERED_MODULES['Database'](config, verbose_start)
userHandler = REGISTERED_MODULES['UserHandler'](config, app, dbConnector)
taskCoordinator = REGISTERED_MODULES['TaskCoordinator'](config, app,
dbConnector,
verbose_start)
taskCoordinator.addLoginCheckFun(userHandler.checkAuthenticated)
for i in instance_args:
moduleName = i.strip()
if moduleName == 'UserHandler':
continue
moduleClass = REGISTERED_MODULES[moduleName]
# verify
_verify_unique(instances, moduleClass)
# create instance
if moduleName == 'AIController':
instance = moduleClass(config, app, dbConnector, taskCoordinator,
verbose_start, passive_mode)
else:
instance = moduleClass(config, app, dbConnector, verbose_start)
instances[moduleName] = instance
# add authentication functionality
if hasattr(instance, 'addLoginCheckFun'):
instance.addLoginCheckFun(userHandler.checkAuthenticated)
# launch project meta modules
if moduleName == 'LabelUI':
aideAdmin = REGISTERED_MODULES['AIDEAdmin'](config, app,
dbConnector,
verbose_start)
aideAdmin.addLoginCheckFun(userHandler.checkAuthenticated)
reception = REGISTERED_MODULES['Reception'](config, app,
dbConnector)
reception.addLoginCheckFun(userHandler.checkAuthenticated)
configurator = REGISTERED_MODULES['ProjectConfigurator'](
config, app, dbConnector)
configurator.addLoginCheckFun(userHandler.checkAuthenticated)
statistics = REGISTERED_MODULES['ProjectStatistics'](config, app,
dbConnector)
statistics.addLoginCheckFun(userHandler.checkAuthenticated)
elif moduleName == 'FileServer':
from modules.DataAdministration.backend import celery_interface as daa_int
elif moduleName == 'AIController':
from modules.AIController.backend import celery_interface as aic_int
# launch model marketplace with AIController
modelMarketplace = REGISTERED_MODULES['ModelMarketplace'](
config, app, dbConnector, taskCoordinator)
modelMarketplace.addLoginCheckFun(userHandler.checkAuthenticated)
elif moduleName == 'AIWorker':
from modules.AIWorker.backend import celery_interface as aiw_int
# launch globally required modules
dataAdmin = REGISTERED_MODULES['DataAdministrator'](config, app,
dbConnector,
taskCoordinator)
dataAdmin.addLoginCheckFun(userHandler.checkAuthenticated)
staticFiles = REGISTERED_MODULES['StaticFileServer'](config, app,
dbConnector)
staticFiles.addLoginCheckFun(userHandler.checkAuthenticated)
if verbose_start:
print('\n')
return app
import os
from configparser import ConfigParser
from celery import Celery
from celery import signals
from celery.bin import Option
from modules.AIWorker.app import AIWorker
from util.configDef import Config
# parse system config
if not 'AIDE_CONFIG_PATH' in os.environ:
raise ValueError('Missing system environment variable "AIDE_CONFIG_PATH".')
config = Config()
if config.getProperty('Project', 'demoMode', type=bool, fallback=False):
raise Exception(
'AIController and AIWorkers cannot be launched in demo mode.')
app = Celery('AIController',
broker=config.getProperty('AIController', 'broker_URL'),
backend=config.getProperty('AIController', 'result_backend'))
app.conf.update(
result_backend=config.getProperty('AIController', 'result_backend'),
task_ignore_result=False,
result_persistent=True,
accept_content=['json'],
task_serializer='json',
result_serializer='json',
task_track_started=True,
broker_heartbeat=0, # required to avoid peer connection resets
'ALTER TABLE {schema}.labelclass ADD COLUMN IF NOT EXISTS keystroke SMALLINT UNIQUE;',
'ALTER TABLE {schema}.image ADD COLUMN IF NOT EXISTS last_requested TIMESTAMPTZ;'
]
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='Update AIde database structure.')
parser.add_argument('--settings_filepath', type=str, default='config/settings.ini', const=1, nargs='?',
help='Manual specification of the directory of the settings.ini file; only considered if environment variable unset (default: "config/settings.ini").')
args = parser.parse_args()
if not 'AIDE_CONFIG_PATH' in os.environ:
os.environ['AIDE_CONFIG_PATH'] = str(args.settings_filepath)
from util.configDef import Config
from modules import Database
config = Config()
dbConn = Database(config)
if dbConn.connectionPool is None:
raise Exception('Error connecting to database.')
dbSchema = config.getProperty('Database', 'schema')
# make modifications one at a time
for mod in MODIFICATIONS_sql:
dbConn.execute(mod.format(schema=dbSchema), None, None)
print('Project {} is now up-to-date for the latest changes in AIde.'.format(config.getProperty('Project', 'projectName')))
if args.label_folder is not None and not args.label_folder.endswith(
os.sep):
args.label_folder += os.sep
currentDT = datetime.datetime.now()
currentDT = '{}-{}-{} {}:{}:{}'.format(currentDT.year, currentDT.month,
currentDT.day, currentDT.hour,
currentDT.minute, currentDT.second)
config = Config()
dbConn = Database(config)
if not dbConn.canConnect():
raise Exception('Error connecting to database.')
# check if running on file server
imgBaseDir = config.getProperty('FileServer', 'staticfiles_dir')
if not os.path.isdir(imgBaseDir):
raise Exception(
'"{}" is not a valid directory on this machine. Are you running the script from the file server?'
.format(imgBaseDir))
if args.label_folder is not None and not os.path.isdir(args.label_folder):
raise Exception(
'"{}" is not a valid directory on this machine.'.format(
args.label_folder))
if not imgBaseDir.endswith(os.sep):
imgBaseDir += os.sep
# parse class names and indices
if args.label_folder is not None:
def _verify_unique(instances, moduleClass):
'''
Compares the newly requested module, address and port against
already launched modules on this instance.
Raises an Exception if another module from the same type has already been launched on this instance
'''
for i in instances:
if moduleClass.__class__.__name__ == i.__class__.__name__:
raise Exception(
'Module {} already launched on this server.'.format(
moduleClass.__class__.__name__))
# load configuration
config = Config()
demoMode = config.getProperty('Project', 'demoMode', type=bool, fallback=False)
# prepare bottle
app = Bottle()
# parse requested instances
instance_args = os.environ['AIDE_MODULES'].split(',')
instances = []
# create user handler
userHandler = REGISTERED_MODULES['UserHandler'](config, app)
for i in instance_args:
moduleName = i.strip()
if moduleName == 'UserHandler':