def setupDB():
config = Config()
dbConn = Database(config)
# read SQL skeleton
with open(os.path.join(os.getcwd(), 'setup/db_create.sql'), 'r') as f:
sql = f.read()
# fill in placeholders
sql = sql.replace('&user', config.getProperty('Database', 'user'))
# run SQL
dbConn.execute(sql, None, None)
# add admin user
sql = '''
INSERT INTO aide_admin.user (name, email, hash, issuperuser)
VALUES (%s, %s, %s, %s)
ON CONFLICT (name) DO NOTHING;
'''
adminPass = config.getProperty('Project', 'adminPassword')
uHandler = UserHandling.backend.middleware.UserMiddleware(config)
adminPass = uHandler._create_hash(adminPass.encode('utf8'))
values = (config.getProperty('Project', 'adminName'), config.getProperty('Project', 'adminEmail'), adminPass, True,)
dbConn.execute(sql, values, None)
def migrate_aide():
from modules import Database, UserHandling
from util.configDef import Config
config = Config()
dbConn = Database(config)
if dbConn.connectionPool is None:
raise Exception('Error connecting to database.')
warnings = []
errors = []
# bring all projects up-to-date (if registered within AIDE)
projects = dbConn.execute('SELECT shortname FROM aide_admin.project;',
None, 'all')
if projects is not None and len(projects):
# get all schemata and check if project still exists
schemata = dbConn.execute(
'SELECT schema_name FROM information_schema.schemata', None, 'all')
if schemata is not None and len(schemata):
schemata = set([s['schema_name'].lower() for s in schemata])
for p in projects:
try:
pName = p['shortname']
# check if project still exists
if not pName.lower() in schemata:
warnings.append(
f'WARNING: project "{pName}" is registered but does not exist in database.'
)
#TODO: option to auto-remove?
continue
# make modifications one at a time
for mod in MODIFICATIONS_sql:
dbConn.execute(mod.format(schema=pName), None, None)
except Exception as e:
errors.append(str(e))
else:
warnings.append(
'WARNING: no project schemata found within database.')
else:
warnings.append('WARNING: no project registered within AIDE.')
return warnings, errors
# parser.add_argument('--append_confidences', type=bool, default=False, const=1, nargs='?',
# help='Whether to append confidences to each prediction (default: False).')
# parser.add_argument('--predictions_min_date', type=str, default=None, const=1, nargs='?',
# help='Timestamp of earliest predictions to consider (default: None, i.e. all).')
args = parser.parse_args()
# setup
print('Setup...\n')
if not 'AIDE_CONFIG_PATH' in os.environ:
os.environ['AIDE_CONFIG_PATH'] = str(args.settings_filepath)
from tqdm import tqdm
from util.configDef import Config
from modules import Database
config = Config()
# check if correct type of annotations
exportAnnotations = args.export_annotations
if exportAnnotations and not config.getProperty(
'Project', 'annotationType') == 'boundingBoxes':
print(
'Warning: project annotations are not bounding boxes; skipping annotation export...'
)
exportAnnotations = False
# setup DB connection
dbConn = Database(config)
if dbConn.connectionPool is None:
raise Exception('Error connecting to database.')
dbSchema = config.getProperty('Database', 'schema')
type=str,
default='model_state.pth',
const=1,
nargs='?',
help='Target filename for the model.')
args = parser.parse_args()
# setup
print('Setup...')
if not 'AIDE_CONFIG_PATH' in os.environ:
os.environ['AIDE_CONFIG_PATH'] = str(args.settings_filepath)
from util.configDef import Config
from modules import Database
config = Config()
# setup DB connection
dbConn = Database(config)
if dbConn.connectionPool is None:
raise Exception('Error connecting to database.')
dbSchema = config.getProperty('Database', 'schema')
# get state dict
print('Retrieving model state...')
stateDict_raw = dbConn.execute(
'SELECT statedict FROM {schema}.cnnstate WHERE partial IS FALSE ORDER BY timecreated DESC LIMIT 1;'
.format(schema=config.getProperty('Database', 'schema')), None, 1)
# convert from bytes and save to disk
print('Saving model state...')
description=
'Upgrade and register an AIDE v1 project to an existing v2 installation.'
)
parser.add_argument(
'--settings_filepath',
type=str,
help=
'Path of the configuration .ini file for the v1 project to be upgraded to v2.'
)
args = parser.parse_args()
from util.configDef import Config
from modules import Database
# v2 config file
config = Config()
dbConn = Database(config)
if dbConn.connectionPool is None:
raise Exception('Error connecting to database.')
# v1 config file
v1Config = Config(args.settings_filepath)
# db schema of v1 project
dbSchema = v1Config.getProperty('Database', 'schema')
projectName = v1Config.getProperty('Project', 'projectName')
# verify we're running a database on v2 standards
isV2 = dbConn.execute(
'''
SELECT 1
'''
import os
from configparser import ConfigParser
from celery import Celery
from celery import signals
from celery.bin import Option
from modules.AIWorker.app import AIWorker
from util.configDef import Config
# parse system config
if not 'AIDE_CONFIG_PATH' in os.environ:
raise ValueError('Missing system environment variable "AIDE_CONFIG_PATH".')
config = Config()
if config.getProperty('Project', 'demoMode', type=bool, fallback=False):
raise Exception(
'AIController and AIWorkers cannot be launched in demo mode.')
app = Celery('AIController',
broker=config.getProperty('AIController', 'broker_URL'),
backend=config.getProperty('AIController', 'result_backend'))
app.conf.update(
result_backend=config.getProperty('AIController', 'result_backend'),
task_ignore_result=False,
result_persistent=True,
accept_content=['json'],
task_serializer='json',
result_serializer='json',
task_track_started=True,
from PIL import Image
import base64
from io import BytesIO
from util.configDef import Config
from modules import Database
if args.label_folder == '':
args.label_folder = None
if args.label_folder is not None and not args.label_folder.endswith('/'):
args.label_folder += '/'
currentDT = datetime.datetime.now()
currentDT = '{}-{}-{} {}:{}:{}'.format(currentDT.year, currentDT.month, currentDT.day, currentDT.hour, currentDT.minute, currentDT.second)
config = Config()
dbConn = Database(config)
if dbConn.connectionPool is None:
raise Exception('Error connecting to database.')
dbSchema = config.getProperty('Database', 'schema')
valid_image_extensions = (
'.jpg',
'.jpeg',
'.png',
'.gif',
'.tif',
'.tiff',
'.bmp',
'.ico',
'.jfif',
parser.add_argument(
'--modelPath',
type=str,
help=
'Directory (absolute path) on this machine of the model state file to be considered.'
)
args = parser.parse_args()
# setup
print('Setup...')
if not 'AIDE_CONFIG_PATH' in os.environ:
os.environ['AIDE_CONFIG_PATH'] = str(args.settings_filepath)
from util.configDef import Config
from modules.Database.app import Database
config = Config()
dbConn = Database(config)
# load model class function
print('Load and verify state dict...')
from util.helpers import get_class_executable, current_time
modelClass = getattr(
get_class_executable(
config.getProperty('AIController', 'model_lib_path')),
'model_class')
# load state dict
stateDict = torch.load(open(args.modelPath, 'rb'))
# verify model state
model = modelClass.loadFromStateDict(stateDict)
'''
Decorator that enables Cross-Origin Resource Sharing (CORS).
2019-20 Benjamin Kellenberger
'''
from urllib.parse import urlparse
import bottle
from bottle import request, response
from util.configDef import Config
# CORS is only required if the FileServer URI is a valid URL
fileServerURI = Config().getProperty('Server', 'dataServer_uri')
result = urlparse(fileServerURI)
cors_required = all([result.scheme,
result.netloc]) #TODO: might be too strict...
def enable_cors(fn):
def _enable_cors(*args, **kwargs):
# set CORS headers
if cors_required:
response.set_header('Access-Control-Allow-Origin', fileServerURI)
response.set_header('Access-Control-Allow-Credentials', 'true')
response.add_header('Access-Control-Allow-Methods',
'GET, POST, PUT, OPTIONS')
response.add_header(
'Access-Control-Allow-Headers',
'Origin, Accept, Content-Type, X-Requested-With, X-CSRF-Token, *'
)
import os
import celery
from celery import Celery
from kombu import Queue
from kombu.common import Broadcast
from util.configDef import Config
# force enable passive mode
os.environ['PASSIVE_MODE'] = '1'
# parse system config
if not 'AIDE_CONFIG_PATH' in os.environ:
raise ValueError('Missing system environment variable "AIDE_CONFIG_PATH".')
if not 'AIDE_MODULES' in os.environ:
raise ValueError('Missing system environment variable "AIDE_MODULES".')
config = Config()
# parse AIDE modules and set up queues
queues = []
aideModules = os.environ['AIDE_MODULES'].split(',')
aideModules = set([a.strip().lower() for a in aideModules])
for m in aideModules:
module = m.strip().lower()
if module == 'aicontroller':
queues.append(Queue('AIController'))
elif module == 'fileserver':
queues.append(Queue('FileServer'))
elif module == 'aiworker':
queues.append(Queue('AIWorker'))
elif module == 'labelui':
def migrate_aide(forceMigrate=False):
from modules import Database, UserHandling
from util.configDef import Config
config = Config()
dbConn = Database(config)
if not dbConn.canConnect():
raise Exception('Error connecting to database.')
warnings = []
errors = []
# skip if not forced and if database has same version
doMigrate = True
# check if DB has version already implemented
dbVersion = None
hasVersion = dbConn.execute('''
SELECT EXISTS (
SELECT FROM information_schema.tables
WHERE table_schema = 'aide_admin'
AND table_name = 'version'
) AS hasVersion;
''', None, 1)
if hasVersion[0]['hasversion']:
# check DB version
dbVersion = dbConn.execute('SELECT version FROM aide_admin.version;', None, 1)
if dbVersion is not None and len(dbVersion):
dbVersion = dbVersion[0]['version']
needsUpdate = version.compare_versions(version.AIDE_VERSION, dbVersion)
if needsUpdate is not None:
if needsUpdate < 0:
# running an older version of AIDE with a newer DB version
warnings.append(f'WARNING: local AIDE version ({version.AIDE_VERSION}) is older than the one in the database ({dbVersion}); please update your installation.')
elif needsUpdate == 0:
doMigrate = False
else:
doMigrate = True
if not doMigrate and not forceMigrate:
return warnings, errors
# bring all projects up-to-date (if registered within AIDE)
projects = dbConn.execute('SELECT shortname FROM aide_admin.project;', None, 'all')
if projects is not None and len(projects):
# get all schemata and check if project still exists
schemata = dbConn.execute('SELECT schema_name FROM information_schema.schemata', None, 'all')
if schemata is not None and len(schemata):
schemata = set([s['schema_name'].lower() for s in schemata])
for p in projects:
try:
pName = p['shortname']
# check if project still exists
if not pName.lower() in schemata:
warnings.append(f'WARNING: project "{pName}" is registered but does not exist in database.')
#TODO: option to auto-remove?
continue
# special modification for CNN-to-labelclass map: drop only dep. on version (remove ancient tests)
if version.compare_versions(version.AIDE_VERSION, dbVersion) in (-1, None):
dbConn.execute(sql.SQL('DROP TABLE IF EXISTS {};').format(
sql.Identifier(pName, 'cnn_labelclass')
), None)
# make modifications one at a time
for mod in MODIFICATIONS_sql:
dbConn.execute(mod.format(schema=pName), None, None)
# pre-official 2.0: mark existing CNN states as "labelclass_autoupdate" (as this was the default behavior)
if version.compare_versions(dbVersion, '2.0.210514') == -1:
dbConn.execute(sql.SQL('''
UPDATE {}
SET labelclass_autoupdate = TRUE;
''').format(sql.Identifier(pName, 'cnnstate')), None)
except Exception as e:
errors.append(str(e))
else:
warnings.append('WARNING: no project schemata found within database.')
else:
warnings.append('WARNING: no project registered within AIDE.')
# update DB version accordingly
dbConn.execute('''
DELETE FROM aide_admin.version;
INSERT INTO aide_admin.version (version)
VALUES (%s);
''', (version.AIDE_VERSION, ))
return warnings, errors
description='Set up AIDE database schema.')
parser.add_argument(
'--settings_filepath',
type=str,
default='config/settings.ini',
const=1,
nargs='?',
help=
'Manual specification of the directory of the settings.ini file; only considered if environment variable unset (default: "config/settings.ini").'
)
args = parser.parse_args()
if not 'AIDE_CONFIG_PATH' in os.environ:
os.environ['AIDE_CONFIG_PATH'] = args.settings_filepath
config = Config()
dbConn = Database(config)
# read SQL skeleton
with open(os.path.join(os.getcwd(), 'projectCreation/db_create.sql'),
'r') as f:
sql = f.read()
# fill in placeholders
annoType_frontend = config.getProperty('Project', 'annotationType')
annoFields_frontend, additionalTables_frontend = _constructAnnotationFields(
annoType_frontend, 'annotation')
if additionalTables_frontend is not None:
sql += additionalTables_frontend
annoType_backend = config.getProperty('Project', 'predictionType')
def _verify_unique(instances, moduleClass):
'''
Compares the newly requested module, address and port against
already launched modules on this instance.
Raises an Exception if another module from the same type has already been launched on this instance
'''
for key in instances.keys():
instance = instances[key]
if moduleClass.__class__.__name__ == instance.__class__.__name__:
raise Exception(
'Module {} already launched on this server.'.format(
moduleClass.__class__.__name__))
# load configuration
config = Config()
# check if config file points to unmigrated v1 project
dbConnector = Database(config)
hasAdminTable = dbConnector.execute(
'''
SELECT EXISTS (
SELECT FROM information_schema.tables
WHERE table_schema = 'aide_admin'
AND table_name = 'project'
);
''', None, 1)
if not hasAdminTable[0]['exists']:
# not (yet) migrated, raise Exception with instructions to ensure compatibility
print(f'''
The current installation of AIDE:
description='Update AIde database structure.')
parser.add_argument(
'--settings_filepath',
type=str,
default='config/settings.ini',
const=1,
nargs='?',
help=
'Manual specification of the directory of the settings.ini file; only considered if environment variable unset (default: "config/settings.ini").'
)
args = parser.parse_args()
if not 'AIDE_CONFIG_PATH' in os.environ:
os.environ['AIDE_CONFIG_PATH'] = str(args.settings_filepath)
from util.configDef import Config
from modules import Database
config = Config()
dbConn = Database(config)
if dbConn.connectionPool is None:
raise Exception('Error connecting to database.')
dbSchema = config.getProperty('Database', 'schema')
# make modifications one at a time
for mod in MODIFICATIONS_sql:
dbConn.execute(mod.format(schema=dbSchema), None, None)
print(
'Project {} is now up-to-date for the latest changes in AIde.'.format(
config.getProperty('Project', 'projectName')))
def _verify_unique(instances, moduleClass):
'''
Compares the newly requested module, address and port against
already launched modules on this instance.
Raises an Exception if another module from the same type has already been launched on this instance
'''
for key in instances.keys():
instance = instances[key]
if moduleClass.__class__.__name__ == instance.__class__.__name__:
raise Exception(
'Module {} already launched on this server.'.format(
moduleClass.__class__.__name__))
# load configuration
config = Config(verbose_start=True)
# connect to database
dbConnector = Database(config, verbose_start=True)
# check if config file points to unmigrated v1 project
print('Checking database...'.ljust(10), end='')
hasAdminTable = dbConnector.execute(
'''
SELECT EXISTS (
SELECT FROM information_schema.tables
WHERE table_schema = 'aide_admin'
AND table_name = 'project'
);
''', None, 1)
if not hasAdminTable[0]['exists']:
Function "init_celery_dispatcher" is to be initia-
lized at launch time with a Celery app instance.
2020 Benjamin Kellenberger
'''
import os
from celery import current_app
from .dataWorker import DataWorker
from util.configDef import Config
# initialise dataWorker
modules = os.environ['AIDE_MODULES']
passiveMode = (os.environ['PASSIVE_MODE'] == '1' if 'PASSIVE_MODE' in
os.environ else False) or not ('fileserver' in modules.lower())
worker = DataWorker(Config())
@current_app.task()
def aide_internal_notify(message):
return worker.aide_internal_notify(message)
@current_app.task(name='DataAdministration.list_images')
def listImages(project,
imageAddedRange=None,
lastViewedRange=None,
viewcountRange=None,
numAnnoRange=None,
numPredRange=None,
orderBy=None,
# setup
print('Setup...')
if not 'AIDE_CONFIG_PATH' in os.environ:
os.environ['AIDE_CONFIG_PATH'] = str(args.settings_filepath)
import glob
from tqdm import tqdm
import datetime
import numpy as np
from PIL import Image
import base64
from io import BytesIO
from util.configDef import Config
from modules import Database
config = Config()
# check if correct type of annotations
exportAnnotations = args.export_annotations
if exportAnnotations and not config.getProperty('Project', 'annotationType') == 'segmentationMasks':
print('Warning: project annotations are not segmentation masks; skipping annotation export...')
exportAnnotations = False
# setup DB connection
dbConn = Database(config)
if dbConn.connectionPool is None:
raise Exception('Error connecting to database.')
dbSchema = config.getProperty('Database', 'schema')
# check if valid file format provided
from util.configDef import Config
from modules import Database, UserHandling
if __name__ == '__main__':
# setup
parser = argparse.ArgumentParser(description='Set up AIDE database schema.')
parser.add_argument('--settings_filepath', type=str, default='config/settings.ini', const=1, nargs='?',
help='Manual specification of the directory of the settings.ini file; only considered if environment variable unset (default: "config/settings.ini").')
args = parser.parse_args()
if not 'AIDE_CONFIG_PATH' in os.environ:
os.environ['AIDE_CONFIG_PATH'] = args.settings_filepath
config = Config()
dbConn = Database(config)
# read SQL skeleton
with open(os.path.join(os.getcwd(), 'setup/db_create.sql'), 'r') as f:
sql = f.read()
# fill in placeholders
sql = sql.replace('&user', config.getProperty('Database', 'user'))
# run SQL
dbConn.execute(sql, None, None)
# add admin user
sql = '''
INSERT INTO aide_admin.user (name, email, hash, issuperuser)
# parser.add_argument('--append_confidences', type=bool, default=False, const=1, nargs='?',
# help='Whether to append confidences to each prediction (default: False).')
# parser.add_argument('--predictions_min_date', type=str, default=None, const=1, nargs='?',
# help='Timestamp of earliest predictions to consider (default: None, i.e. all).')
args = parser.parse_args()
# setup
print('Setup...\n')
if not 'AIDE_CONFIG_PATH' in os.environ:
os.environ['AIDE_CONFIG_PATH'] = str(args.settings_filepath)
from tqdm import tqdm
from util.configDef import Config
from modules import Database
config = Config()
# setup DB connection
dbConn = Database(config)
if dbConn.connectionPool is None:
raise Exception('Error connecting to database.')
dbSchema = config.getProperty('Database', 'schema')
# check if correct type of annotations
annotationType = dbConn.execute(
'SELECT annotationtype FROM aide_admin.project WHERE shortname = %s;',
(args.project, ), 1)
if not len(annotationType):
raise Exception(
f'Project with name "{args.project}" could not be found in database.'
)
2020-21 Benjamin Kellenberger
'''
import os
from celery import current_app
from modules.AIController.backend.functional import AIControllerWorker
from util.configDef import Config
# init AIController middleware
modules = os.environ['AIDE_MODULES']
passiveMode = (os.environ['PASSIVE_MODE'] == '1'
if 'PASSIVE_MODE' in os.environ else
False) or not ('aicontroller' in modules.lower())
aim = AIControllerWorker(Config(), current_app)
@current_app.task(name='AIController.get_training_images')
def get_training_images(blank,
project,
epoch,
numEpochs,
minTimestamp='lastState',
includeGoldenQuestions=True,
minNumAnnoPerImage=0,
maxNumImages=None,
numWorkers=1):
return aim.get_training_images(project, epoch, numEpochs, minTimestamp,
includeGoldenQuestions, minNumAnnoPerImage,
maxNumImages, numWorkers)
parser.add_argument(
'--force_migrate',
type=int,
default=0,
help=
'If set to 1, database upgrade will be enforced even if AIDE versions already match.'
)
parser.add_argument(
'--verbose',
type=int,
default=1,
help='Set to 1 to print launch information to console.')
args = parser.parse_args()
try:
app = assemble_server(args.verbose, args.check_v1, args.migrate_db,
args.force_migrate, not bool(args.launch))
except Exception as e:
print(e)
sys.exit(1)
if bool(args.launch):
if args.verbose:
print('Launching server...')
config = Config(False)
host = config.getProperty('Server', 'host')
port = config.getProperty('Server', 'port')
app.run(host=host, port=port)
else:
sys.exit(0)
def assemble_server(verbose_start=True,
check_v1_config=True,
migrate_database=True,
force_migrate=False,
passive_mode=False):
# force verbosity if any of the pre-flight checks is enabled
verbose_start = any((verbose_start, check_v1_config, migrate_database))
instance_args = os.environ['AIDE_MODULES'].split(',')
if verbose_start:
configPath = os.environ['AIDE_CONFIG_PATH']
aideModules = ', '.join(instance_args)
print(f'''\033[96m
#################################
version {AIDE_VERSION}
### #### ######## ########
## ## ## ## ## ## {platform.platform()}
## ## ## ## ## ##
## ## ## ## ## ###### [config]
######### ## ## ## ## .> {configPath}
## ## ## ## ## ##
## ## #### ######## ######## [modules]
.> {aideModules}
#################################\033[0m
''')
statusOffset = LogDecorator.get_ljust_offset()
# load configuration
config = Config(None, verbose_start)
bottle.BaseRequest.MEMFILE_MAX = 1024**3 #TODO: make hyperparameter in config?
# connect to database
dbConnector = Database(config, verbose_start)
if check_v1_config:
# check if config file points to unmigrated v1 project
print('Checking database...'.ljust(statusOffset), end='')
hasAdminTable = dbConnector.execute(
'''
SELECT EXISTS (
SELECT FROM information_schema.tables
WHERE table_schema = 'aide_admin'
AND table_name = 'project'
);
''', None, 1)
if not hasAdminTable[0]['exists']:
# not (yet) migrated, raise Exception with instructions to ensure compatibility
LogDecorator.print_status('fail')
print(f'''
The current installation of AIDE:
database host: {config.getProperty('Database', 'host')}
database name: {config.getProperty('Database', 'name')}
schema: {config.getProperty('Database', 'schema', str, '(not specified)')}
points to an installation of the legacy AIDE v1.
If you wish to continue using AIDE v2, you have to upgrade the project accordingly.
For instructions to do so, see here:
https://github.com/microsoft/aerial_wildlife_detection/blob/multiProject/doc/upgrade_from_v1.md
''')
sys.exit(1)
else:
LogDecorator.print_status('ok')
# check if projects have been migrated
print('Checking projects...'.ljust(statusOffset), end='')
dbSchema = config.getProperty('Database', 'schema', str, None)
if dbSchema is not None:
isMigrated = dbConnector.execute(
'''
SELECT COUNT(*) AS cnt
FROM aide_admin.project
WHERE shortname = %s;
''', (dbSchema, ), 1)
if isMigrated is not None and len(
isMigrated) and isMigrated[0]['cnt'] == 0:
LogDecorator.print_status('warn')
print(f'''
WARNING: the selected configuration .ini file
("{os.environ['AIDE_CONFIG_PATH']}")
points to a project that has not yet been migrated to AIDE v2.
Details:
database host: {config.getProperty('Database', 'host')}
database name: {config.getProperty('Database', 'name')}
schema: {dbSchema}
If you wish to continue using AIDE v2 for this project, you have to upgrade it
to v2 accordingly.
For instructions to do so, see here:
https://github.com/microsoft/aerial_wildlife_detection/blob/multiProject/doc/upgrade_from_v1.md
''')
else:
LogDecorator.print_status('ok')
else:
LogDecorator.print_status('ok')
if migrate_database:
# bring AIDE up-to-date
print('Updating database...'.ljust(statusOffset), end='')
warnings, errors = migrate_aide(force_migrate)
if len(warnings) or len(errors):
if len(errors):
LogDecorator.print_status('fail')
else:
LogDecorator.print_status('warn')
print(
f'Warnings and/or errors occurred while updating AIDE to the latest version ({AIDE_VERSION}):'
)
print('\nWarnings:')
for w in warnings:
print(f'\t"{w}"')
print('\nErrors:')
for e in errors:
print(f'\t"{e}"')
if len(errors):
sys.exit(2)
else:
LogDecorator.print_status('ok')
# prepare bottle
app = Bottle()
# parse requested instances
instances = {}
# "singletons"
dbConnector = REGISTERED_MODULES['Database'](config, verbose_start)
userHandler = REGISTERED_MODULES['UserHandler'](config, app, dbConnector)
taskCoordinator = REGISTERED_MODULES['TaskCoordinator'](config, app,
dbConnector,
verbose_start)
taskCoordinator.addLoginCheckFun(userHandler.checkAuthenticated)
for i in instance_args:
moduleName = i.strip()
if moduleName == 'UserHandler':
continue
moduleClass = REGISTERED_MODULES[moduleName]
# verify
_verify_unique(instances, moduleClass)
# create instance
if moduleName == 'AIController':
instance = moduleClass(config, app, dbConnector, taskCoordinator,
verbose_start, passive_mode)
else:
instance = moduleClass(config, app, dbConnector, verbose_start)
instances[moduleName] = instance
# add authentication functionality
if hasattr(instance, 'addLoginCheckFun'):
instance.addLoginCheckFun(userHandler.checkAuthenticated)
# launch project meta modules
if moduleName == 'LabelUI':
aideAdmin = REGISTERED_MODULES['AIDEAdmin'](config, app,
dbConnector,
verbose_start)
aideAdmin.addLoginCheckFun(userHandler.checkAuthenticated)
reception = REGISTERED_MODULES['Reception'](config, app,
dbConnector)
reception.addLoginCheckFun(userHandler.checkAuthenticated)
configurator = REGISTERED_MODULES['ProjectConfigurator'](
config, app, dbConnector)
configurator.addLoginCheckFun(userHandler.checkAuthenticated)
statistics = REGISTERED_MODULES['ProjectStatistics'](config, app,
dbConnector)
statistics.addLoginCheckFun(userHandler.checkAuthenticated)
elif moduleName == 'FileServer':
from modules.DataAdministration.backend import celery_interface as daa_int
elif moduleName == 'AIController':
from modules.AIController.backend import celery_interface as aic_int
# launch model marketplace with AIController
modelMarketplace = REGISTERED_MODULES['ModelMarketplace'](
config, app, dbConnector, taskCoordinator)
modelMarketplace.addLoginCheckFun(userHandler.checkAuthenticated)
elif moduleName == 'AIWorker':
from modules.AIWorker.backend import celery_interface as aiw_int
# launch globally required modules
dataAdmin = REGISTERED_MODULES['DataAdministrator'](config, app,
dbConnector,
taskCoordinator)
dataAdmin.addLoginCheckFun(userHandler.checkAuthenticated)
staticFiles = REGISTERED_MODULES['StaticFileServer'](config, app,
dbConnector)
staticFiles.addLoginCheckFun(userHandler.checkAuthenticated)
if verbose_start:
print('\n')
return app
'''
Wrapper for the Celery message broker concerning
the Model Marketplace module.
2020-21 Benjamin Kellenberger
'''
import os
from celery import current_app
from .marketplaceWorker import ModelMarketplaceWorker
from modules.Database.app import Database
from util.configDef import Config
# initialize Model Marketplace worker
modules = os.environ['AIDE_MODULES']
config = Config()
worker = ModelMarketplaceWorker(config, Database(config))
@current_app.task(name='ModelMarketplace.shareModel')
def share_model(project, username, modelID, modelName, modelDescription, tags,
public, anonymous):
return worker.shareModel(project, username, modelID, modelName,
modelDescription, tags, public, anonymous)
@current_app.task(name='ModelMarketplace.importModelDatabase')
def import_model_database(modelID, project, username):
return worker.importModelDatabase(project, username, modelID)
'ALTER TABLE {schema}.labelclass ADD COLUMN IF NOT EXISTS keystroke SMALLINT UNIQUE;',
'ALTER TABLE {schema}.image ADD COLUMN IF NOT EXISTS last_requested TIMESTAMPTZ;'
]
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='Update AIde database structure.')
parser.add_argument('--settings_filepath', type=str, default='config/settings.ini', const=1, nargs='?',
help='Manual specification of the directory of the settings.ini file; only considered if environment variable unset (default: "config/settings.ini").')
args = parser.parse_args()
if not 'AIDE_CONFIG_PATH' in os.environ:
os.environ['AIDE_CONFIG_PATH'] = str(args.settings_filepath)
from util.configDef import Config
from modules import Database
config = Config()
dbConn = Database(config)
if dbConn.connectionPool is None:
raise Exception('Error connecting to database.')
dbSchema = config.getProperty('Database', 'schema')
# make modifications one at a time
for mod in MODIFICATIONS_sql:
dbConn.execute(mod.format(schema=dbSchema), None, None)
print('Project {} is now up-to-date for the latest changes in AIde.'.format(config.getProperty('Project', 'projectName')))
def main():
# parse arguments
parser = argparse.ArgumentParser(description='AIDE local model tester')
parser.add_argument('--project',
type=str,
required=True,
help='Project shortname to draw sample data from.')
parser.add_argument(
'--mode',
type=str,
required=True,
help=
'Evaluation mode (function to call). One of {"train", "inference"}.')
parser.add_argument(
'--modelLibrary',
type=str,
required=False,
help=
'Optional AI model library override. Provide a dot-separated Python import path here.'
)
parser.add_argument(
'--modelSettings',
type=str,
required=False,
help=
'Optional AI model settings override (absolute or relative path to settings file, or else "none" to not use any predefined settings).'
)
args = parser.parse_args()
#TODO: continue
assert args.mode.lower() in (
'train', 'inference'), f'"{args.mode}" is not a known evaluation mode.'
mode = args.mode.lower()
# initialize required modules
config = Config()
dbConnector = Database(config)
fileServer = FileServer(config).get_secure_instance(args.project)
aiw = AIWorker(config, dbConnector, True)
aicw = AIControllerWorker(config, None)
# check if AIDE file server is reachable
admin = AdminMiddleware(config, dbConnector)
connDetails = admin.getServiceDetails(True, False)
fsVersion = connDetails['FileServer']['aide_version']
if not isinstance(fsVersion, str):
# no file server running
raise Exception(
'ERROR: AIDE file server is not running, but required for running models. Make sure to launch it prior to running this script.'
)
elif fsVersion != AIDE_VERSION:
print(
f'WARNING: the AIDE version of File Server instance ({fsVersion}) differs from this one ({AIDE_VERSION}).'
)
# get model trainer instance and settings
queryStr = '''
SELECT ai_model_library, ai_model_settings FROM aide_admin.project
WHERE shortname = %s;
'''
result = dbConnector.execute(queryStr, (args.project, ), 1)
if result is None or not len(result):
raise Exception(
f'Project "{args.project}" could not be found in this installation of AIDE.'
)
modelLibrary = result[0]['ai_model_library']
modelSettings = result[0]['ai_model_settings']
customSettingsSpecified = False
if hasattr(args, 'modelSettings') and isinstance(
args.modelSettings, str) and len(args.modelSettings):
# settings override specified
if args.modelSettings.lower() == 'none':
modelSettings = None
customSettingsSpecified = True
elif not os.path.isfile(args.modelSettings):
print(
f'WARNING: model settings override provided, but file cannot be found ("{args.modelSettings}"). Falling back to project default ("{modelSettings}").'
)
else:
modelSettings = args.modelSettings
customSettingsSpecified = True
if hasattr(args, 'modelLibrary') and isinstance(
args.modelLibrary, str) and len(args.modelLibrary):
# library override specified; try to import it
try:
modelClass = helpers.get_class_executable(args.modelLibrary)
if modelClass is None:
raise
modelLibrary = args.modelLibrary
# re-check if current model settings are compatible; warn and set to None if not
if modelLibrary != result[0][
'ai_model_library'] and not customSettingsSpecified:
# project model settings are not compatible with provided model
print(
'WARNING: custom model library specified differs from the one currently set in project. Model settings will be set to None.'
)
modelSettings = None
except Exception as e:
print(
f'WARNING: model library override provided ("{args.modelLibrary}"), but could not be imported. Falling back to project default ("{modelLibrary}").'
)
# initialize instance
print(f'Using model library "{modelLibrary}".')
modelTrainer = aiw._init_model_instance(args.project, modelLibrary,
modelSettings)
stateDict = None #TODO: load latest unless override is specified?
# get data
data = aicw.get_training_images(project=args.project, maxNumImages=512)
data = __load_metadata(args.project, dbConnector, data[0],
(mode == 'train'))
# helper functions
def updateStateFun(state, message, done=None, total=None):
print(message, end='')
if done is not None and total is not None:
print(f': {done}/{total}')
else:
print('')
# launch task
if mode == 'train':
result = modelTrainer.train(stateDict, data, updateStateFun)
if result is None:
raise Exception(
'Training function must return an object (i.e., trained model state) to be stored in the database.'
)
elif mode == 'inference':
result = modelTrainer.inference(stateDict, data, updateStateFun)
if not 'AIDE_CONFIG_PATH' in os.environ:
os.environ['AIDE_CONFIG_PATH'] = str(args.settings_filepath)
import glob
from tqdm import tqdm
import datetime
from PIL import Image
from util.configDef import Config
from modules import Database
currentDT = datetime.datetime.now()
currentDT = '{}-{}-{} {}:{}:{}'.format(currentDT.year, currentDT.month,
currentDT.day, currentDT.hour,
currentDT.minute, currentDT.second)
config = Config()
dbConn = Database(config)
if dbConn.connectionPool is None:
raise Exception('Error connecting to database.')
dbSchema = config.getProperty('Database', 'schema')
valid_extensions = ('.jpg', '.jpeg', '.png', '.gif', '.tif', '.tiff',
'.bmp', '.ico', '.jfif', '.pjpeg', '.pjp')
# check if running on file server
imgBaseDir = config.getProperty('FileServer', 'staticfiles_dir')
if not os.path.isdir(imgBaseDir):
raise Exception(
'"{}" is not a valid directory on this machine. Are you running the script from the file server?'
.format(imgBaseDir))
from util.configDef import Config
from modules import Database
if args.label_folder == '':
args.label_folder = None
if args.label_folder is not None and not args.label_folder.endswith(
os.sep):
args.label_folder += os.sep
currentDT = datetime.datetime.now()
currentDT = '{}-{}-{} {}:{}:{}'.format(currentDT.year, currentDT.month,
currentDT.day, currentDT.hour,
currentDT.minute, currentDT.second)
config = Config()
dbConn = Database(config)
if not dbConn.canConnect():
raise Exception('Error connecting to database.')
# check if running on file server
imgBaseDir = config.getProperty('FileServer', 'staticfiles_dir')
if not os.path.isdir(imgBaseDir):
raise Exception(
'"{}" is not a valid directory on this machine. Are you running the script from the file server?'
.format(imgBaseDir))
if args.label_folder is not None and not os.path.isdir(args.label_folder):
raise Exception(
'"{}" is not a valid directory on this machine.'.format(
args.label_folder))
def _verify_unique(instances, moduleClass):
'''
Compares the newly requested module, address and port against
already launched modules on this instance.
Raises an Exception if another module from the same type has already been launched on this instance
'''
for i in instances:
if moduleClass.__class__.__name__ == i.__class__.__name__:
raise Exception(
'Module {} already launched on this server.'.format(
moduleClass.__class__.__name__))
# load configuration
config = Config()
demoMode = config.getProperty('Project', 'demoMode', type=bool, fallback=False)
# prepare bottle
app = Bottle()
# parse requested instances
instance_args = os.environ['AIDE_MODULES'].split(',')
instances = []
# create user handler
userHandler = REGISTERED_MODULES['UserHandler'](config, app)
for i in instance_args:
moduleName = i.strip()