def setupDB(): config = Config() dbConn = Database(config) # read SQL skeleton with open(os.path.join(os.getcwd(), 'setup/db_create.sql'), 'r') as f: sql = f.read() # fill in placeholders sql = sql.replace('&user', config.getProperty('Database', 'user')) # run SQL dbConn.execute(sql, None, None) # add admin user sql = ''' INSERT INTO aide_admin.user (name, email, hash, issuperuser) VALUES (%s, %s, %s, %s) ON CONFLICT (name) DO NOTHING; ''' adminPass = config.getProperty('Project', 'adminPassword') uHandler = UserHandling.backend.middleware.UserMiddleware(config) adminPass = uHandler._create_hash(adminPass.encode('utf8')) values = (config.getProperty('Project', 'adminName'), config.getProperty('Project', 'adminEmail'), adminPass, True,) dbConn.execute(sql, values, None)
# setup print('Setup...') if not 'AIDE_CONFIG_PATH' in os.environ: os.environ['AIDE_CONFIG_PATH'] = str(args.settings_filepath) from util.configDef import Config from modules.Database.app import Database config = Config() dbConn = Database(config) # load model class function print('Load and verify state dict...') from util.helpers import get_class_executable, current_time modelClass = getattr( get_class_executable( config.getProperty('AIController', 'model_lib_path')), 'model_class') # load state dict stateDict = torch.load(open(args.modelPath, 'rb')) # verify model state model = modelClass.loadFromStateDict(stateDict) # load class definitions from database classdef_db = {} labelClasses = dbConn.execute( 'SELECT * FROM {schema}.labelclass;'.format( schema=config.getProperty('Database', 'schema')), None, 'all') for lc in labelClasses: classdef_db[lc['id']] = lc
# setup print('Setup...\n') if not 'AIDE_CONFIG_PATH' in os.environ: os.environ['AIDE_CONFIG_PATH'] = str(args.settings_filepath) from tqdm import tqdm from util.configDef import Config from modules import Database config = Config() # setup DB connection dbConn = Database(config) if dbConn.connectionPool is None: raise Exception('Error connecting to database.') dbSchema = config.getProperty('Database', 'schema') # check if correct type of annotations annotationType = dbConn.execute( 'SELECT annotationtype FROM aide_admin.project WHERE shortname = %s;', (args.project, ), 1) if not len(annotationType): raise Exception( f'Project with name "{args.project}" could not be found in database.' ) annotationType = annotationType[0]['annotationtype'] exportAnnotations = args.export_annotations if exportAnnotations and not (annotationType == 'boundingBoxes'): print( 'Warning: project annotations are not bounding boxes; skipping annotation export...' )
import glob from tqdm import tqdm import datetime import numpy as np from PIL import Image import base64 from io import BytesIO from util.configDef import Config from modules import Database config = Config() # check if correct type of annotations exportAnnotations = args.export_annotations if exportAnnotations and not config.getProperty('Project', 'annotationType') == 'segmentationMasks': print('Warning: project annotations are not segmentation masks; skipping annotation export...') exportAnnotations = False # setup DB connection dbConn = Database(config) if dbConn.connectionPool is None: raise Exception('Error connecting to database.') dbSchema = config.getProperty('Database', 'schema') # check if valid file format provided valid_file_formats = ( 'jpg', 'jpeg', 'png',
args = parser.parse_args() # setup print('Setup...\n') if not 'AIDE_CONFIG_PATH' in os.environ: os.environ['AIDE_CONFIG_PATH'] = str(args.settings_filepath) from tqdm import tqdm from util.configDef import Config from modules import Database config = Config() # check if correct type of annotations exportAnnotations = args.export_annotations if exportAnnotations and not config.getProperty( 'Project', 'annotationType') == 'boundingBoxes': print( 'Warning: project annotations are not bounding boxes; skipping annotation export...' ) exportAnnotations = False # setup DB connection dbConn = Database(config) if dbConn.connectionPool is None: raise Exception('Error connecting to database.') dbSchema = config.getProperty('Database', 'schema') # query label definition labeldef = {} # label UUID : (name, index,) labeldef_inv = [] # list of label names in order (for export) labelQuery = dbConn.execute(
from tqdm import tqdm import datetime from PIL import Image from util.configDef import Config from modules import Database currentDT = datetime.datetime.now() currentDT = '{}-{}-{} {}:{}:{}'.format(currentDT.year, currentDT.month, currentDT.day, currentDT.hour, currentDT.minute, currentDT.second) config = Config() dbConn = Database(config) if dbConn.connectionPool is None: raise Exception('Error connecting to database.') dbSchema = config.getProperty('Database', 'schema') valid_extensions = ('.jpg', '.jpeg', '.png', '.gif', '.tif', '.tiff', '.bmp', '.ico', '.jfif', '.pjpeg', '.pjp') # check if running on file server imgBaseDir = config.getProperty('FileServer', 'staticfiles_dir') if not os.path.isdir(imgBaseDir): raise Exception( '"{}" is not a valid directory on this machine. Are you running the script from the file server?' .format(imgBaseDir)) if not imgBaseDir.endswith('/'): imgBaseDir += '/' # locate all images and their base names
help='Target filename for the model.') args = parser.parse_args() # setup print('Setup...') if not 'AIDE_CONFIG_PATH' in os.environ: os.environ['AIDE_CONFIG_PATH'] = str(args.settings_filepath) from util.configDef import Config from modules import Database config = Config() # setup DB connection dbConn = Database(config) if dbConn.connectionPool is None: raise Exception('Error connecting to database.') dbSchema = config.getProperty('Database', 'schema') # get state dict print('Retrieving model state...') stateDict_raw = dbConn.execute( 'SELECT statedict FROM {schema}.cnnstate WHERE partial IS FALSE ORDER BY timecreated DESC LIMIT 1;' .format(schema=config.getProperty('Database', 'schema')), None, 1) # convert from bytes and save to disk print('Saving model state...') stateDict_parsed = io.BytesIO(stateDict_raw[0]['statedict']) stateDict_parsed = torch.load(stateDict_parsed, map_location=lambda storage, loc: storage) torch.save(stateDict_parsed, open(args.target_file, 'wb'))
args = parser.parse_args() from util.configDef import Config from modules import Database # v2 config file config = Config() dbConn = Database(config) if dbConn.connectionPool is None: raise Exception('Error connecting to database.') # v1 config file v1Config = Config(args.settings_filepath) # db schema of v1 project dbSchema = v1Config.getProperty('Database', 'schema') projectName = v1Config.getProperty('Project', 'projectName') # verify we're running a database on v2 standards isV2 = dbConn.execute( ''' SELECT 1 FROM information_schema.tables WHERE table_schema = 'aide_admin' AND table_name = 'project'; ''', None, 'all') if isV2 is None or not len(isV2): # upgrade to v2 dbName = config.getProperty('Database', 'name') print( f'WARNING: Target database "{dbName}" has not (yet) been upgraded to the AIDE v2 schema; we will attempt to do this now...'
parser.add_argument( '--force_migrate', type=int, default=0, help= 'If set to 1, database upgrade will be enforced even if AIDE versions already match.' ) parser.add_argument( '--verbose', type=int, default=1, help='Set to 1 to print launch information to console.') args = parser.parse_args() try: app = assemble_server(args.verbose, args.check_v1, args.migrate_db, args.force_migrate, not bool(args.launch)) except Exception as e: print(e) sys.exit(1) if bool(args.launch): if args.verbose: print('Launching server...') config = Config(False) host = config.getProperty('Server', 'host') port = config.getProperty('Server', 'port') app.run(host=host, port=port) else: sys.exit(0)
parser.add_argument('--settings_filepath', type=str, default='config/settings.ini', const=1, nargs='?', help='Manual specification of the directory of the settings.ini file; only considered if environment variable unset (default: "config/settings.ini").') args = parser.parse_args() if not 'AIDE_CONFIG_PATH' in os.environ: os.environ['AIDE_CONFIG_PATH'] = args.settings_filepath config = Config() dbConn = Database(config) # read SQL skeleton with open(os.path.join(os.getcwd(), 'setup/db_create.sql'), 'r') as f: sql = f.read() # fill in placeholders sql = sql.replace('&user', config.getProperty('Database', 'user')) # run SQL dbConn.execute(sql, None, None) # add admin user sql = ''' INSERT INTO aide_admin.user (name, email, hash, issuperuser) VALUES (%s, %s, %s, %s) ''' adminPass = config.getProperty('Project', 'adminPassword') uHandler = UserHandling.backend.middleware.UserMiddleware(config) adminPass = uHandler._create_hash(adminPass.encode('utf8')) values = (config.getProperty('Project', 'adminName'), config.getProperty('Project', 'adminEmail'), adminPass, True,) dbConn.execute(sql, values, None)
elif module == 'fileserver': queues.append(Queue('FileServer')) elif module == 'aiworker': queues.append(Queue('AIWorker')) elif module == 'labelui': queues.append(Queue('ModelMarketplace')) #TODO: allow outsourcing? Move to FileServer? queues.extend([ Broadcast('aide_broadcast'), Queue('aide@'+celery.utils.nodenames.gethostname()) ]) app = Celery('AIDE', broker=config.getProperty('AIController', 'broker_URL'), #TODO backend=config.getProperty('AIController', 'result_backend')) #TODO app.conf.update( result_backend=config.getProperty('AIController', 'result_backend'), #TODO task_ignore_result=False, result_persistent=True, accept_content = ['json'], task_serializer = 'json', result_serializer = 'json', task_track_started = True, broker_pool_limit=None, # required to avoid peer connection resets broker_heartbeat = 0, # required to avoid peer connection resets worker_max_tasks_per_child = 1, # required to free memory (also CUDA) after each process # task_default_rate_limit = 3, #TODO worker_prefetch_multiplier = 1, #TODO task_acks_late = True,
) args = parser.parse_args() if not 'AIDE_CONFIG_PATH' in os.environ: os.environ['AIDE_CONFIG_PATH'] = args.settings_filepath config = Config() dbConn = Database(config) # read SQL skeleton with open(os.path.join(os.getcwd(), 'projectCreation/db_create.sql'), 'r') as f: sql = f.read() # fill in placeholders annoType_frontend = config.getProperty('Project', 'annotationType') annoFields_frontend, additionalTables_frontend = _constructAnnotationFields( annoType_frontend, 'annotation') if additionalTables_frontend is not None: sql += additionalTables_frontend annoType_backend = config.getProperty('Project', 'predictionType') annoFields_backend, additionalTables_backend = _constructAnnotationFields( annoType_backend, 'prediction') if additionalTables_backend is not None: sql += additionalTables_backend sql = sql.replace('&annotationFields', annoFields_frontend) sql = sql.replace('&predictionFields', annoFields_backend) sql = sql.replace('&dbName', config.getProperty('Database', 'name'))
print(f''' The current installation of AIDE: database host: {config.getProperty('Database', 'host')} database name: {config.getProperty('Database', 'name')} schema: {config.getProperty('Database', 'schema', str, '(not specified)')} points to an installation of the legacy AIDE v1. If you wish to continue using AIDE v2, you have to upgrade the project accordingly. For instructions to do so, see here: https://github.com/microsoft/aerial_wildlife_detection/blob/multiProject/doc/upgrade_from_v1.md ''') import sys sys.exit(1) # check if project has been migrated dbSchema = config.getProperty('Database', 'schema', str, None) if dbSchema is not None: isMigrated = dbConnector.execute( ''' SELECT COUNT(*) AS cnt FROM aide_admin.project WHERE shortname = %s; ''', (dbSchema, ), 1) if isMigrated is not None and len( isMigrated) and isMigrated[0]['cnt'] == 0: print(f''' WARNING: the selected configuration .ini file ("{os.environ['AIDE_CONFIG_PATH']}") points to a project that has not yet been migrated to AIDE v2. Details: database host: {config.getProperty('Database', 'host')}
description='Update AIde database structure.') parser.add_argument( '--settings_filepath', type=str, default='config/settings.ini', const=1, nargs='?', help= 'Manual specification of the directory of the settings.ini file; only considered if environment variable unset (default: "config/settings.ini").' ) args = parser.parse_args() if not 'AIDE_CONFIG_PATH' in os.environ: os.environ['AIDE_CONFIG_PATH'] = str(args.settings_filepath) from util.configDef import Config from modules import Database config = Config() dbConn = Database(config) if dbConn.connectionPool is None: raise Exception('Error connecting to database.') dbSchema = config.getProperty('Database', 'schema') # make modifications one at a time for mod in MODIFICATIONS_sql: dbConn.execute(mod.format(schema=dbSchema), None, None) print( 'Project {} is now up-to-date for the latest changes in AIde.'.format( config.getProperty('Project', 'projectName')))
from modules import Database if args.label_folder == '': args.label_folder = None if args.label_folder is not None and not args.label_folder.endswith('/'): args.label_folder += '/' currentDT = datetime.datetime.now() currentDT = '{}-{}-{} {}:{}:{}'.format(currentDT.year, currentDT.month, currentDT.day, currentDT.hour, currentDT.minute, currentDT.second) config = Config() dbConn = Database(config) if dbConn.connectionPool is None: raise Exception('Error connecting to database.') dbSchema = config.getProperty('Database', 'schema') valid_image_extensions = ( '.jpg', '.jpeg', '.png', '.gif', '.tif', '.tiff', '.bmp', '.ico', '.jfif', '.pjpeg', '.pjp' )
def assemble_server(verbose_start=True, check_v1_config=True, migrate_database=True, force_migrate=False, passive_mode=False): # force verbosity if any of the pre-flight checks is enabled verbose_start = any((verbose_start, check_v1_config, migrate_database)) instance_args = os.environ['AIDE_MODULES'].split(',') if verbose_start: configPath = os.environ['AIDE_CONFIG_PATH'] aideModules = ', '.join(instance_args) print(f'''\033[96m ################################# version {AIDE_VERSION} ### #### ######## ######## ## ## ## ## ## ## {platform.platform()} ## ## ## ## ## ## ## ## ## ## ## ###### [config] ######### ## ## ## ## .> {configPath} ## ## ## ## ## ## ## ## #### ######## ######## [modules] .> {aideModules} #################################\033[0m ''') statusOffset = LogDecorator.get_ljust_offset() # load configuration config = Config(None, verbose_start) bottle.BaseRequest.MEMFILE_MAX = 1024**3 #TODO: make hyperparameter in config? # connect to database dbConnector = Database(config, verbose_start) if check_v1_config: # check if config file points to unmigrated v1 project print('Checking database...'.ljust(statusOffset), end='') hasAdminTable = dbConnector.execute( ''' SELECT EXISTS ( SELECT FROM information_schema.tables WHERE table_schema = 'aide_admin' AND table_name = 'project' ); ''', None, 1) if not hasAdminTable[0]['exists']: # not (yet) migrated, raise Exception with instructions to ensure compatibility LogDecorator.print_status('fail') print(f''' The current installation of AIDE: database host: {config.getProperty('Database', 'host')} database name: {config.getProperty('Database', 'name')} schema: {config.getProperty('Database', 'schema', str, '(not specified)')} points to an installation of the legacy AIDE v1. If you wish to continue using AIDE v2, you have to upgrade the project accordingly. For instructions to do so, see here: https://github.com/microsoft/aerial_wildlife_detection/blob/multiProject/doc/upgrade_from_v1.md ''') sys.exit(1) else: LogDecorator.print_status('ok') # check if projects have been migrated print('Checking projects...'.ljust(statusOffset), end='') dbSchema = config.getProperty('Database', 'schema', str, None) if dbSchema is not None: isMigrated = dbConnector.execute( ''' SELECT COUNT(*) AS cnt FROM aide_admin.project WHERE shortname = %s; ''', (dbSchema, ), 1) if isMigrated is not None and len( isMigrated) and isMigrated[0]['cnt'] == 0: LogDecorator.print_status('warn') print(f''' WARNING: the selected configuration .ini file ("{os.environ['AIDE_CONFIG_PATH']}") points to a project that has not yet been migrated to AIDE v2. Details: database host: {config.getProperty('Database', 'host')} database name: {config.getProperty('Database', 'name')} schema: {dbSchema} If you wish to continue using AIDE v2 for this project, you have to upgrade it to v2 accordingly. For instructions to do so, see here: https://github.com/microsoft/aerial_wildlife_detection/blob/multiProject/doc/upgrade_from_v1.md ''') else: LogDecorator.print_status('ok') else: LogDecorator.print_status('ok') if migrate_database: # bring AIDE up-to-date print('Updating database...'.ljust(statusOffset), end='') warnings, errors = migrate_aide(force_migrate) if len(warnings) or len(errors): if len(errors): LogDecorator.print_status('fail') else: LogDecorator.print_status('warn') print( f'Warnings and/or errors occurred while updating AIDE to the latest version ({AIDE_VERSION}):' ) print('\nWarnings:') for w in warnings: print(f'\t"{w}"') print('\nErrors:') for e in errors: print(f'\t"{e}"') if len(errors): sys.exit(2) else: LogDecorator.print_status('ok') # prepare bottle app = Bottle() # parse requested instances instances = {} # "singletons" dbConnector = REGISTERED_MODULES['Database'](config, verbose_start) userHandler = REGISTERED_MODULES['UserHandler'](config, app, dbConnector) taskCoordinator = REGISTERED_MODULES['TaskCoordinator'](config, app, dbConnector, verbose_start) taskCoordinator.addLoginCheckFun(userHandler.checkAuthenticated) for i in instance_args: moduleName = i.strip() if moduleName == 'UserHandler': continue moduleClass = REGISTERED_MODULES[moduleName] # verify _verify_unique(instances, moduleClass) # create instance if moduleName == 'AIController': instance = moduleClass(config, app, dbConnector, taskCoordinator, verbose_start, passive_mode) else: instance = moduleClass(config, app, dbConnector, verbose_start) instances[moduleName] = instance # add authentication functionality if hasattr(instance, 'addLoginCheckFun'): instance.addLoginCheckFun(userHandler.checkAuthenticated) # launch project meta modules if moduleName == 'LabelUI': aideAdmin = REGISTERED_MODULES['AIDEAdmin'](config, app, dbConnector, verbose_start) aideAdmin.addLoginCheckFun(userHandler.checkAuthenticated) reception = REGISTERED_MODULES['Reception'](config, app, dbConnector) reception.addLoginCheckFun(userHandler.checkAuthenticated) configurator = REGISTERED_MODULES['ProjectConfigurator']( config, app, dbConnector) configurator.addLoginCheckFun(userHandler.checkAuthenticated) statistics = REGISTERED_MODULES['ProjectStatistics'](config, app, dbConnector) statistics.addLoginCheckFun(userHandler.checkAuthenticated) elif moduleName == 'FileServer': from modules.DataAdministration.backend import celery_interface as daa_int elif moduleName == 'AIController': from modules.AIController.backend import celery_interface as aic_int # launch model marketplace with AIController modelMarketplace = REGISTERED_MODULES['ModelMarketplace']( config, app, dbConnector, taskCoordinator) modelMarketplace.addLoginCheckFun(userHandler.checkAuthenticated) elif moduleName == 'AIWorker': from modules.AIWorker.backend import celery_interface as aiw_int # launch globally required modules dataAdmin = REGISTERED_MODULES['DataAdministrator'](config, app, dbConnector, taskCoordinator) dataAdmin.addLoginCheckFun(userHandler.checkAuthenticated) staticFiles = REGISTERED_MODULES['StaticFileServer'](config, app, dbConnector) staticFiles.addLoginCheckFun(userHandler.checkAuthenticated) if verbose_start: print('\n') return app
import os from configparser import ConfigParser from celery import Celery from celery import signals from celery.bin import Option from modules.AIWorker.app import AIWorker from util.configDef import Config # parse system config if not 'AIDE_CONFIG_PATH' in os.environ: raise ValueError('Missing system environment variable "AIDE_CONFIG_PATH".') config = Config() if config.getProperty('Project', 'demoMode', type=bool, fallback=False): raise Exception( 'AIController and AIWorkers cannot be launched in demo mode.') app = Celery('AIController', broker=config.getProperty('AIController', 'broker_URL'), backend=config.getProperty('AIController', 'result_backend')) app.conf.update( result_backend=config.getProperty('AIController', 'result_backend'), task_ignore_result=False, result_persistent=True, accept_content=['json'], task_serializer='json', result_serializer='json', task_track_started=True, broker_heartbeat=0, # required to avoid peer connection resets
'ALTER TABLE {schema}.labelclass ADD COLUMN IF NOT EXISTS keystroke SMALLINT UNIQUE;', 'ALTER TABLE {schema}.image ADD COLUMN IF NOT EXISTS last_requested TIMESTAMPTZ;' ] if __name__ == '__main__': parser = argparse.ArgumentParser(description='Update AIde database structure.') parser.add_argument('--settings_filepath', type=str, default='config/settings.ini', const=1, nargs='?', help='Manual specification of the directory of the settings.ini file; only considered if environment variable unset (default: "config/settings.ini").') args = parser.parse_args() if not 'AIDE_CONFIG_PATH' in os.environ: os.environ['AIDE_CONFIG_PATH'] = str(args.settings_filepath) from util.configDef import Config from modules import Database config = Config() dbConn = Database(config) if dbConn.connectionPool is None: raise Exception('Error connecting to database.') dbSchema = config.getProperty('Database', 'schema') # make modifications one at a time for mod in MODIFICATIONS_sql: dbConn.execute(mod.format(schema=dbSchema), None, None) print('Project {} is now up-to-date for the latest changes in AIde.'.format(config.getProperty('Project', 'projectName')))
if args.label_folder is not None and not args.label_folder.endswith( os.sep): args.label_folder += os.sep currentDT = datetime.datetime.now() currentDT = '{}-{}-{} {}:{}:{}'.format(currentDT.year, currentDT.month, currentDT.day, currentDT.hour, currentDT.minute, currentDT.second) config = Config() dbConn = Database(config) if not dbConn.canConnect(): raise Exception('Error connecting to database.') # check if running on file server imgBaseDir = config.getProperty('FileServer', 'staticfiles_dir') if not os.path.isdir(imgBaseDir): raise Exception( '"{}" is not a valid directory on this machine. Are you running the script from the file server?' .format(imgBaseDir)) if args.label_folder is not None and not os.path.isdir(args.label_folder): raise Exception( '"{}" is not a valid directory on this machine.'.format( args.label_folder)) if not imgBaseDir.endswith(os.sep): imgBaseDir += os.sep # parse class names and indices if args.label_folder is not None:
def _verify_unique(instances, moduleClass): ''' Compares the newly requested module, address and port against already launched modules on this instance. Raises an Exception if another module from the same type has already been launched on this instance ''' for i in instances: if moduleClass.__class__.__name__ == i.__class__.__name__: raise Exception( 'Module {} already launched on this server.'.format( moduleClass.__class__.__name__)) # load configuration config = Config() demoMode = config.getProperty('Project', 'demoMode', type=bool, fallback=False) # prepare bottle app = Bottle() # parse requested instances instance_args = os.environ['AIDE_MODULES'].split(',') instances = [] # create user handler userHandler = REGISTERED_MODULES['UserHandler'](config, app) for i in instance_args: moduleName = i.strip() if moduleName == 'UserHandler':