def _do_inference(self, process): # send job task_id = self.messageProcessor.task_id() result = process.apply_async(task_id=task_id, ignore_result=False, result_extended=True, headers={ 'type': 'inference', 'submitted': str(current_time()) }) # start listener self.messageProcessor.register_job(result, 'inference', None) return
def __add_worker_task(self, task): result = AsyncResult(task['id']) if not task['id'] in self.messages: try: timeSubmitted = datetime.fromtimestamp(time.time() - (kombu.five.monotonic() - t['time_start'])) except: timeSubmitted = str(current_time()) #TODO: dirty hack to make failsafe with UI self.messages[task['id']] = { 'type': ('train' if 'train' in task['name'] else 'inference'), #TODO 'submitted': timeSubmitted, 'status': celery.states.PENDING, 'meta': {'message':'job at worker'} } #TODO: needed? if result.ready(): result.forget()
def listImages(project): ''' Returns a list of images and various properties and statistics (id, filename, viewcount, etc.), all filterable by date and value ranges. ''' if not self.loginCheck(project=project, admin=True): abort(401, 'forbidden') # parse parameters now = helpers.current_time() username = html.escape(request.get_cookie('username')) params = request.json folder = (params['folder'] if 'folder' in params else None) imageAddedRange = self._parse_range(params, 'imageAddedRange', datetime.time.min, now) lastViewedRange = self._parse_range(params, 'lastViewedRange', datetime.time.min, now) viewcountRange = self._parse_range(params, 'viewcountRange', 0, 1e9) numAnnoRange = self._parse_range(params, 'numAnnoRange', 0, 1e9) numPredRange = self._parse_range(params, 'numPredRange', 0, 1e9) orderBy = (params['orderBy'] if 'orderBy' in params else None) order = (params['order'].lower() if 'order' in params else None) if 'start_from' in params: startFrom = params['start_from'] if isinstance(startFrom, str): startFrom = uuid.UUID(startFrom) else: startFrom = None limit = (params['limit'] if 'limit' in params else None) # get images result = self.middleware.listImages(project, username, folder, imageAddedRange, lastViewedRange, viewcountRange, numAnnoRange, numPredRange, orderBy, order, startFrom, limit) return {'response': result}
def pollNow(self): i = self.celery_app.control.inspect() stats = i.stats() if stats is not None and len(stats): active_tasks = i.active() if active_tasks is None: return for key in active_tasks.keys(): taskList = active_tasks[key] for t in taskList: taskID = t['id'] taskType = t['name'] if 'project' not in t['kwargs']: # non-project-specific task; ignore (TODO) continue project = t['kwargs']['project'] if not project in self.messages: self.messages[project] = {} if not taskID in self.messages[project]: # task got lost (e.g. due to server restart); re-add try: timeSubmitted = datetime.fromtimestamp( time.time() - (kombu.five.monotonic() - t['time_start'])) except: timeSubmitted = current_time( ) #TODO: dirty hack to make failsafe with UI self.messages[project][taskID] = { 'type': taskType, 'submitted': str(timeSubmitted), 'status': celery.states.PENDING, 'meta': { 'message': 'job at worker' } } job = celery.result.AsyncResult( taskID) #TODO: task.ready() if not project in self.jobs: self.jobs[project] = [] self.jobs[project].append(job)
def pollNow(self): i = self.celery_app.control.inspect() stats = i.stats() if stats is not None and len(stats): active_tasks = i.active() for key in active_tasks.keys(): taskList = active_tasks[key] for t in taskList: taskID = t['id'] if not taskID in self.messages: # task got lost (e.g. due to server restart); re-add try: timeSubmitted = datetime.fromtimestamp(time.time() - (kombu.five.monotonic() - t['time_start'])) except: timeSubmitted = str(current_time()) #TODO: dirty hack to make failsafe with UI self.messages[taskID] = { 'type': ('train' if 'train' in t['name'] else 'inference'), #TODO 'submitted': timeSubmitted, 'status': celery.states.PENDING, 'meta': {'message':'job at worker'} } job = celery.result.AsyncResult(taskID) #TODO: task.ready() self.jobs.append(job)
def prepareModelDownload(self, project, modelID, username, source='marketplace', modelName=None, modelDescription='', modelTags=[]): ''' Attempts to create a file from a model state, either from the database (if "modelID" is a UUID) or directly from one of the built-in configu- rations (if it is a str, corresponding to the built-in name). Constructs an AIDE Model Marketplace-compliant definition JSON file in the process, optionally wrapped together with a state dict binary file in a zip file, if needed. Saves the file to a temporary directory and returns the file path as a Celery result if successful. Various parameters like "modelName", "modelDescription", etc., are only needed if model state is pulled from the project's table and therefore does not automatically come with these metadata. ''' # try to parse modelID as UUID try: modelID = UUID(modelID) except: # need to load model from built-ins pass if isinstance(modelID, UUID): # load from database modelDefinition = { 'aide_model_version': MODEL_MARKETPLACE_VERSION, 'name': (modelName if modelName is not None else str(modelID)), 'description': modelDescription, 'tags': modelTags } if source.lower() == 'marketplace': # load from marketplace table queryStr = ''' SELECT name, description, author, timeCreated, tags, labelclasses, annotationType, predictionType, model_library, --TODO: more? stateDict FROM aide_admin.modelMarketplace WHERE id = %s; ''' elif source.lower() == 'project': # load from project table queryStr = sql.SQL(''' SELECT timeCreated, model_library, stateDict FROM {id_cnnstate} WHERE id = %s; ''').format(id_cnnstate=sql.Identifier(project, 'cnnstate')) result = self.dbConnector.execute(queryStr, (modelID, ), 1) if result is None or not len(result): raise Exception( f'Model state with id "{str(modelID)}" could not be found in database.' ) result = result[0] for key in result.keys(): if key == 'timecreated': modelDefinition['time_created'] = result[key].timestamp() elif key == 'labelclasses': modelDefinition[key] = json.loads(result[key]) elif key == 'tags': modelDefinition[key] = result[key].split(';;') elif key == 'model_library': modelDefinition['ai_model_library'] = result[key] elif key == 'annotationtype': modelDefinition['annotation_type'] = result[key] elif key == 'predictiontype': modelDefinition['prediction_type'] = result[key] elif key in modelDefinition: modelDefinition[key] = result[key] if 'author' not in modelDefinition: # append user name as author modelDefinition['author'] = username # get model implementation meta data modelImplementationID = modelDefinition['ai_model_library'] if modelImplementationID not in PREDICTION_MODELS: raise Exception( f'Model implementation with ID "{modelImplementationID}" is not installed in this instance of AIDE.' ) modelMeta = PREDICTION_MODELS[modelImplementationID] if 'annotation_type' not in modelDefinition: modelDefinition['annotation_type'] = modelMeta[ 'annotationType'] if 'prediction_type' not in modelDefinition: modelDefinition['prediction_type'] = modelMeta[ 'predictionType'] if 'labelclasses' not in modelDefinition: # query from current project and just append as a list queryStr = sql.SQL( 'SELECT name FROM {} ORDER BY name;').format( sql.Identifier(project, 'labelclass')) labelClasses = self.dbConnector.execute( queryStr, (project, ), 'all') labelClasses = [l['name'] for l in labelClasses] modelDefinition['labelclasses'] = labelClasses # model settings: grab from project if possible #TODO # state dict stateDict = result['statedict'] # prepare temporary output file destName = modelDefinition['name'] + '_' + current_time().strftime( '%Y-%m-%d_%H-%M-%S') for char in FILENAMES_PROHIBITED_CHARS: destName = destName.replace(char, '_') # write contents if stateDict is None: destName += '.json' json.dump(modelDefinition, open(os.path.join(self.tempDir, destName), 'w')) else: destName += '.zip' with zipfile.ZipFile(os.path.join(self.tempDir, destName), 'w', zipfile.ZIP_DEFLATED) as f: f.writestr('modelDefinition.json', json.dumps(modelDefinition)) bio = io.BytesIO(stateDict) f.writestr('modelState.bin', bio.getvalue()) return destName else: # built-in model; copy to temp dir and return path directly sourcePath = modelID.replace('aide://', '').strip('/') if not os.path.exists(sourcePath): raise Exception( f'Model file "{sourcePath}" could not be found.') _, fileName = os.path.split(sourcePath) destPath = os.path.join(self.tempDir, fileName) if not os.path.exists(destPath): shutil.copyfile(sourcePath, destPath) return destPath
def _import_model_state_file(self, project, modelURI, modelDefinition, stateDict=None, public=True, anonymous=False, namePolicy='skip', customName=None): ''' Receives two files: - "modelDefinition": JSON-encoded metadata file - "stateDict" (optional): BytesIO file containing model state Parses the files for completeness and integrity and attempts to launch a model with them. If all checks are successful, the new model state is inserted into the database and the UUID is returned. Parameter "namePolicy" can have one of three values: - "skip" (default): skips model import if another with the same name already exists in the Model Marketplace database - "increment": appends or increments a number at the end of the name if it is already found in the database - "custom": uses the value under "customName" ''' # check inputs if not isinstance(modelDefinition, dict): try: if isinstance(modelDefinition, bytes): modelDefinition = json.load(io.BytesIO(modelDefinition)) elif isinstance(modelDefinition, str): modelDefinition = json.loads(modelDefinition) except Exception as e: raise Exception( f'Invalid model state definition file (message: "{e}").') if stateDict is not None and not isinstance(stateDict, bytes): raise Exception( 'Invalid model state dict provided (not a binary file).') # check naming policy namePolicy = str(namePolicy).lower() assert namePolicy in ( 'skip', 'increment', 'custom'), f'Invalid naming policy "{namePolicy}".' if namePolicy == 'skip': # check if model with same name already exists modelID = self.getModelIdByName(modelDefinition['name']) if modelID is not None: return modelID elif namePolicy == 'custom': assert isinstance( customName, str) and len(customName), 'Invalid custom name provided.' # check if name is available if self.getModelIdByName(customName) is not None: raise Exception(f'Custom name "{customName}" is unavailable.') # project metadata projectMeta = self.dbConnector.execute( ''' SELECT annotationType, predictionType FROM aide_admin.project WHERE shortname = %s; ''', (project, ), 1) if projectMeta is None or not len(projectMeta): raise Exception( f'Project with shortname "{project}" not found in database.') projectMeta = projectMeta[0] # check fields for field in ('author', 'citation_info', 'license'): if field not in modelDefinition: modelDefinition[field] = None for field in self.MODEL_STATE_REQUIRED_FIELDS: if field not in modelDefinition: raise Exception(f'Missing field "{field}" in AIDE JSON file.') if field == 'aide_model_version': # check model definition version modelVersion = modelDefinition['aide_model_version'] if isinstance(modelVersion, str): if not modelVersion.isnumeric(): raise Exception( f'Invalid AIDE model version "{modelVersion}" in JSON file.' ) else: modelVersion = float(modelVersion) modelVersion = float(modelVersion) if modelVersion > self.MAX_AIDE_MODEL_VERSION: raise Exception( f'Model state contains a newer model version than supported by this installation of AIDE ({modelVersion} > {self.MAX_AIDE_MODEL_VERSION}).\nPlease update AIDE to the latest version.' ) if field == 'ai_model_library': # check if model library is installed modelLibrary = modelDefinition[field] if modelLibrary not in PREDICTION_MODELS: raise Exception( f'Model library "{modelLibrary}" is not installed in this instance of AIDE.' ) # check if annotation and prediction types match if projectMeta['annotationtype'] not in PREDICTION_MODELS[ modelLibrary]['annotationType']: raise Exception( 'Project\'s annotation type is not compatible with this model state.' ) if projectMeta['predictiontype'] not in PREDICTION_MODELS[ modelLibrary]['predictionType']: raise Exception( 'Project\'s prediction type is not compatible with this model state.' ) # check if model state URI provided if stateDict is None and hasattr(modelDefinition, 'ai_model_state_uri'): stateDictURI = modelDefinition['ai_model_state_uri'] try: if stateDictURI.lower().startswith('aide://'): # load from disk stateDictPath = stateDictURI.replace('aide://', '').strip('/') if not os.path.isfile(stateDictPath): raise Exception( f'Model state file path provided ("{stateDictPath}"), but file could not be found.' ) with open(stateDictPath, 'rb') as f: stateDict = f.read() #TODO: BytesIO instead else: # network import with request.urlopen(stateDictURI) as f: stateDict = f.read( ) #TODO: progress bar; load in chunks; etc. except Exception as e: raise Exception( f'Model state URI provided ("{stateDictURI}"), but could not be loaded (message: "{str(e)}").' ) # model name modelName = modelDefinition['name'] if namePolicy == 'increment': # check if model name needs to be incremented allNames = self.dbConnector.execute( 'SELECT name FROM "aide_admin".modelmarketplace;', None, 'all') allNames = (set([a['name'].strip() for a in allNames]) if allNames is not None else set()) if modelName.strip() in allNames: startIdx = 1 insertPos = len(modelName) trailingNumber = re.findall(' \d+$', modelName.strip()) if len(trailingNumber): startIdx = int(trailingNumber[0]) insertPos = modelName.rfind(str(startIdx)) - 1 while modelName.strip() in allNames: modelName = modelName[:insertPos] + f' {startIdx}' startIdx += 1 elif namePolicy == 'custom': modelName = customName # remaining parameters modelAuthor = modelDefinition['author'] modelDescription = (modelDefinition['description'] if 'description' in modelDefinition else None) modelTags = (';;'.join(modelDefinition['tags']) if 'tags' in modelDefinition else None) labelClasses = modelDefinition['labelclasses'] #TODO: parse? if not isinstance(labelClasses, str): labelClasses = json.dumps(labelClasses) modelOptions = (modelDefinition['ai_model_settings'] if 'ai_model_settings' in modelDefinition else None) modelLibrary = modelDefinition['ai_model_library'] alCriterion_library = (modelDefinition['alcriterion_library'] if 'alcriterion_library' in modelDefinition else None) annotationType = PREDICTION_MODELS[modelLibrary][ 'annotationType'] #TODO predictionType = PREDICTION_MODELS[modelLibrary][ 'predictionType'] #TODO citationInfo = modelDefinition['citation_info'] license = modelDefinition['license'] if not isinstance(annotationType, str): annotationType = ','.join(annotationType) if not isinstance(predictionType, str): predictionType = ','.join(predictionType) timeCreated = (modelDefinition['time_created'] if 'time_created' in modelDefinition else None) try: timeCreated = datetime.fromtimestamp(timeCreated) except: timeCreated = current_time() # try to launch model with data try: modelClass = get_class_executable(modelLibrary) modelClass(project=project, config=self.config, dbConnector=self.dbConnector, fileServer=FileServer( self.config).get_secure_instance(project), options=modelOptions) # verify options if modelOptions is not None: try: optionMeta = modelClass.verifyOptions(modelOptions) if 'options' in optionMeta: modelOptions = optionMeta['options'] #TODO: parse warnings and errors except: # model library does not support option verification pass if isinstance(modelOptions, dict): modelOptions = json.dumps(modelOptions) except Exception as e: raise Exception( f'Model from imported state could not be launched (message: "{str(e)}").' ) # import model state into Model Marketplace success = self.dbConnector.execute( ''' INSERT INTO aide_admin.modelMarketplace (name, description, tags, labelclasses, author, statedict, model_library, model_settings, alCriterion_library, annotationType, predictionType, citation_info, license, timeCreated, origin_project, origin_uuid, origin_uri, public, anonymous) VALUES %s RETURNING id; ''', [(modelName, modelDescription, modelTags, labelClasses, modelAuthor, stateDict, modelLibrary, modelOptions, alCriterion_library, annotationType, predictionType, citationInfo, license, timeCreated, project, None, modelURI, public, anonymous)], 1) if success is None or not len(success): #TODO: temporary fix to get ID: try again by re-querying DB success = self.dbConnector.execute( ''' SELECT id FROM aide_admin.modelMarketplace WHERE name = %s; ''', (modelName, ), 1) if success is None or not len(success): raise Exception( 'Model could not be imported into Model Marketplace.') # model import to Marketplace successful; now import to projet return success[0]['id']
def requestDownload(project): ''' Parses request parameters and then assembles project- related metadata (annotations, predictions, etc.) by storing them as files on the server in a temporary directory. Returns the download links to those temporary files. ''' #TODO: allow download for non-admins? if not self.loginCheck(project=project, admin=True): abort(401, 'forbidden') # parse parameters try: params = request.json dataType = params['dataType'] if 'dateRange' in params: dateRange = [] if 'start' in params['dateRange']: dateRange.append(params['dateRange']['start']) else: dateRange.append(0) if 'end' in params['dateRange']: dateRange.append(params['dateRange']['end']) else: dateRange.append(helpers.current_time()) else: dateRange = None if 'users' in params: userList = params['users'] else: userList = None # extra query fields if 'extra_fields' in params: extraFields = params['extra_fields'] else: extra_fields = { 'meta': False } # advanced parameters for segmentation masks if 'segmask_filename' in params: segmaskFilenameOptions = params['segmask_filename'] else: segmaskFilenameOptions = { 'baseName': 'filename', 'prefix': None, 'suffix': None } if 'segmask_encoding' in params: segmaskEncoding = params['segmask_encoding'] else: segmaskEncoding = 'rgb' taskID = self.middleware.prepareDataDownload(project, dataType, userList, dateRange, extraFields, segmaskFilenameOptions, segmaskEncoding) return {'response': taskID} except Exception as e: abort(401, str(e))
confirmation = 'Y' elif 'n' in confirmation.lower(): confirmation = 'n' forceManualAssignment = True else: raise Exception('Invalid value') except: confirmation = None # export model state print('Exporting model state...') bio = io.BytesIO() stateDict = model.getStateDict() # append new label class map definition stateDict['labelclassMap'] = classdef_final torch.save(stateDict, bio) stateDict = bio.getvalue() # commit to DB print('Committing to DB...') sql = ''' INSERT INTO {schema}.cnnstate (timeCreated, stateDict, partial) VALUES ( %s, %s, FALSE); '''.format(schema=config.getProperty('Database', 'schema')) now = current_time() dbConn.execute(sql, ( now, stateDict, ), None)
def start_training(self, minTimestamp='lastState', minNumAnnoPerImage=0, maxNumImages=None, maxNumWorkers=-1): ''' Initiates a training round for the model, based on the set of data (images, annotations) as specified in the parameters. Distributes data to the set of registered AIWorker instan- ces, which then call the 'train' function of the AI model given in the configuration. Upon training completion, the model states as returned by the function, and eventually the AIWorker instances are collected, and the AIController calls the 'average_states' function of the AI model to get one single, most recent state. This is finally inserted to the data- base. Note that this function only loads the annotation data from the database, but not the images. Retrieving images is part of the AI model's 'train' function. TODO: feature vectors? Input parameters: - minTimestamp: Defines the earliest point in time of the annotations to be considered for model training. May take one of the following values: - 'lastState' (default): Limits the annotations to those made after the time- stamp of the latest model state. If no model state is found, all annotations are considered. - None, -1, or 'all': Includes all annotations. - (a datetime object): Includes annotations made after a custom timestamp. - minNumAnnoPerImage: Minimum number of annotations per image to be considered for training. This may be useful for e.g. detection tasks with a lot of false alarms in order to limit the "forgetting factor" of the model subject to training. - maxNumImages: Maximum number of images to train on at a time. - maxNumWorkers: Specify the maximum number of workers to distribute training to. If set to 1, the model is trained on just one worker (no model state averaging appended). If set to a number, that number of workers (up to the maximum number of connected) is consulted for training the model. Upon completion, all model state dictionaries are averaged by one random worker. If set to -1, all connected workers are considered. //TODO: load balancing? Returns: - A dict with a status message. May take one of the following: - TODO: status ok, fail, no annotations, etc. Make threaded so that it immediately returns something. ''' process, numWorkers = self._get_training_job_signature( minTimestamp=minTimestamp, minNumAnnoPerImage=minNumAnnoPerImage, maxNumImages=maxNumImages, maxNumWorkers=maxNumWorkers) # submit job task_id = self.messageProcessor.task_id() if numWorkers > 1: # also append average model states job job = process.apply_async( task_id=task_id, ignore_result=False, result_extended=True, headers={ 'type': 'train', 'submitted': str(current_time()) }, link=celery_interface.call_average_model_states.s()) else: job = process.apply_async(task_id=task_id, ignore_result=False, result_extended=True, headers={ 'type': 'train', 'submitted': str(current_time()) }) # start listener self.messageProcessor.register_job(job, 'train', self._training_completed) return 'ok'
def get_project_info(self, username=None, isSuperUser=False): ''' Returns metadata about projects: - names - whether the projects are archived or not - links to interface (if user is authenticated) - requests for authentication (else) TODO - links to stats and review page (if admin) TODO - etc. ''' now = current_time() if isSuperUser: authStr = sql.SQL('') queryVals = None elif username is not None: authStr = sql.SQL( 'WHERE username = %s OR demoMode = TRUE OR isPublic = TRUE') queryVals = (username, ) else: authStr = sql.SQL('WHERE demoMode = TRUE OR isPublic = TRUE') queryVals = None queryStr = sql.SQL('''SELECT shortname, name, description, archived, username, isAdmin, admitted_until, blocked_until, annotationType, predictionType, isPublic, demoMode, interface_enabled, archived, ai_model_enabled, ai_model_library, CASE WHEN username = owner THEN TRUE ELSE FALSE END AS is_owner FROM aide_admin.project AS proj FULL OUTER JOIN (SELECT * FROM aide_admin.authentication ) AS auth ON proj.shortname = auth.project {authStr}; ''').format(authStr=authStr) result = self.dbConnector.execute(queryStr, queryVals, 'all') response = {} if result is not None and len(result): for r in result: projShort = r['shortname'] if not projShort in response: userAdmitted = True if r['admitted_until'] is not None and r[ 'admitted_until'] < now: userAdmitted = False if r['blocked_until'] is not None and r[ 'blocked_until'] >= now: userAdmitted = False response[projShort] = { 'name': r['name'], 'description': r['description'], 'archived': r['archived'], 'isOwner': r['is_owner'], 'annotationType': r['annotationtype'], 'predictionType': r['predictiontype'], 'isPublic': r['ispublic'], 'demoMode': r['demomode'], 'interface_enabled': r['interface_enabled'] and not r['archived'], 'aiModelEnabled': r['ai_model_enabled'], 'aiModelSelected': (isinstance(r['ai_model_library'], str) and len(r['ai_model_library']) > 0), 'userAdmitted': userAdmitted } if isSuperUser: response[projShort]['role'] = 'super user' elif username is not None and r['username'] == username: if r['isadmin']: response[projShort]['role'] = 'admin' else: response[projShort]['role'] = 'member' return response
def _current_time(self): return current_time()
def _check_authorized(self, project, username, admin, return_all=False): ''' Verifies whether a user has access rights to a project. If "return_all" is set to True, a dict with the following bools is returned: - enrolled: if the user is member of the project - isAdmin: if the user is a project administrator - isPublic: if the project is publicly visible (*) - demoMode: if the project runs in demo mode (*) (* note that these are here for convenience, but do not count as authorization tokens) If "return_all" is False, only a single bool is returned, with criteria as follows: - if "admin" is set to True, the user must be a project admini- strator - else, the user must be enrolled, admitted, and not blocked for the current date and time In this case, options like the demo mode and public flag are not relevant for the decision. ''' now = current_time() response = {'enrolled': False, 'isAdmin': False, 'isPublic': False} queryStr = sql.SQL(''' SELECT * FROM aide_admin.authentication AS auth JOIN (SELECT shortname, demoMode, isPublic FROM aide_admin.project) AS proj ON auth.project = proj.shortname WHERE project = %s AND username = %s; ''') try: result = self.dbConnector.execute(queryStr, ( project, username, ), 1) if len(result): response['isAdmin'] = result[0]['isadmin'] response['isPublic'] = result[0]['ispublic'] admitted_until = True blocked_until = False if result[0]['admitted_until'] is not None: admitted_until = (result[0]['admitted_until'] >= now) if result[0]['blocked_until'] is not None: blocked_until = (result[0]['blocked_until'] >= now) response['enrolled'] = (admitted_until and not blocked_until) except: # no results to fetch: user is not authenticated pass # check if super user superUser = self._check_user_privileges(username, superuser=True) if superUser: response['enrolled'] = True response['isAdmin'] = True if return_all: return response else: if admin: return response['isAdmin'] else: return response['enrolled']
def pollNow(self): self.worker_status = {} i = self.celery_app.control.inspect() stats = i.stats() if stats is not None and len(stats): active_tasks = i.active() if active_tasks is None: return # worker status for key in stats: workerName = key.replace('celery@', '') activeTasks = [] if key in active_tasks: for task in active_tasks[key]: # append task if of correct project if 'project' in task['kwargs']: activeTasks.append({ 'id': task['id'], 'project': task['kwargs']['project'] }) # # also add active tasks to current set if not already there # self.__add_worker_task(task) self.worker_status[workerName] = { 'active_tasks': activeTasks, # 'scheduled_tasks': scheduled_tasks[key] } # active tasks for key in active_tasks.keys(): taskList = active_tasks[key] for t in taskList: taskID = t['id'] taskType = t['name'] if 'project' not in t['kwargs']: # non-project-specific task; ignore (TODO) continue project = t['kwargs']['project'] if not project in self.messages: self.messages[project] = {} if not taskID in self.messages[project]: # task got lost (e.g. due to server restart); re-add try: timeSubmitted = datetime.fromtimestamp( time.time() - (time.monotonic() - t['time_start'])) except: timeSubmitted = current_time( ) #TODO: dirty hack to make failsafe with UI self.messages[project][taskID] = { 'type': taskType, 'submitted': str(timeSubmitted), 'status': celery.states.PENDING, 'meta': { 'message': 'job at worker' } } job = celery.result.AsyncResult( taskID) #TODO: task.ready() if not project in self.jobs: self.jobs[project] = [] self.jobs[project].append(job)