def save_entity(self, session_type, session_id): ''' This method overrides the identical method from the inherited superclass, 'BaseData'. Specifically, this method updates an existing entity within the corresponding database table, 'tbl_dataset_entity'. @session_id, is synonymous to 'entity_id', and provides context to update 'modified_xx' columns within the 'tbl_dataset_entity' database table. ''' premodel_settings = self.premodel_data['properties'] premodel_entity = { 'title': premodel_settings.get('session_name', None), 'uid': self.uid, 'id_entity': session_id, } db_save = Entity(premodel_entity, session_type) # save dataset element db_return = db_save.save() # return if db_return['status']: return {'status': True, 'error': None} else: self.list_error.append(db_return['error']) return {'status': False, 'error': self.list_error}
def remove_collection(): ''' This router function removes a collection, with respect to a database type. @collection, indicates a nosql implementation @entity, indicates a sql database ''' if request.method == 'POST': # local variables response = None entity = Entity() collection = Collection() # programmatic-interface if request.get_json(): r = request.get_json() uid = r['uid'] type = r['type'] cname = r['collection'] if (cname and type == 'collection'): payload = {'properties.uid': uid} response = collection.query(cname, 'drop_collection', payload) elif (type == 'entity'): response = entity.remove_entity(uid, cname) # lastrowid returned must be greater than 0 if response and response['result']: return json.dumps({'response': response['result']}) else: return json.dumps({'response': response})
def save_info(dataset, session_type, userid): ''' This method saves the current entity into the database, then returns the corresponding entity id. ''' list_error = [] premodel_settings = dataset['data']['settings'] premodel_entity = { 'title': premodel_settings.get('session_name', None), 'uid': userid, 'id_entity': None } db_save = Entity(premodel_entity, session_type) # save dataset element db_return = db_save.save() # return error(s) if not db_return['status']: list_error.append(db_return['error']) return {'id': None, 'error': list_error} # return session id elif db_return['status'] and session_type == 'data_new': return {'id': db_return['id'], 'error': None}
def remove_collection(): ''' This router function removes a collection, with respect to a database type. @collection, indicates a nosql implementation @entity, indicates a sql database ''' if request.method == 'POST': # local variables response = None entity = Entity() collection = Collection() # programmatic-interface if request.get_json(): r = request.get_json() uid = r['uid'] type = r['type'] cname = r['collection'] if (cname and type == 'collection'): payload = {'properties.uid': uid} response = collection.query(cname, 'drop_collection', payload) elif (type == 'entity'): response = entity.remove_entity(uid, cname) # lastrowid returned must be greater than 0 if response and response['result']: return json.dumps({'response': response['result']}) else: return json.dumps({'response': response})
def save_entity(self, session_type, session_id): ''' This method overrides the identical method from the inherited superclass, 'BaseData'. Specifically, this method updates an existing entity within the corresponding database table, 'tbl_dataset_entity'. @session_id, is synonymous to 'entity_id', and provides context to update 'modified_xx' columns within the 'tbl_dataset_entity' database table. ''' # local variables db_return = None entity = Entity() cursor = Collection() premodel_settings = self.premodel_data['properties'] collection = premodel_settings['collection'] collection_adjusted = collection.lower().replace(' ', '_') collection_count = entity.get_collection_count(self.uid) document_count = cursor.query(collection_adjusted, 'count_documents') # define entity properties premodel_entity = { 'title': premodel_settings.get('session_name', None), 'uid': self.uid, 'id_entity': session_id, } # store entity values in database if ( collection_adjusted and collection_count and collection_count['result'] < self.max_collection and document_count and document_count['result'] < self.max_document ): db_save = Entity(premodel_entity, session_type) db_return = db_save.save() if db_return and db_return['status']: return {'status': True, 'error': None} else: self.list_error.append(db_return['error']) return {'status': False, 'error': self.list_error} else: return {'status': True, 'error': None}
def save_premodel_dataset(self): ''' This method saves the entire the dataset collection, as a json document, into the nosql implementation. ''' # local variables entity = Entity() cursor = Collection() collection = self.premodel_data['properties']['collection'] collection_adjusted = collection.lower().replace(' ', '_') collection_count = entity.get_collection_count(self.uid) document_count = cursor.query(collection_adjusted, 'count_documents') # enfore collection limit: oldest collection name is obtained from the # sql database. Then, the corresponding collection (i.e. target) is # removed from the nosql database. if (not self.uid and collection_count and collection_count['result'] >= self.max_collection and collection_adjusted): target = entity.get_collections(self.uid)['result'][0] cursor.query(target, 'drop_collection') entity.remove_entity(self.uid, target) collection_count = entity.get_collection_count(self.uid) document_count = cursor.query(collection_adjusted, 'count_documents') # save dataset if (collection_adjusted and collection_count and collection_count['result'] < self.max_collection and document_count and document_count['result'] < self.max_document): current_utc = datetime.datetime.utcnow().strftime( "%Y-%m-%dT%H:%M:%S") self.premodel_data['properties']['datetime_saved'] = current_utc self.premodel_data['properties']['uid'] = self.uid document = { 'properties': self.premodel_data['properties'], 'dataset': self.dataset } response = cursor.query(collection_adjusted, 'insert_one', document) else: response = None # return result if response and response['error']: self.list_error.append(response['error']) return {'result': None, 'error': response['error']} elif response and response['result']: return {'result': response['result'], 'error': None} else: return {'result': None, 'error': 'no dataset provided'}
def save_entity(self, session_type, id_entity=None): ''' This method overrides the identical method from the inherited superclass, 'BaseData'. Specifically, this method updates an existing entity within the corresponding database table, 'tbl_dataset_entity'. @session_id, is synonymous to 'entity_id', and provides context to update 'modified_xx' columns within the 'tbl_dataset_entity' database table. @numeric_model_type, list indices begin at 0, and needs to be corrected by adding 1. This allows the numeric representation of the 'model_type' to relate to another database table, which maps integer values with the corresponding 'model_type' name. The integer column of the mapping table begins at 1. ''' # assign numerical representation numeric_model_type = self.list_model_type.index(self.model_type) + 1 # store entity values in database premodel_settings = self.premodel_data['properties'] premodel_entity = { 'title': premodel_settings.get('session_name', None), 'collection': premodel_settings['collection'], 'model_type': numeric_model_type, 'uid': self.uid, } db_save = Entity(premodel_entity, session_type) # save dataset element db_return = db_save.save() # return if db_return['status']: return {'status': True, 'error': None, 'id': db_return['id']} else: self.list_error.append(db_return['error']) return {'status': False, 'error': self.list_error}
def collection_count(): ''' This router function retrieves the number of collections, saved by a specified user. ''' if request.method == 'POST': # local variables count = None entity = Entity() # programmatic-interface if request.get_json(): r = request.get_json() uid = r['uid'] count = entity.get_collection_count(uid) if count and isinstance(count['result'], (int, long)): return json.dumps({'count': count['result']}) else: return json.dumps({'count': -1})
def collection_count(): ''' This router function retrieves the number of collections, saved by a specified user. ''' if request.method == 'POST': # local variables count = None entity = Entity() # programmatic-interface if request.get_json(): r = request.get_json() uid = r['uid'] count = entity.get_collection_count(uid) if count and isinstance(count['result'], (int, long)): return json.dumps({'count': count['result']}) else: return json.dumps({'count': -1})
def save_entity(self, session_type, session_id): ''' This method overrides the identical method from the inherited superclass, 'BaseData'. Specifically, this method updates an existing entity within the corresponding database table, 'tbl_dataset_entity'. @session_id, is synonymous to 'entity_id', and provides context to update 'modified_xx' columns within the 'tbl_dataset_entity' database table. ''' # local variables db_return = None entity = Entity() cursor = Collection() premodel_settings = self.premodel_data['properties'] collection = premodel_settings['collection'] collection_adjusted = collection.lower().replace(' ', '_') collection_count = entity.get_collection_count(self.uid) document_count = cursor.query(collection_adjusted, 'count_documents') # define entity properties premodel_entity = { 'title': premodel_settings.get('session_name', None), 'uid': self.uid, 'id_entity': session_id, } # store entity values in database if (collection_adjusted and collection_count and collection_count['result'] < self.max_collection and document_count and document_count['result'] < self.max_document): db_save = Entity(premodel_entity, session_type) db_return = db_save.save() if db_return and db_return['status']: return {'status': True, 'error': None} else: self.list_error.append(db_return['error']) return {'status': False, 'error': self.list_error} else: return {'status': True, 'error': None}
def generate(model, kernel_type, session_id, feature_request, list_error): ''' This method generates an sv (i.e. svm, or svr) model using feature data, retrieved from the database. The generated model, is then stored within the NoSQL datastore. @grouped_features, a matrix of observations, where each nested vector, or python list, is a collection of features within the containing observation. @encoded_labels, observation labels (dependent variable labels), encoded into a unique integer representation. ''' # local variables dataset = feature_request.get_dataset(session_id, model) get_feature_count = feature_request.get_count(session_id) label_encoder = preprocessing.LabelEncoder() logger = Logger(__name__, 'error', 'error') list_model_type = current_app.config.get('MODEL_TYPE') # get dataset if dataset['error']: logger.log(dataset['error']) list_error.append(dataset['error']) dataset = None else: dataset = numpy.asarray(dataset['result']) # get feature count if get_feature_count['error']: logger.log(get_feature_count['error']) list_error.append(get_feature_count['error']) feature_count = None else: feature_count = get_feature_count['result'][0][0] # check dataset integrity, build model if len(dataset) % feature_count == 0: features_list = dataset[:, [[0], [2], [1]]] current_features = [] grouped_features = [] observation_labels = [] feature_labels = [] # group features into observation instances, record labels for index, feature in enumerate(features_list): # svm: observation labels if model == list_model_type[0]: current_features.append(feature[1][0]) if (index+1) % feature_count == 0: grouped_features.append(current_features) observation_labels.append(feature[0][0]) current_features = [] # svr: observation labels elif model == list_model_type[1]: current_features.append(float(feature[1][0])) if (index+1) % feature_count == 0: grouped_features.append(current_features) observation_labels.append(float(feature[0][0])) current_features = [] # general feature labels in every observation if not len(feature_labels) == feature_count: feature_labels.append(feature[2][0]) # case 1: svm model if model == list_model_type[0]: # convert observation labels to a unique integer representation label_encoder = preprocessing.LabelEncoder() label_encoder.fit(dataset[:, 0]) encoded_labels = label_encoder.transform(observation_labels) # create model clf = svm.SVC(kernel=kernel_type, probability=True) # cache encoded labels Model(label_encoder).cache(model + '_labels', session_id) # fit model clf.fit(grouped_features, encoded_labels) # case 2: svr model elif model == list_model_type[1]: # create model clf = svm.SVR(kernel=kernel_type) # fit model clf.fit(grouped_features, observation_labels) # compute, and cache coefficient of determination r2 = clf.score(grouped_features, observation_labels) Hset().cache( model + '_r2', session_id, r2 ) # get title entity = Entity() title = entity.get_title(session_id)['result'][0][0] # cache model, title Model(clf).cache( model + '_model', str(session_id) + '_' + title ) Hset().cache(model + '_title', session_id, title) # cache feature labels, with respect to given session id Hset().cache( model + '_feature_labels', str(session_id), json.dumps(feature_labels) ) # return error(s) if exists return {'error': list_error}
def save_entity(self, session_type, id_entity=None): ''' This method overrides the identical method from the inherited superclass, 'BaseData'. Specifically, this method updates an existing entity within the corresponding database table, 'tbl_dataset_entity'. @session_id, is synonymous to 'entity_id', and provides context to update 'modified_xx' columns within the 'tbl_dataset_entity' database table. @numeric_model_type, list indices begin at 0, and needs to be corrected by adding 1. This allows the numeric representation of the 'model_type' to relate to another database table, which maps integer values with the corresponding 'model_type' name. The integer column of the mapping table begins at 1. ''' # local variables db_return = None entity = Entity() cursor = Collection() premodel_settings = self.premodel_data['properties'] collection = premodel_settings['collection'] collection_adjusted = collection.lower().replace(' ', '_') collection_count = entity.get_collection_count(self.uid) document_count = cursor.query(collection_adjusted, 'count_documents') # assign numerical representation numeric_model_type = self.list_model_type.index(self.model_type) + 1 # define entity properties premodel_entity = { 'title': premodel_settings.get('session_name', None), 'collection': collection, 'model_type': numeric_model_type, 'uid': self.uid, } # store entity values in database if ( collection_adjusted and collection_count and collection_count['result'] < self.max_collection and document_count and document_count['result'] < self.max_document ): entity = Entity(premodel_entity, session_type) db_return = entity.save() # return if db_return and db_return['error']: self.list_error.append(db_return['error']) return {'status': False, 'error': self.list_error} elif db_return and db_return['status']: return {'status': True, 'error': None, 'id': db_return['id']} else: return {'status': True, 'error': 'Entity was not saved', 'id': None}
def save_premodel_dataset(self): ''' This method saves the entire the dataset collection, as a json document, into the nosql implementation. ''' # local variables response = None entity = Entity() cursor = Collection() collection = self.premodel_data['properties']['collection'] collection_adjusted = collection.lower().replace(' ', '_') collection_count = entity.get_collection_count(self.uid) document_count = cursor.query(collection_adjusted, 'count_documents') # enfore collection limit: oldest collection name is obtained from the # sql database. Then, the corresponding collection (i.e. target) is # removed from the nosql database. if ( not self.uid and collection_count and collection_count['result'] >= self.max_collection and collection_adjusted ): target = entity.get_collections(self.uid)['result'][0] cursor.query(target, 'drop_collection') entity.remove_entity(self.uid, target) collection_count = entity.get_collection_count(self.uid) document_count = cursor.query(collection_adjusted, 'count_documents') # save dataset if ( collection_adjusted and collection_count and collection_count['result'] < self.max_collection and document_count and document_count['result'] < self.max_document ): current_utc = datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%S") self.premodel_data['properties']['datetime_saved'] = current_utc if self.dataset: document = { 'properties': self.premodel_data['properties'], 'dataset': self.dataset } response = cursor.query( collection_adjusted, 'insert_one', document ) else: response = None # return result if response and response['error']: self.list_error.append(response['error']) return {'result': None, 'error': response['error']} elif response and response['result']: return {'result': response['result'], 'error': None} else: return {'result': None, 'error': 'no dataset provided'}