def __init__(self): self.entities = [] self.columns = [] self.relationships = [] self.synonyms_col = [] self.synonyms_tab = [] self.entity_graph = [] self.loaded_entities = [] self.config = Configuration() self.conn = pyodbc.connect(self.config.get_sql_connection_string()) lookups = Lookups() self.lemmatizer = Lemmatizer(lookups) self.load_db_model()
def __init__(self): self.entities = [] self.columns = [] self.relationships = [] self.synonyms_col = [] self.synonyms_tab = [] self.entity_graph = [] self.loaded_entities = [] self.config = Configuration() self.conn = pyodbc.connect(self.config.get_sql_connection_string()) #self.lemmatizer = Lemmatizer(LEMMA_INDEX, LEMMA_EXC, LEMMA_RULES) self.lemmatizer = English.Defaults.create_lemmatizer() self.load_db_model()
def __init__(self, entities, columns, db_model): self.columns = columns self.entities = entities self.db_model = db_model self.entity_column_mapping = [] self.joins = [] self.conditions = [] self.select = [] self.query = "" self.entities_parsed = [] self.isMaxRequired = "" self.isMaxRequiredEntity = "" self.isMinRequired = "" self.isMinRequiredEntity = "" self.isAverage = "" self.isAverageEntity = "" self.isCount = "" self.isCountEntity = "" self.isSum = "" self.isSumEntity = "" self.config = Configuration() self.conn = pyodbc.connect(self.config.get_sql_connection_string())
def __init__(self): self.config = Configuration().get_config() self.transformer_dict = self.config['preprocessing']['transformers'] self.shadow_train_size = self.config['models']['sample_sizes'][ 'shadow_train'] self.probe_size = self.config['models']['sample_sizes']['probe_size'] self.update_size = self.config['models']['sample_sizes']['update_size'] self.num_train_update_samples = self.config['models']['sample_sizes'][ 'num_train_update_samples'] self.num_test_update_samples = self.config['models']['sample_sizes'][ 'num_test_update_samples'] self.data_file_path = 'data/raw_data_files/' + Utils().get_model_name( self.num_train_update_samples, self.num_test_update_samples, self.shadow_train_size)
def __init__(self, parameters): self.config = Configuration().get_config() self.device = Utils().get_device() self.log_dir = 'runs' self.model_dir = 'saved_models/' self.update_set_size = self.config['models']['sample_sizes'][ 'update_size'] self.num_train_update_samples = self.config['models']['sample_sizes'][ 'num_train_update_samples'] self.num_test_update_samples = self.config['models']['sample_sizes'][ 'num_test_update_samples'] self.image_size = (3, 32, 32) self.time = datetime.now().strftime("%d-%m-%H-%M-%S") self.noise_dim = self.config['models']['GAN']['noise_dim'] self.datasets_directory = f'data/datasets' self.shadow_num_epochs = self.config['models']['shadow_model'][ 'num_epochs'] self.shadow_in_channel = parameters['shdow_in_channel'] self.shadow_out_channels = parameters['shdow_out_channels'] self.shadow_fc_size = self.shadow_out_channels[1] * 25 self.shadow_learning_rate = parameters['shdow_learning_rate'] self.update_learning_rate = parameters['update_learning_rate'] self.update_shadow_epochs = parameters['update_shadow_epochs'] self.encoder_leky_reul = parameters['encoder_leaky_relu'] self.encoder_dropout = parameters['encoder_dropout'] self.dis_leaky = parameters['dis_leaky'] self.dis_ndf = parameters['dis_ndf'] self.generator_leaky = parameters['generator_leaky'] self.generator_ngf = parameters['generator_ngf'] self.gan_loss_weight = parameters['gan_loss_weight'] self.discriminator_type = parameters['discriminator_type'] self.generator_type = parameters['generator_type'] self.noisy_label_change = parameters['noisy_label_change'] self.discriminator_dropout = parameters['discriminator_dropout'] self.lr_gamma = parameters['lr_gamma'] self.cache_name = 'cache/' + Utils.get_folder_name( self.update_set_size, self.num_train_update_samples, self.num_test_update_samples)
class HandleParameterize: """ 参数化类 """ invested_user_pwd_pattern = r'{invest_user_pwd}' invested_user_id_pattern = r'{invest_user_id}' configuration = Configuration(CONFIG_USER_ACCOUNT_FILE_PATH) @classmethod def do_param(cls, data): # 替换密码 if re.search(cls.invested_user_pwd_pattern, data): data = re.sub(cls.invested_user_pwd_pattern, cls.configuration.getConfig('Invest', 'pwd'), data) # 替换id if re.search(cls.invested_user_id_pattern, data): data = re.sub(cls.invested_user_id_pattern, str(cls.configuration.getConfig('Invest', 'id')), data) return data
def __predict_output__(): plt.interactive(False) cfg = Configuration() GPU = True if GPU != True: os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID" os.environ["CUDA_VISIBLE_DEVICES"] = "" # Input Path root_dir = os.path.dirname(os.path.abspath(__file__)) image_path = cfg.image_path json_path = os.path.join(root_dir, cfg.input_filename) testingset = os.path.join(root_dir, 'testingset') Preprocessor.__generate_kijiji_set__(root_dir, image_path, json_path, testingset, 'model') # ------------------generator to compile training data of kijiji dataset---------------------------------------- image_path = os.path.join(root_dir, 'testingset') data_path = glob(image_path + "/*") # Image Segmentation Parameters model_path = os.path.expanduser(cfg.model_path) assert model_path.endswith('.h5'), 'Keras model must be a .h5 file.' anchors_path = os.path.expanduser(cfg.anchors_path) classes_path = os.path.expanduser(cfg.classes_path) test_path = os.path.expanduser(cfg.test_path) output_path = os.path.expanduser(cfg.segmented_output_path) json_path = os.path.expanduser(cfg.json_output) if not os.path.exists(output_path): print('Creating output path {}'.format(output_path)) os.mkdir(output_path) sess = K.get_session() class_names = Preprocessor.__return_class_names__(classes_path) anchors = Preprocessor.__return_anchors__(anchors_path) yolo_model = load_model(model_path) # Verify model, anchors, and classes are compatible num_classes = len(class_names) num_anchors = len(anchors) info = 'Mismatch between model and given anchor and class sizes. ' \ 'Specify matching anchors and classes with --anchors_path and --classes_path flags.' model_output_channels = yolo_model.layers[-1].output_shape[-1] assert model_output_channels == num_anchors * (num_classes + 5), info print('{} model, anchors, and classes loaded.'.format(model_path)) # Check if model is fully convolutional, assuming channel last order. model_image_size = yolo_model.layers[0].input_shape[1:3] is_fixed_size = model_image_size != (None, None) # Generate Colors for drawing bounding boxes hsv_tuples, colors = Preprocessor.__generate_colors_for_bounding_boxes__( class_names) yolo_outputs = yolo_head(yolo_model.output, anchors, len(class_names)) input_image_shape = K.placeholder(shape=(2, )) boxes, scores, classes = yolo_eval(yolo_outputs, input_image_shape, score_threshold=cfg.score_threshold, iou_threshold=cfg.iou_threshold) # Load Images from the root folder input_images_model_1, all_images, data_path, data_path_with_image_name = Preprocessor.__load_image_data_thumbnails__( data_path, cfg.compressed_image_height, cfg.compressed_image_width, cfg.compressed_channel, cfg.number_of_categories, cfg.number_of_images_per_category, root_dir, is_fixed_size, model_image_size, sess, yolo_model, input_image_shape, boxes, scores, classes, cfg.font_path, class_names, colors, output_path, json_path, test_path, True, # Segmentation Flag False, # Edge-detection Flag True, # Extract object Flag False) # Gray Scale Flag input_images_model_2, all_images, data_path, data_path_with_image_name = Preprocessor.__load_image_data_thumbnails__( data_path, cfg.compressed_image_height, cfg.compressed_image_width, cfg.compressed_channel, cfg.number_of_categories, cfg.number_of_images_per_category, root_dir, is_fixed_size, model_image_size, sess, yolo_model, input_image_shape, boxes, scores, classes, cfg.font_path, class_names, colors, output_path, json_path, test_path, False, True, False, False) input_images_model_3, all_images, data_path, data_path_with_image_name = Preprocessor.__load_image_data_thumbnails__( data_path, cfg.image_height, cfg.image_width, cfg.channel, cfg.number_of_categories, cfg.number_of_images_per_category, root_dir, is_fixed_size, model_image_size, sess, yolo_model, input_image_shape, boxes, scores, classes, cfg.font_path, class_names, colors, output_path, json_path, test_path, False, False, False, False) input_shape = [ cfg.compressed_image_height, cfg.compressed_image_width, cfg.compressed_channel ] input_shape_3 = [cfg.image_height, cfg.image_width, cfg.channel] # load (pre-trained) weights for model_1 print('-' * 30) print('Loading model weights...\n') weight_folder = cfg.model_1_save # the path where the model weights are stored weight_file = 'model_1.h5' model_1 = Preprocessor.__load_model_weights__(weight_folder, weight_file, input_shape, input_shape_3, "Model_1") # load (pre-trained) weights for model_2 print('-' * 30) print('Loading model weights...\n') weight_folder = cfg.model_2_save # the path where the model weights are stored weight_file = 'model_2.h5' model_2 = Preprocessor.__load_model_weights__(weight_folder, weight_file, input_shape, input_shape_3, "Model_2") # load (pre-trained) weights for model_2 print('-' * 30) print('Loading model weights...\n') weight_folder = cfg.model_3_save # the path where the model weights are stored weight_file = 'model_3.h5' model_3 = Preprocessor.__load_model_weights__(weight_folder, weight_file, input_shape, input_shape_3, "Model_3") print(root_dir) print(os.path.join(root_dir, cfg.output_model_1)) output_path_model_1 = os.path.join(root_dir + cfg.output_model_1) output_path_model_2 = os.path.join(root_dir + cfg.output_model_2) output_path_model_3 = os.path.join(root_dir + cfg.output_model_3) Preprocessor.__create_output_directories__(output_path_model_1) Preprocessor.__create_output_directories__(output_path_model_2) Preprocessor.__create_output_directories__(output_path_model_3) features_from_model_1 = Preprocessor.__get_score_model__( model_1, input_images_model_1, output_path_model_1) features_from_model_2 = Preprocessor.__get_score_model__( model_2, input_images_model_2, output_path_model_2) features_from_model_3 = Preprocessor.__get_score_model__( model_3, input_images_model_3, output_path_model_3) features_from_model_1 = Preprocessor.__flatten_img_data__( features_from_model_1) features_from_model_2 = Preprocessor.__flatten_img_data__( features_from_model_2) features_from_model_3 = Preprocessor.__flatten_img_data__( features_from_model_3) fused_features = np.concatenate([ features_from_model_1, features_from_model_2, features_from_model_3 ], axis=1) fused_features = [ Preprocessor.__binarize__(features) for features in fused_features ] counter_for_predictions = 0 sub_average_precision_make, sub_average_precision_color = [], [] sub_average_precision_body, sub_average_precision_model = [], [] cum_average_precision_make, cum_average_precision_color = [], [] cum_average_precision_body, cum_average_precision_model = [], [] precision_at_3_5_10_all = ''.join(cfg.precision_counter).split(',') while counter_for_predictions <= 2: test_image_idx = int(len(input_images_model_1) * random()) if test_image_idx < len(data_path_with_image_name): idx_closest = Preprocessor.__get_closest_images__( test_image_idx, fused_features, cfg.number_of_predictions) test_image = Preprocessor.__get_concatenated_images__( data_path_with_image_name, [test_image_idx], cfg.compressed_image_width) results_image = Preprocessor.__get_concatenated_images__( data_path_with_image_name, idx_closest, cfg.compressed_image_width) source_category = str( data_path_with_image_name[test_image_idx]).split('/') similar_image = [] similar_idx_closest = [] for counter_for_recommendations in range(0, len(idx_closest)): category = str(data_path_with_image_name[ idx_closest[counter_for_recommendations]]).split('/') if str(source_category[-2]).strip() == str( category[-2].strip()): similar_image.append(data_path_with_image_name[ idx_closest[counter_for_recommendations]]) similar_idx_closest.append( idx_closest[counter_for_recommendations]) print("Test Image ID:", test_image_idx) print("\n") print("Closest Images ID:", idx_closest) print("\n") print("Similar Images ID", similar_idx_closest) print("\n") precision_per_make, precision_per_color = [], [] precision_per_body_wise, precision_per_model_wise = [], [] results_image_recommendations = [] for i in range(0, len(precision_at_3_5_10_all)): results_image_recommendations = Preprocessor.__get_concatenated_images__( data_path_with_image_name, similar_idx_closest, cfg.compressed_image_width) list_of_similar_image_names = Preprocessor.__return_image_names__( data_path_with_image_name, similar_idx_closest) name_of_test_image = Preprocessor.__return_image_names__( data_path_with_image_name, [test_image_idx]) dict_of_attributes_of_similar_images = Preprocessor.__get_attributes_list__( list_of_similar_image_names, os.path.join(root_dir, cfg.input_filename)) dict_of_attributes_of_test_image = Preprocessor.__get_attributes_list__( name_of_test_image, os.path.join(root_dir, cfg.input_filename)) similar_make_wise = Preprocessor.__get_similar__( dict_of_attributes_of_test_image, dict_of_attributes_of_similar_images[:int( precision_at_3_5_10_all[i])], 'make') similar_color_wise = Preprocessor.__get_similar__( dict_of_attributes_of_test_image, dict_of_attributes_of_similar_images[:int( precision_at_3_5_10_all[i])], 'color') similar_body_wise = Preprocessor.__get_similar__( dict_of_attributes_of_test_image, dict_of_attributes_of_similar_images[:int( precision_at_3_5_10_all[i])], 'body') similar_model_wise = Preprocessor.__get_similar__( dict_of_attributes_of_test_image, dict_of_attributes_of_similar_images[:int( precision_at_3_5_10_all[i])], 'model') precision_per_make.append( float( float(len(similar_make_wise)) / int(precision_at_3_5_10_all[i]))) precision_per_color.append( float( float(len(similar_color_wise)) / int(precision_at_3_5_10_all[i]))) precision_per_body_wise.append( float( float(len(similar_body_wise)) / int(precision_at_3_5_10_all[i]))) precision_per_model_wise.append( float( float(len(similar_model_wise)) / int(precision_at_3_5_10_all[i]))) sub_average_precision_make.append(precision_per_make) sub_average_precision_color.append(precision_per_color) sub_average_precision_body.append(precision_per_body_wise) sub_average_precision_model.append(precision_per_model_wise) imsave('test.png', test_image) imsave('recommendations.png', results_image_recommendations) imsave('total_results.png', results_image) counter_for_predictions += 1 time.sleep(1) else: print("Index is out of bound") cum_average_precision_make.append( map(Preprocessor.__mean__, zip(*sub_average_precision_make))) cum_average_precision_color.append( map(Preprocessor.__mean__, zip(*sub_average_precision_color))) cum_average_precision_body.append( map(Preprocessor.__mean__, zip(*sub_average_precision_body))) cum_average_precision_model.append( map(Preprocessor.__mean__, zip(*sub_average_precision_model))) print("\n \n \n") print( "-----------------------------------------------------------------------------------" ) print("Average Precision Make-Wise", precision_at_3_5_10_all, map(Preprocessor.__mean__, zip(*cum_average_precision_make))) print("Average Precision Color-Wise", precision_at_3_5_10_all, map(Preprocessor.__mean__, zip(*cum_average_precision_color))) print("Average Precision Body-Wise", precision_at_3_5_10_all, map(Preprocessor.__mean__, zip(*cum_average_precision_body))) print("Average Precision Model-Wise", precision_at_3_5_10_all, map(Preprocessor.__mean__, zip(*cum_average_precision_model))) writer = csv.writer(open(os.path.join(root_dir, 'results.csv'), 'w')) writer.writerow([ "Make-Wise: Precision at 3", "Make-Wise: Precision at 5", "Make-Wise: Precision at 10" ]) for row in zip(*cum_average_precision_make): writer.writerow(row) writer.writerow('\n') writer.writerow([ "Color-Wise: Precision at 3", "Color-Wise: Precision at 5", "Color-Wise: Precision at 10" ]) for row in zip(*cum_average_precision_color): writer.writerow(row) writer.writerow('\n') writer.writerow([ "Body-Wise: Precision at 3", "Body-Wise: Precision at 5", "Body-Wise: Precision at 10" ]) for row in zip(*cum_average_precision_body): writer.writerow(row) writer.writerow('\n') writer.writerow([ "Model-Wise: Precision at 3", "Model-Wise: Precision at 5", "Model-Wise: Precision at 10" ]) for row in zip(*cum_average_precision_model): writer.writerow(row) writer.writerow('\n')
class SQLGenerator(object): def __init__(self, entities, columns, db_model): self.columns = columns self.entities = entities self.db_model = db_model self.entity_column_mapping = [] self.joins = [] self.conditions = [] self.select = [] self.query = "" self.entities_parsed = [] self.isMaxRequired = "" self.isMaxRequiredEntity = "" self.isMinRequired = "" self.isMinRequiredEntity = "" self.isAverage = "" self.isAverageEntity = "" self.isCount = "" self.isCountEntity = "" self.isSum = "" self.isSumEntity = "" self.config = Configuration() self.conn = pyodbc.connect(self.config.get_sql_connection_string()) def run_query(self): cursor = self.conn.cursor() cursor.execute(self.query) result = [] columns = [] for row in cursor: result.append([col for col in row]) columns = [column[0] for column in cursor.description] return [result, columns] def sortSecond(self, join_comb): return join_comb[0] def get_from_clause(self, level): # build the from_clause from_clause = "" if len(self.entity_column_mapping) == 1: from_clause = self.entity_column_mapping[0][0] + " " + self.entity_column_mapping[0][0] + level elif len(self.entity_column_mapping) > 1: from_clause = "" join_index = 0 entity_included_in_join = [] for join in self.joins: if join_index == 0: from_clause = from_clause + join[0] + " " + join[0] + level + " JOIN " + join[1] + " " + join[1] + level + " ON " + join[0] + level + "." + join[2] + "=" + join[1] + level + "." + join[3] entity_included_in_join.append(join[0]) entity_included_in_join.append(join[1]) else: if join[0] in entity_included_in_join: from_clause = from_clause + " " + " JOIN " + join[1] + " " + join[1] + level + " ON " + join[0]+ level + "." + join[2] + " = " + join[1] + level + "." + join[3] else: from_clause = from_clause + " JOIN " + join[0] + " " + join[0] + level + " ON " + join[0] + level + "." + join[2] + " = " + join[1] + level + "." + join[3] join_index = join_index + 1 return from_clause def get_where_clause(self, level): return " and ".join([cond[0] + level + "." + cond[1] + " " + cond[2] + " " + cond[3] for cond in self.conditions]) def get_select_clause(self, level): return ", ".join([col[0] + level + "." + col[1] for col in self.select]) def correlated_sub_query_in_where(self, column, entity, type_): # type = min, max # from clause from_clause = self.get_from_clause("1") # select clause select_clause = self.get_select_clause("1") # where clause where_clause = self.get_where_clause("1") type_sub_query_where_clause = self.get_where_clause("2") type_sub_query_from_clause = self.get_from_clause("2") if type_sub_query_where_clause != "": type_sub_query_where_clause = " Where " + type_sub_query_where_clause typeQuery = "SELECT " + \ type_ + "(" + entity + "2." + column + ") " + \ " From " + \ type_sub_query_from_clause + \ type_sub_query_where_clause if select_clause != "": select_clause = select_clause + ", " if where_clause != "": where_clause = where_clause + " and " self.query = "SELECT " + \ select_clause + entity + "1." + column + " " + \ " From " + \ from_clause + \ " Where " + \ where_clause + \ entity + "1." + column + " = (" + typeQuery + ")" def correlated_sub_query_in_select(self, column, entity, type_): # type = avg, sum, count # from clause from_clause = self.get_from_clause("1") # select clause select_clause = self.get_select_clause("1") # where clause where_clause = self.get_where_clause("1") type_sub_query_where_clause = self.get_where_clause("2") type_sub_query_from_clause = self.get_from_clause("2") # find the identifier column of the entity in parameter db_model_ent = next(e for e in self.db_model.entities if e.name.lower() == entity.lower()) # db_model_ent.primaryKey # correlation # find where this table is being referenced entity_relationships = [(rel.entity1, rel.column1) for rel in self.db_model.relationships if (rel.entity2 == entity)] correlation_entity = entity correlation_entity_column = db_model_ent.primaryKey parent_entity_exists = False parent_entries = [] if len(self.entity_column_mapping) > 1: for ecm in self.entity_column_mapping: if ecm[0] != entity: if ecm[0] in [ent[0] for ent in entity_relationships]: parent_entry = next(ent for ent in entity_relationships if ent[0] == ecm[0]) parent_entity_exists = True parent_entries.append(parent_entry) elif len(self.entity_column_mapping) == 1: # only one entity, use where filters correlation = " and ".join([cond[0] + "1" + "." + cond[1] + " = " + cond[0] + "2" + "." + cond[1] for cond in self.conditions]) if len(self.entity_column_mapping) > 1: if parent_entity_exists == True and len(parent_entry) > 0: correlations = [] for parent_entry in parent_entries: correlations.append(parent_entry[0] + "2." + parent_entry[1] + "=" + parent_entry[0] + "1." + parent_entry[1]) correlation = " and ".join(correlations) else: correlation = entity + "2." + db_model_ent.primaryKey + "=" + entity + "1." + db_model_ent.primaryKey if type_sub_query_where_clause == "": type_sub_query_where_clause = correlation else: type_sub_query_where_clause = type_sub_query_where_clause + " and " + correlation if type_sub_query_where_clause != "": type_sub_query_where_clause = " Where " + type_sub_query_where_clause type_sub_query = "SELECT " + \ type_ + "(" + entity + "2." + column + ") " + \ " From " + \ type_sub_query_from_clause + \ type_sub_query_where_clause if select_clause != "": select_clause = select_clause + ", " if where_clause != "": where_clause = " Where " + where_clause self.query = "SELECT distinct " + \ select_clause + "(" + type_sub_query + ") as " + type_ + "_" + column + " " + \ " From " + \ from_clause + \ where_clause def build_query(self): # maximum case if self.isMaxRequired != "": self.correlated_sub_query_in_where(self.isMaxRequired, self.isMaxRequiredEntity,"max") # minimum case elif self.isMinRequired != "": self.correlated_sub_query_in_where(self.isMinRequired, self.isMinRequiredEntity,"min") # average case elif self.isAverage != "": self.correlated_sub_query_in_select(self.isAverage, self.isAverageEntity, "avg") # count elif self.isCount != "": self.correlated_sub_query_in_select(self.isCount, self.isCountEntity, "count") # sum elif self.isSum != "": self.correlated_sub_query_in_select(self.isSum, self.isSumEntity, "sum") # regular else: # from clause from_clause = self.get_from_clause("1") # select clause select_clause = self.get_select_clause("1") # where clause where_clause = self.get_where_clause("1") if where_clause != "": where_clause = " Where " + where_clause self.query = "SELECT distinct " + \ select_clause + " " + \ " From " + \ from_clause + \ where_clause def find_select(self): for ecm in self.entity_column_mapping: # column mapping within entity for cm in ecm[1]: # if cm.condition is None and cm.value_ is None: if cm.value_ is None or cm.value_ == "NoValue": # entity, column name, [Avg, Min, Max, Sum, Count] # add the where clause here for min, max and sum conditions if cm.isMax == True: self.isMaxRequired = cm.name.lower() self.isMaxRequiredEntity = ecm[0] elif cm.isMin == True: self.isMinRequired = cm.name.lower() self.isMinRequiredEntity = ecm[0] elif cm.isAverage == True: self.isAverage = cm.name.lower() self.isAverageEntity = ecm[0] elif cm.isCount == True: self.isCount = cm.name.lower() self.isCountEntity = ecm[0] elif cm.isSum == True: self.isSum = cm.name.lower() self.isSumEntity = ecm[0] else: # check for duplicates if len([sel for sel in self.select if sel[0].lower() == ecm[0].lower() and sel[1].lower() == cm.name.lower()]) == 0: self.select.append((ecm[0], cm.name.lower(), None)) for ent in self.entities: # TODO... add max, min..etc case # get default column from db_model db_model_ent = next(e for e in self.db_model.entities if e.name.lower() == ent.name.lower()) # check for duplicates if len([sel for sel in self.select if sel[0].lower() == ent.name.lower() and sel[1].lower() == db_model_ent.defaultColumn.lower()]) == 0: self.select.append((ent.name.lower(), db_model_ent.defaultColumn, None)) def find_conditions(self): # entity column mapping for ecm in self.entity_column_mapping: # column mapping within entity for cm in ecm[1]: if cm.condition is not None and cm.value_ is not None and cm.value_ != "NoValue": val = cm.value_ if cm.type_ == "string": val = "'" + val + "'" self.conditions.append((ecm[0], cm.name.lower(), cm.condition, str(val))) def find_relationships(self): i = 0 j = 0 while i < len(self.entity_column_mapping): j = i + 1 base_entity = self.entity_column_mapping[i][0] while j < len(self.entity_column_mapping): join_entity = self.entity_column_mapping[j][0] if len([rel for rel in self.db_model.relationships if ((rel.entity1 == base_entity and rel.entity2 == join_entity) or (rel.entity2 == base_entity and rel.entity1 == join_entity))]) == 1: rel = next(rel for rel in self.db_model.relationships if ((rel.entity1 == base_entity and rel.entity2 == join_entity) or (rel.entity2 == base_entity and rel.entity1 == join_entity))) if rel.entity1 == base_entity: self.joins.append((base_entity, join_entity, rel.column1, rel.column2)) else: self.joins.append((join_entity, base_entity, rel.column1, rel.column2)) j = j + 1 i = i + 1 if len(self.joins) == 0 and len(self.entity_column_mapping) > 1: # try to find the relationship using db model's entity_graph i = 0 entities_mapped = [] while i < (len(self.entity_column_mapping) - 1): base_entity = self.entity_column_mapping[i] join_entity = self.entity_column_mapping[i + 1] if base_entity[0] not in entities_mapped: entities_mapped.append(entities_mapped) found, entities_mapped = self.find_entities_relationship(base_entity, join_entity, entities_mapped) i = i + 1 i = 0 j = 0 while i < len(entities_mapped): j = i + 1 base_entity = entities_mapped[i] while j < len(entities_mapped): join_entity = entities_mapped[j] if len([rel for rel in self.db_model.relationships if ((rel.entity1 == base_entity and rel.entity2 == join_entity) or (rel.entity2 == base_entity and rel.entity1 == join_entity))]) == 1: rel = next(rel for rel in self.db_model.relationships if ((rel.entity1 == base_entity and rel.entity2 == join_entity) or (rel.entity2 == base_entity and rel.entity1 == join_entity))) if rel.entity1 == base_entity: self.joins.append((base_entity, join_entity, rel.column1, rel.column2)) else: self.joins.append((join_entity, base_entity, rel.column1, rel.column2)) j = j + 1 i = i + 1 def find_entities_relationship(self, base_entity, join_entity, entities_mapped): entities_to_be_included = copy.copy(entities_mapped) found = False base_entity_graph = next(eg for eg in self.db_model.entity_graph if eg[0].lower() == base_entity[0].lower()) for child_entity_in_graph in base_entity_graph[1]: if child_entity_in_graph == join_entity[0]: entities_to_be_included.append(child_entity_in_graph) found = True break; child_entity_graph = next(eg for eg in self.db_model.entity_graph if eg[0].lower() == child_entity_in_graph.lower()) entities_to_be_included_temp = copy.copy(entities_to_be_included) if child_entity_in_graph not in entities_to_be_included_temp: entities_to_be_included_temp.append(child_entity_in_graph) found, entities_to_be_included = self.find_entities_relationship(child_entity_graph, join_entity, entities_to_be_included_temp) if found: break; if found: for entity_to_be_included in entities_to_be_included: if entity_to_be_included not in entities_mapped: entities_mapped.append(entity_to_be_included) return (found, entities_to_be_included) def find_column(self, column, entityName): column_parent_entity_found = False # get the db model for entity db_model_entity = next(model_entity for model_entity in self.db_model.entities if model_entity.name == entityName.lower()) # add entity into parsed collection self.entities_parsed.append(entityName) # check if the column exists in the db_model if column.name.lower() in [db_model_column.name for db_model_column in db_model_entity.columns]: # column parent found, break the loop column_parent_entity_found = True return (column_parent_entity_found, db_model_entity.name, column) # if column does not exists in db_model_entity # then look for the related entities if column_parent_entity_found == False: # look for related entities for model_entity in [model_entity for model_entities in self.db_model.entity_graph if model_entities[0].lower() == entityName.lower() for model_entity in model_entities[1]]: # only process, if not processed before if len([ep for ep in self.entities_parsed if ep.lower() == model_entity]) == 0: column_parent_entity_found, model_name, columnName = self.find_column(column, model_entity) # column found, return entity with column if column_parent_entity_found == True: return (column_parent_entity_found, model_name, columnName) # column not found return (column_parent_entity_found, None, None) def find_entity(self, column): column_parent_entity_found = False for entity in self.entities: column_parent_entity_found, model_name, columnName = self.find_column(column, entity.name) # column found, return entity with column if column_parent_entity_found == True: return (column_parent_entity_found, model_name, columnName) return (column_parent_entity_found, None, None) def get_sql(self): if len(self.entities) > 0: for column in self.columns: # reset the entities_parsed array for new column self.entities_parsed = [] column_parent_entity_found, model_name, columnName = self.find_entity(column) if column_parent_entity_found == True: if len([ecm for ecm in self.entity_column_mapping if ecm[0] == model_name]) == 1: ecm = next(ecm for ecm in self.entity_column_mapping if ecm[0] == model_name) ecm[1].append(columnName) else: self.entity_column_mapping.append((model_name, [columnName])) else: print("Column " + column.name + " not found.. ignoring column") for entity in self.entities: if entity.condition is not None and entity.value_ is not None: # reset the entities_parsed array for new column model_name = entity.name ent = next(en for en in self.db_model.entities if en.name.lower() == entity.name.lower()) default_column = next(col for col in ent.columns if col.name.lower() == ent.defaultColumn.lower()) copy_default_column = copy.copy(default_column) copy_default_column.condition = entity.condition copy_default_column.value_ = entity.value_ copy_default_column.isSum = entity.isSum copy_default_column.isAverage = entity.isAverage copy_default_column.isCount = entity.isCount copy_default_column.isMin = entity.isMin copy_default_column.isMax = entity.isMax if len([ecm for ecm in self.entity_column_mapping if ecm[0].lower() == model_name.lower()]) == 1: ecm = next(ecm for ecm in self.entity_column_mapping if ecm[0].lower() == model_name.lower()) ecm[1].append(copy_default_column) else: self.entity_column_mapping.append((model_name.lower(), [copy_default_column])) else: if len([ecm for ecm in self.entity_column_mapping if ecm[0].lower() == entity.name.lower()]) == 0: self.entity_column_mapping.append((entity.name.lower(), [])) else: ecm = next(ecm for ecm in self.entity_column_mapping if ecm[0].lower() == entity.name.lower()) ent = next(en for en in self.db_model.entities if en.name.lower() == entity.name.lower()) default_column = next(col for col in ent.columns if col.name.lower() == ent.defaultColumn.lower()) copy_default_column = copy.copy(default_column) copy_default_column.condition = entity.condition copy_default_column.value_ = entity.value_ copy_default_column.isSum = entity.isSum copy_default_column.isAverage = entity.isAverage copy_default_column.isCount = entity.isCount copy_default_column.isMin = entity.isMin copy_default_column.isMax = entity.isMax ecm[1].append(copy_default_column) elif len(self.columns) > 0: # No entities identified in the phrase # Finding...entities as per the columns identified in the phrase for col in self.columns: max_col_found_count = 0 max_col_found_count_entity = "" # look for columns and find max column in an entity and related entities" for entity in self.db_model.entities: column_found_count = 0 if col.name.lower() in [col.name.lower() for col in entity.columns]: column_found_count = column_found_count + 1 if max_col_found_count < column_found_count: max_col_found_count = column_found_count max_col_found_count_entity = entity.name if max_col_found_count_entity != "": if len([ecm for ecm in self.entity_column_mapping if ecm[0] == max_col_found_count_entity]) == 1: ecm = next(ecm for ecm in self.entity_column_mapping if ecm[0] == max_col_found_count_entity) ecm[1].append(col) else: self.entity_column_mapping.append((max_col_found_count_entity, [col])) else: # no column and entity identified return [] # print([(e[0], [ec.name for ec in e[1]]) for e in self.entity_column_mapping]) # build the sql self.find_relationships() self.find_conditions() self.find_select() self.build_query() print(self.query) return self.run_query()
class DBModel(object): def __init__(self): self.entities = [] self.columns = [] self.relationships = [] self.synonyms_col = [] self.synonyms_tab = [] self.entity_graph = [] self.loaded_entities = [] self.config = Configuration() self.conn = pyodbc.connect(self.config.get_sql_connection_string()) lookups = Lookups() self.lemmatizer = Lemmatizer(lookups) self.load_db_model() def load_db_model(self): # loading the database from sql server cursor = self.conn.cursor() cursor.execute(self.config.get_tables_sql_query()) for row in cursor: self.entities.append( Entities(row.table_name, self.config.get_default_column(row.table_name))) cursor.execute(self.config.get_columns_sql_query()) current_entity = None current_entity_name = "" for row in cursor: if current_entity_name != row.table_name: current_entity_name = row.table_name current_entity = next(en for en in self.entities if en.name == current_entity_name) col_type = row.type_name if col_type == "varchar" or col_type == "nvarchar": col_type = "string" current_entity.columns.append(Columns(row.column_name, col_type)) current_entity = None current_entity_name = "" cursor.execute(self.config.get_FK_sql_query()) for row in cursor: self.relationships.append( Relationship(row.parent_table, row.refrenced_table, row.parent_table_col, row.referenced_table_col)) if len([ en for en in self.entity_graph if en[0] == row.parent_table ]) > 0: current_entity = next(en for en in self.entity_graph if en[0] == row.parent_table) current_entity[1].append(row.refrenced_table) else: self.entity_graph.append( (row.parent_table, [row.refrenced_table])) if len([ en for en in self.entity_graph if en[0] == row.refrenced_table ]) > 0: current_entity = next(en for en in self.entity_graph if en[0] == row.refrenced_table) current_entity[1].append(row.parent_table) else: self.entity_graph.append( (row.refrenced_table, [row.parent_table])) current_entity = None current_entity_name = "" cursor.execute(self.config.get_PK_sql_query()) for row in cursor: if len([en for en in self.entity_graph if en[0] == row.table_name]) == 1: current_entity = next(en for en in self.entities if en.name == row.table_name) current_entity.primaryKey = row.primary_key for entity_to_load in self.config.get_entitites_to_load(): entity_load_query = "select distinct " + entity_to_load[ "column"] + " from " + entity_to_load["entity"] cursor.execute(entity_load_query) entity_data = (entity_to_load["entity"], []) for row in cursor: entity_data[1].append(row[0]) # add lemma strings lemmas = self.lemmatizer(str(row[0]), u'NOUN') for lemma in lemmas: entity_data[1].append(str(lemma)) self.loaded_entities.append(entity_data) # load synonyms from declarative file # table sysnonyms for table_synonym in self.config.get_synonyms()["table"]: orginal_val = table_synonym["original"] synonyms_vals = table_synonym["synonyms"] for synonyms_val in synonyms_vals: self.synonyms_tab.append(Synonyms(orginal_val, synonyms_val)) # column sysnonyms for column_synonym in self.config.get_synonyms()["column"]: orginal_val = column_synonym["original"] synonyms_vals = column_synonym["synonyms"] for synonyms_val in synonyms_vals: self.synonyms_col.append(Synonyms(orginal_val, synonyms_val)) # make a single array self.columns = [ column for entity in self.entities for column in entity.columns ] # might have to write a custom matcher TODO # build the matcher based upon the original value and domain synonyms defined def get_matcher(self, matcher, nlp): for entity in self.entities: matcher.add(entity.name.upper() + "_TABLE", None, nlp(entity.name.lower())) for column in entity.columns: matcher.add(column.name.upper() + "_COLUMN", None, nlp(column.name.lower())) # add table synonyms to matcher for synonym in self.synonyms_tab: for entity in self.entities: if synonym.column.lower() == entity.name.lower(): matcher.add(entity.name.upper() + "_TABLE", None, nlp(synonym.synonym.lower())) # add column synonyms to matcher for synonym in self.synonyms_col: for column in self.columns: if synonym.column.lower() == column.name.lower(): matcher.add(column.name.upper() + "_COLUMN", None, nlp(synonym.synonym.lower())) return matcher def get_custom_matcher(self, matcher, nlp): for entity in self.entities: matcher.add(entity.name.upper() + "_TABLE", nlp(entity.name.lower())) for column in entity.columns: matcher.add(column.name.upper() + "_COLUMN", nlp(column.name.lower())) # add table synonyms to matcher for synonym in self.synonyms_tab: for entity in self.entities: if synonym.column.lower() == entity.name.lower(): matcher.add(entity.name.upper() + "_TABLE", nlp(synonym.synonym.lower())) # add column synonyms to matcher for synonym in self.synonyms_col: for column in self.columns: if synonym.column.lower() == column.name.lower(): matcher.add(column.name.upper() + "_COLUMN", nlp(synonym.synonym.lower())) return matcher
def __init__(self): self.config=Configuration().get_config() self.experiment_parameters_list=Configuration().get_experiments_list() self.device=Utils().get_device()
def __init__(self): self.config = Configuration().get_config() self.device = torch.device( self.config['device']['gpu'] if torch.cuda.is_available( ) else self.config['device']['cpu'])
def __run_training__(): cfg = Configuration() # These variable would be parametrized GPU = True if GPU != True: os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID" os.environ["CUDA_VISIBLE_DEVICES"] = "" # Input Path root_dir = os.path.dirname(os.path.abspath(__file__)) image_path = cfg.image_path json_path = os.path.join(root_dir, cfg.input_filename) trainingset = os.path.join(root_dir, 'trainingset') Preprocessor.__generate_kijiji_set__(root_dir, image_path, json_path, trainingset, 'make') # -------------------------------------------------------------------------------------------------------------- image_path = os.path.join(root_dir, 'trainingset') data_path = glob(image_path + "/*") # Image Segmentation Parameters model_path = os.path.expanduser(cfg.model_path) assert model_path.endswith('.h5'), 'Keras model must be a .h5 file.' anchors_path = os.path.expanduser(cfg.anchors_path) classes_path = os.path.expanduser(cfg.classes_path) test_path = os.path.expanduser(cfg.test_path) output_path = os.path.expanduser(cfg.segmented_output_path) json_path = os.path.expanduser(cfg.json_output) if not os.path.exists(output_path): print('Creating output path {}'.format(output_path)) os.mkdir(output_path) sess = K.get_session() class_names = Preprocessor.__return_class_names__(classes_path) anchors = Preprocessor.__return_anchors__(anchors_path) yolo_model = load_model(model_path) # Verify model, anchors, and classes are compatible num_classes = len(class_names) num_anchors = len(anchors) info = 'Mismatch between model and given anchor and class sizes. ' \ 'Specify matching anchors and classes with --anchors_path and --classes_path flags.' model_output_channels = yolo_model.layers[-1].output_shape[-1] assert model_output_channels == num_anchors * (num_classes + 5), info print('{} model, anchors, and classes loaded.'.format(model_path)) # Check if model is fully convolutional, assuming channel last order. model_image_size = yolo_model.layers[0].input_shape[1:3] is_fixed_size = model_image_size != (None, None) # Generate Colors for drawing bounding boxes hsv_tuples, colors = Preprocessor.__generate_colors_for_bounding_boxes__( class_names) yolo_outputs = yolo_head(yolo_model.output, anchors, len(class_names)) input_image_shape = K.placeholder(shape=(2, )) boxes, scores, classes = yolo_eval(yolo_outputs, input_image_shape, score_threshold=cfg.score_threshold, iou_threshold=cfg.iou_threshold) # Load Images from the root folder input_images_model_1, all_images, data_path, data_path_with_image_name = Preprocessor.__load_image_data_thumbnails__( data_path, cfg.compressed_image_height, cfg.compressed_image_width, cfg.compressed_channel, cfg.number_of_categories, cfg.number_of_images_per_category, root_dir, is_fixed_size, model_image_size, sess, yolo_model, input_image_shape, boxes, scores, classes, cfg.font_path, class_names, colors, output_path, json_path, test_path, True, # Segmentation Flag False, # Edge-detection Flag True, # Extract object Flag False) # Gray Scale Flag input_images_model_2, all_images, data_path, data_path_with_image_name = Preprocessor.__load_image_data_thumbnails__( data_path, cfg.compressed_image_height, cfg.compressed_image_width, cfg.compressed_channel, cfg.number_of_categories, cfg.number_of_images_per_category, root_dir, is_fixed_size, model_image_size, sess, yolo_model, input_image_shape, boxes, scores, classes, cfg.font_path, class_names, colors, output_path, json_path, test_path, False, # Segmentation Flag True, # Edge-detection Flag False, # Extract object Flag False) # Gray Scale Flag input_images_model_3, all_images, data_path, data_path_with_image_name = Preprocessor.__load_image_data_thumbnails__( data_path, cfg.image_height, cfg.image_width, cfg.channel, cfg.number_of_categories, cfg.number_of_images_per_category, root_dir, is_fixed_size, model_image_size, sess, yolo_model, input_image_shape, boxes, scores, classes, cfg.font_path, class_names, colors, output_path, json_path, test_path, False, # Segmentation Flag False, # Edge-detection Flag False, # Extract object Flag False) # Gray Scale Flag input_shape = [ cfg.compressed_image_height, cfg.compressed_image_width, cfg.compressed_channel ] input_shape_3 = [cfg.image_height, cfg.image_width, cfg.channel] # Model Save Paths model_1_save_path = os.path.join(root_dir + cfg.model_1_save) model_2_save_path = os.path.join(root_dir + cfg.model_2_save) model_3_save_path = os.path.join(root_dir + cfg.model_3_save) Preprocessor.__create_output_directories__(model_1_save_path) Preprocessor.__create_output_directories__(model_2_save_path) Preprocessor.__create_output_directories__(model_3_save_path) # Instantiating the training class train = Train(input_images_model_1, input_images_model_2, input_images_model_3, input_shape, input_shape_3, cfg.batch_size, cfg.epochs, model_1_save_path, model_2_save_path, model_3_save_path) # Output Path output_path_model_1 = os.path.join(root_dir + cfg.output_model_1) output_path_model_2 = os.path.join(root_dir + cfg.output_model_2) output_path_model_3 = os.path.join(root_dir + cfg.output_model_3) Preprocessor.__create_output_directories__(output_path_model_1) Preprocessor.__create_output_directories__(output_path_model_2) Preprocessor.__create_output_directories__(output_path_model_3) # FCN Model model_1 = train.__train_model_1__() # VGG Model model_2 = train.__train_model_2__() # Inception-v3 model_3 = train.__train_model_3__() features_from_model_1 = Preprocessor.__get_score_model__( model_1, input_images_model_1, output_path_model_1) features_from_model_2 = Preprocessor.__get_score_model__( model_2, input_images_model_2, output_path_model_2) features_from_model_3 = Preprocessor.__get_score_model__( model_3, input_images_model_3, output_path_model_3) print("Output FeatureMap For Model 1 \n") print(features_from_model_1.shape) print("\n") print("Output FeatureMap For Model 2 \n") print(features_from_model_2.shape) print("\n") print("Output FeatureMap For Model 3 \n") print(features_from_model_3.shape) print("\n")