def main(): # parse config config_file = sys.argv[1] config = Config(config_file) # setup logger setup_logging(config.working_dir) # encoding func encoding_func = ENCODING_METHOD_MAP[config.encoding_method] encoding_func2= ENCODING_METHOD_MAP[config.encoding_method2] log_to_file('Encoding method2', config.encoding_method2) data_provider=[] for p in range(config.base_model_count): temp_provider = DataProvider( encoding_func, encoding_func2, config.data_file, config.test_file, config.batch_size, max_len_hla=config.max_len_hla, max_len_pep=config.max_len_pep, model_count=config.model_count ) data_provider.append(temp_provider) log_to_file('max_len_hla', data_provider[0].max_len_hla) log_to_file('max_len_pep', data_provider[0].max_len_pep) test(config, data_provider[0])
def __init_brand_list(): config_file = r"D:\Users\Achelics\liu_project\mutil_device_recongition\handle_brand_information\database_config.ini" settion = "MyDataBase" db_config = Config(config_file, settion) url = db_config.get("url") user = db_config.get("user") pawd = db_config.get("pawd") database = db_config.get("database") try: # open the database db = MySQLdb.connect(url, user, pawd, database) # Using the cursor() method to get the operate cursor. cursor = db.cursor() # SQL select by vulflag sql_default = "SELECT DISTINCT(en_name) FROM iie_brand ORDER BY LENGTH(en_name) DESC" # excute SQL sentence cursor.execute(sql_default) # Get the all record default_results = cursor.fetchall() for row in default_results: __BRAND_LIST.append(str(row[0]).upper()) # 关闭数据库连接 db.close() except MySQLdb.Error, e: print("MySQL Error:%s" % str(e))
def insert_brand(): config_file = r"D:\Users\Achelics\liu_project\mutil_device_recongition\handle_brand_information\database_config.ini" settion = "MyDataBase" db_config = Config(config_file, settion) url = db_config.get("url") user = db_config.get("user") pawd = db_config.get("pawd") database = db_config.get("database") try: # open the database conn = MySQLdb.connect(url, user, pawd, database) # Using the cursor() method to get the operate cursor. cursor = conn.cursor() for brand_type in __RAW_BRAND_LIST: if str(brand_type[0]).strip().upper() not in __BRAND_LIST: print brand_type default_sql = "insert into iie_brand(en_name, product_type) values('%s', '%s')" % ( brand_type[0], brand_type[1]) cursor.execute(default_sql) # 获取所有结果 conn.commit() # 关闭指针 cursor.close() # 关闭数据库连接 conn.close() except MySQLdb.Error, e: print("MySQL Error:%s" % str(e))
def main(): config = Config() print("Processing text for \'%s\'." % (config.text_file)) data = preprocess.preprocess(config.text_file, 'infer', config) dataloader = dataprocess.load_infer(data, config) G = Generator(config) G.load_state_dict(load_weights(config.checkpoint_file)) G = set_device(G, config.device, config.use_cpu) G.eval() print("Generating spectrogram with \'%s\'." % (config.checkpoint_file)) spec = [] y_prev = torch.zeros(1, config.prev_length, config.fft_size // 2 + 1) for x in tqdm(dataloader, leave=False, ascii=True): x, y_prev = set_device((x, y_prev), config.device, config.use_cpu) y_gen = G(x, y_prev) y_gen = y_gen.squeeze(1) y_prev = y_gen[:, -config.prev_length:, :] spec.append(y_gen.data) print("Generating audio with Griffin-Lim algorithm.") spec = torch.cat(spec, dim=1).transpose(1, 2) # T x D -> D x T wave = dsp.inv_spectrogram(spec, config) savename = config.checkpoint_file.replace('.pt', '_') + os.path.basename( config.text_file).replace('.txt', '.wav') dsp.save(savename, wave, config.sample_rate) print("Audio saved to \'%s\'." % (savename))
def main(): # parse config config_file = sys.argv[1] config = Config(config_file) # setup logger setup_logging(config.working_dir) # encoding func encoding_func = ENCODING_METHOD_MAP[config.encoding_method] encoding_func2 = ENCODING_METHOD_MAP[config.encoding_method2] log_to_file('Encoding method2', config.encoding_method2) data_provider = [] for p in range(config.base_model_count): temp_provider = DataProvider(encoding_func, encoding_func2, config.data_file, config.test_file, config.batch_size, max_len_hla=config.max_len_hla, max_len_pep=config.max_len_pep, model_count=config.model_count) data_provider.append(temp_provider) log_to_file('Traning samples', len(data_provider[0].train_samples[0])) log_to_file('Val samples', len(data_provider[0].validation_samples[0])) log_to_file('Traning steps', data_provider[0].train_steps()) log_to_file('Val steps', data_provider[0].val_steps()) log_to_file('Batch size', data_provider[0].batch_size) log_to_file('max_len_hla', data_provider[0].max_len_hla) log_to_file('max_len_pep', data_provider[0].max_len_pep) for p in range(config.base_model_count): train(config, data_provider[p], p)
def insert_brand_model(): config_file = r"D:\Users\Achelics\liu_project\mutil_device_recongition\handle_brand_information\database_config.ini" settion = "MyDataBase" db_config = Config(config_file, settion) url = db_config.get("url") user = db_config.get("user") pawd = db_config.get("pawd") database = db_config.get("database") try: # open the database conn = MySQLdb.connect(url, user, pawd, database) # Using the cursor() method to get the operate cursor. cursor = conn.cursor() for brand_model in __BRAND_MODEL_LIST: default_sql = "insert into iie_brand_model(brand, model, category) values('%s', '%s', '%s')" % \ (brand_model["Brand"], brand_model["Model"], brand_model["type"]) cursor.execute(default_sql) # 获取所有结果 conn.commit() # 关闭指针 cursor.close() # 关闭数据库连接 conn.close() except MySQLdb.Error, e: print("MySQL Error:%s" % str(e)) print("插入数据库失败")
def main(): # parse config config_file = sys.argv[1] config = Config(config_file) folder = config_file.split('/')[0] encoding_func = ENCODING_METHOD_MAP[config.encoding_method] encoding_func2= ENCODING_METHOD_MAP[config.encoding_method2] data_provider = DataProvider( encoding_func, encoding_func2, config.data_file, config.test_file, config.batch_size, max_len_hla=config.max_len_hla, max_len_pep=config.max_len_pep, model_count=config.model_count ) device = config.device models = config.model_count*config.base_model_count print(models) total_df=pd.DataFrame() for i in range(models): # load and prepare model path = folder + "/best_model_{}.pytorch".format(i) state_dict = torch.load(path) model = Model(config) model.load_state_dict(state_dict) model.to(device) model.eval() data_provider.new_epoch() for _ in range(data_provider.test_steps()): data = data_provider.batch_test() hla_a, hla_mask, hla_a2, hla_mask2, pep, pep_mask, pep2, pep_mask2, uid_list = data temp_attn = {} temp_attn_hla = {} with torch.no_grad(): temp = model.encoder_peptide2.conv_0(pep2.to(device)) temp, att = model.encoder_peptide2.att_0(temp.to(device)) for i in range(config.batch_size): temp_attn[uid_list[i].split('-')[3]]=att[i].tolist() temp_df=pd.DataFrame.from_dict(temp_attn,orient="index") total_df=pd.concat([total_df,temp_df]) avg_= total_df.mean(axis=0) avg_= pd.DataFrame({'position':avg_.index+1, 'avg weight':avg_.values}) avg_.to_csv(folder + "/" + "attn_weight.csv",index=None)
def run(): config = Config('configs.yaml') database = Database(config.database) controller = Controller() while True: controller.fill_wall_post_trigger(database) controller.fill_comment_trigger(database) controller.fill_wall_post_emotion(database) controller.fill_comment_emotion(database) sleep(60 * 60)
def main(): config = Config() config_basename = os.path.basename(config.configs[0]) print("Configuration file: \'%s\'" % (config_basename)) set_list = ['train', 'valid'] file_list = {} # Creating Path for Features create_path(config.feature_path, action='overwrite', verbose=False) for set_type in set_list: path = os.path.join(config.feature_path, set_type) create_path(path, action='overwrite') list_file = set_type + '_list.txt' file_list[set_type] = read_file_list( os.path.join(config.dataset_path, list_file)) # Extracting Features if config.num_proc > 1: if config.use_cpu is False: raise AssertionError("You can not use GPU with multiprocessing.") p = Pool(config.num_proc) for set_type in set_list: p.map(partial(preprocess, set_type=set_type, config=config), file_list[set_type]) else: for set_type in set_list: [ preprocess(f, set_type=set_type, config=config) for f in file_list[set_type] ] # Creating Files Indices for set_type in set_list: path = os.path.join(config.feature_path, set_type) file_indices = make_indices(path) torch.save(file_indices, os.path.join(config.feature_path, set_type + '_indices.pt')) print("Feature saved to \'%s\'." % (config.feature_path))
def create_experiment_info(self): # create subject data directory data_dir = 'data/{}/'.format(self.subject_number.get()) if not os.path.exists(data_dir): os.makedirs(data_dir) else: files = glob.glob(data_dir + '*') [os.remove(f) for f in files] # create config object config = Config(config_filename=self.config_filename.get()) # create trial list and save to csv config.save_trial_list(path=data_dir) print "Trial list created as {}.".format(data_dir + 'trials.csv') # create experiment info and save to txt config.save_experiment_info(glove_hand=self.hand.get(), path=data_dir) print "Experiment info created as {}.".format(data_dir + 'exp_info.txt')
def main(): config = Config() config_basename = os.path.basename(config.configs[0]) print("Configuration file: \'%s\'" % (config_basename)) checkpoint_path = create_path(config.checkpoint_path, action=config.checkpoint_path_action) config.save(os.path.join(checkpoint_path, config_basename)) logger = Logger(os.path.join(checkpoint_path, 'log')) dataloader = dataprocess.load_train(config) step_size = config.step_epoch * len(dataloader.train) G = Generator(config) D = Discriminator(config) G, D = set_device((G, D), config.device, config.use_cpu) criterionL1 = nn.L1Loss() optimizerG = torch.optim.Adam(G.parameters(), lr=config.learn_rate, betas=config.betas, weight_decay=config.weight_decay) optimizerD = torch.optim.Adam(D.parameters(), lr=config.learn_rate, betas=config.betas, weight_decay=config.weight_decay) schedulerG = StepLR(optimizerG, step_size=step_size, gamma=config.decay_factor) schedulerD = StepLR(optimizerD, step_size=step_size, gamma=config.decay_factor) k = 0.0 M = AverageMeter() lossG_train = AverageMeter() lossG_valid = AverageMeter() lossD_train = AverageMeter() print('Training start') for epoch in range(config.stop_epoch + 1): # Training Loop G.train() D.train() for batch in tqdm(dataloader.train, leave=False, ascii=True): x, y_prev, y = set_device(batch, config.device, config.use_cpu) y = y.unsqueeze(1) optimizerG.zero_grad() y_gen = G(x, y_prev) lossL1 = criterionL1(y_gen, y) loss_advG = criterionAdv(D, y_gen) lossG = lossL1 + loss_advG lossG.backward() optimizerG.step() schedulerG.step() optimizerD.zero_grad() loss_real = criterionAdv(D, y) loss_fake = criterionAdv(D, y_gen.detach()) loss_advD = loss_real - k * loss_fake loss_advD.backward() optimizerD.step() schedulerD.step() diff = torch.mean(config.gamma * loss_real - loss_fake) k = k + config.lambda_k * diff.item() k = min(max(k, 0), 1) measure = (loss_real + torch.abs(diff)).data M.step(measure, y.size(0)) logger.log_train(lossL1, loss_advG, lossG, loss_real, loss_fake, loss_advD, M.avg, k, lossG_train.steps) lossG_train.step(lossG.item(), y.size(0)) lossD_train.step(loss_advD.item(), y.size(0)) # Validation Loop G.eval() D.eval() for batch in tqdm(dataloader.valid, leave=False, ascii=True): x, y_prev, y = set_device(batch, config.device, config.use_cpu) y = y.unsqueeze(1) y_gen = G(x, y_prev) lossL1 = criterionL1(y_gen, y) loss_advG = criterionAdv(D, y_gen) lossG = lossL1 + loss_advG logger.log_valid(lossL1, loss_advG, lossG, lossG_valid.steps) lossG_valid.step(lossG.item(), y.size(0)) for param_group in optimizerG.param_groups: learn_rate = param_group['lr'] print( "[Epoch %d/%d] [loss G train: %.5f] [loss G valid: %.5f] [loss D train: %.5f] [lr: %.6f]" % (epoch, config.stop_epoch, lossG_train.avg, lossG_valid.avg, lossD_train.avg, learn_rate)) lossG_train.reset() lossG_valid.reset() lossD_train.reset() savename = os.path.join(checkpoint_path, 'latest_') save_checkpoint(savename + 'G.pt', G, optimizerG, learn_rate, lossG_train.steps) save_checkpoint(savename + 'D.pt', D, optimizerD, learn_rate, lossD_train.steps) if epoch % config.save_epoch == 0: savename = os.path.join(checkpoint_path, 'epoch' + str(epoch) + '_') save_checkpoint(savename + 'G.pt', G, optimizerG, learn_rate, lossG_train.steps) save_checkpoint(savename + 'D.pt', D, optimizerD, learn_rate, lossD_train.steps) print('Training finished')
from time import sleep from src.controller import Controller from src.database import Database from config_parser import Config config = Config('configs.yaml') database = Database(config.database) controller = Controller() while True: controller.fill_trigger(database) controller.fill_emotion(database) print("Update completed") sleep(60 * 60)
def __init__(self, enviornment=None): self.__config_obj = Config(enviornment) self.__oracle_runner = OracleRunner(self.__config_obj.db_user, self.__config_obj.db_password, self.__config_obj.db_sid, False, self.__config_obj.log_path)
def run(): config = Config('configs.yaml') vk = VK(config.vk) database = Database(config.database) controller = Controller(config.controller) group_urls_path = 'data/vk/group_short_url.txt' n = 0 while True: offset = n * 100 for url in get_data(group_urls_path): group_info = controller.get_group_info(vk, url) group_id = group_info[0]['id'] try: wall_posts = controller.get_target_post( vk, group_id, offset, url) for post in wall_posts: post['Text'] = ' '.join( list(filter(None, re.split('\W|\d', post['Text'])))) post['PublishDateTime'] = handle_date( post['PublishDateTime']) controller.save_data_to_bd(bd=database, table='WallPost', target_object=post) for post in wall_posts: comments = controller.get_target_comments( vk, group_id, post_id=post['OuterId'], offset=0) for comment in comments: comment['Text'] = ' '.join( list( filter(None, re.split('\W|\d', comment['Text'])))) comment['PublishDateTime'] = handle_date( comment['PublishDateTime']) controller.save_data_to_bd(bd=database, table='Comment', target_object=comment) for comment in comments: user = controller.get_target_user( vk, comment['OuterId']) controller.save_data_to_bd(bd=database, table='SocialNetworkUser', target_object=user) except Exception as e: print(e) n += 1
cursor = connection.cursor() cursor.execute(sql_select_Query) records = cursor.fetchall() print("Total number of rows in Laptop is: ", cursor.rowcount) print("\nPrinting each laptop record") if db_name == "TriggerWord": return self.get_trigger_word_list(records) if db_name == "WallPost": return self.get_wall_post_list(records) if db_name == 'Comment': return self.get_comment_list(records) except Error as e: print("Error reading data from MySQL table", e) finally: if (connection.is_connected()): connection.close() cursor.close() print("MySQL connection is closed") if __name__ == "__main__": from config_parser import Config config = Config("../configs.yaml") db = Database(config.database) L = db.select('WallPost') print(L)
def main(): config = Config("dup_0/config.json") # get HLA and peptide HLA = sys.argv[1] peptide = sys.argv[2] if len(peptide) > 15: print("please input the peptide shorter than 16 amino acids.") return hla_seq = 0 # get the sequence of HLA hla_path = os.path.join(BASE_DIR, '..', 'dataset', 'mhc_i_protein_seq2.txt') with open(hla_path, 'r') as in_file: for line_num, line in enumerate(in_file): if line_num == 0: continue info = line.strip('\n').split(' ') if info[0] != HLA: continue hla_seq = info[1] break if hla_seq == 0: print("The HLA is not included in the dataset.") return # encode the sequences encoding_func = ENCODING_METHOD_MAP["one_hot"] encoding_func2 = ENCODING_METHOD_MAP["blosum"] hla, hla_mask = encoding_func(hla_seq, 385) hla = torch.reshape(hla, (1, hla.size(0), hla.size(1))) pep, pep_mask = encoding_func(peptide, 15) pep = torch.reshape(pep, (1, pep.size(0), pep.size(1))) hla2, hla_mask2 = encoding_func2(hla_seq, 385) hla2 = torch.reshape(hla2, (1, hla2.size(0), hla2.size(1))) pep2, pep_mask2 = encoding_func2(peptide, 15) pep2 = torch.reshape(pep2, (1, pep2.size(0), pep2.size(1))) # load model device = config.device temp_list = [] for p in range(config.base_model_count): for k in range(config.model_count): state_dict = torch.load( config.model_save_path(p * config.model_count + k)) model = Model(config) model.load_state_dict(state_dict) model.to(device) model.eval() with torch.no_grad(): pred_ic50 = model(hla.to(device), hla_mask.to(device), hla2.to(device), hla_mask2.to(device), pep.to(device), pep_mask.to(device), pep2.to(device), pep_mask2.to(device)) pred_ic50 = math.pow(50000, 1 - pred_ic50) temp_list.append(pred_ic50) pred_ic50 = sum(temp_list) / len(temp_list) print("the predicted IC50 value is : {}".format(pred_ic50))
import os import sys from github import Github import argparse from envdefault import EnvDefault import checks from config_parser import Config import openpyxl config = Config("config.yml") row = 1 def parse_arguments(): """ Parse them args """ parser = argparse.ArgumentParser() parser.add_argument( '-t', '--token', action=EnvDefault, envvar='GIT_API_TOKEN', help="GitHub token: set using -t or setting env var 'GIT_API_TOKEN'") parser.add_argument('-e', '--enterprise', action='store_true', help="Enable Github Enterprise") org_repo_group = parser.add_mutually_exclusive_group(required=True) org_repo_group.add_argument('-o', '--organization',
from config_parser import Config import json import viz import vizact import numpy as np from vis_env import Room, BaseballGlove from stimuli import Ball config_filename = 'config/home.py' config = Config(config_filename) config.save_experiment_info(path='figures/') # read experiment info (config dict) with open('figures/exp_info.txt') as info_file: config = json.load(info_file) room = Room(config) #ball = Ball(room, size=0.15, position=[0,1.8,4], color=[1, 1, 1]) #ball.setContrastMode('constant_dark') glove = viz.addChild('pole.wrl') def rotate_glove(): direction = glove.getMatrix().getForward() print direction print glove.getQuat() #direction[0] = -direction[0] #glove.lookAt(direction, viz.REL_LOCAL) #print glove.getMatrix().getForward()