def cli(context, args): #this import is only available when you execute via cli import util util.set_logger(args.verbose) from resource_manager_common import constant credentials = context.aws.load_credentials() resources = util.get_resources(context) os.environ[c.ENV_DB_TABLE_CONTEXT] = resources[c.RES_DB_TABLE_CONTEXT] os.environ[c.ENV_LAMBDA_CONSUMER] = resources[c.RES_LAMBDA_FIFOCONSUMER] os.environ[c.ENV_LAMBDA_PRODUCER] = resources[c.RES_LAMBDA_FIFOPRODUCER] os.environ[c.ENV_AMOEBA] = resources[c.RES_AMOEBA] os.environ[c.ENV_VERBOSE] = str(args.verbose) if args.verbose else "" os.environ[c.ENV_REGION] = context.config.project_region os.environ[c.ENV_S3_STORAGE] = resources[c.RES_S3_STORAGE] os.environ[c.ENV_DEPLOYMENT_STACK_ARN] = resources[c.ENV_STACK_ID] os.environ["AWS_LAMBDA_FUNCTION_NAME"] = os.environ[c.ENV_LAMBDA_PRODUCER] os.environ[ "AWS_ACCESS_KEY"] = args.aws_access_key if args.aws_access_key else credentials.get( args.profile if args.profile else context.config.user_default_profile, constant.ACCESS_KEY_OPTION) os.environ[ "AWS_SECRET_KEY"] = args.aws_secret_key if args.aws_secret_key else credentials.get( args.profile if args.profile else context.config.user_default_profile, constant.SECRET_KEY_OPTION) print( query(type('obj', (object, ), {c.ENV_STACK_ID: resources[c.ENV_STACK_ID]}), {"sql": args.sql}, sync=True))
def cli(context, args): util.set_logger(args.verbose) from resource_manager_common import constant credentials = context.aws.load_credentials() resources = util.get_resources(context) os.environ[c.ENV_SHARED_BUCKET] = context.config.configuration_bucket_name os.environ[c.ENV_S3_STORAGE] = resources[c.RES_S3_STORAGE] os.environ[c.ENV_DB_TABLE_CONTEXT] = resources[c.RES_DB_TABLE_CONTEXT] os.environ[c.ENV_VERBOSE] = str(args.verbose) if args.verbose else "" os.environ[c.ENV_SERVICE_ROLE] = resources[c.RES_SERVICE_ROLE] os.environ[c.ENV_REGION] = context.config.project_region os.environ[c.ENV_DEPLOYMENT_STACK_ARN] = resources[c.ENV_STACK_ID] os.environ[c.ENV_EVENT_EMITTER] = resources[c.RES_EVENT_EMITTER] os.environ[c.IS_LOCALLY_RUN] = "True" os.environ[ "AWS_ACCESS_KEY"] = args.aws_access_key if args.aws_access_key else credentials.get( args.profile if args.profile else context.config.user_default_profile, constant.ACCESS_KEY_OPTION) os.environ[ "AWS_SECRET_KEY"] = args.aws_secret_key if args.aws_secret_key else credentials.get( args.profile if args.profile else context.config.user_default_profile, constant.SECRET_KEY_OPTION) main({c.ENV_STACK_ID: resources[c.ENV_STACK_ID]}, type('obj', (object, ), {}))
def cli(context, args): util.set_logger(args.verbose) from resource_manager_common import constant credentials = context.aws.load_credentials() resources = util.get_resources(context) os.environ[c.ENV_REGION] = context.config.project_region os.environ[c.ENV_VERBOSE] = str(args.verbose) if args.verbose else "" os.environ[c.IS_LOCALLY_RUN] = "True" os.environ[ "AWS_ACCESS_KEY"] = args.aws_access_key if args.aws_access_key else credentials.get( args.profile if args.profile else context.config.user_default_profile, constant.ACCESS_KEY_OPTION) os.environ[ "AWS_SECRET_KEY"] = args.aws_secret_key if args.aws_secret_key else credentials.get( args.profile if args.profile else context.config.user_default_profile, constant.SECRET_KEY_OPTION) eval(args.function)( { 'RequestType': args.event_type, c.ENV_STACK_ID: resources[c.ENV_STACK_ID] }, type('obj', (object, ), {'function_name': resources[c.RES_LAMBDA_FIFOCONSUMER]}))
def __init__(self, dir): """ Constructor Args: dir (string) directory of the experiment to be run """ super(PermutationTest, self).__init__(dir) # Set the logger set_logger(os.path.join(self.dir, 'experiment.log'), level=logging.INFO, console=True) # unpack parameters self.ppi_matrices = { name: np.load(file) for name, file in self.params.ppi_matrices.items() } self.exclude = set(self.params.exclude) if hasattr( self.params, "exclude") else set() # Log title logging.info("Metric Significance of Diseases in the PPI Network") logging.info("Sabri Eyuboglu -- SNAP Group") logging.info("======================================") logging.info("Loading Disease Associations...") self.diseases = load_diseases(self.params.diseases_path, self.params.disease_subset, exclude_splits=['none']) self.metric_fn = globals()[self.params.metric_fn]
def cli(context, args): util.set_logger(args.verbose) from resource_manager_common import constant credentials = context.aws.load_credentials() resources = util.get_resources(context) os.environ[c.ENV_DB_TABLE_CONTEXT] = resources[c.RES_DB_TABLE_CONTEXT] os.environ[c.ENV_DEPLOYMENT_STACK_ARN] = resources[ c.RES_LAMBDA_FIFOCONSUMER] os.environ[c.ENV_LAMBDA_PRODUCER] = resources[c.RES_LAMBDA_FIFOPRODUCER] os.environ[c.ENV_AMOEBA] = resources[c.RES_AMOEBA] os.environ[c.ENV_VERBOSE] = str(args.verbose) if args.verbose else "" os.environ[c.ENV_REGION] = context.config.project_region os.environ[c.ENV_S3_STORAGE] = resources[c.RES_S3_STORAGE] os.environ[ "AWS_ACCESS_KEY"] = args.aws_access_key if args.aws_access_key else credentials.get( args.profile if args.profile else context.config.user_default_profile, constant.ACCESS_KEY_OPTION) os.environ[ "AWS_SECRET_KEY"] = args.aws_secret_key if args.aws_secret_key else credentials.get( args.profile if args.profile else context.config.user_default_profile, constant.SECRET_KEY_OPTION) print( eval(args.function)(type('obj', (object, ), {c.ENV_STACK_ID: resources[c.ENV_STACK_ID]})))
def cli(context, args): util.set_logger(args.verbose) from resource_manager_common import constant credentials = context.aws.load_credentials() resources = util.get_resources(context) os.environ[c.ENV_DB_TABLE_CONTEXT] = resources[c.RES_DB_TABLE_CONTEXT] os.environ[c.ENV_VERBOSE] = str(args.verbose) if args.verbose else "" os.environ['err'] = str( args.erroneous_metrics) if args.erroneous_metrics else "" os.environ[c.ENV_REGION] = context.config.project_region os.environ["AWS_LAMBDA_FUNCTION_NAME"] = resources[ c.RES_LAMBDA_FIFOPRODUCER] os.environ[ "AWS_ACCESS_KEY"] = args.aws_access_key if args.aws_access_key else credentials.get( args.profile if args.profile else context.config.user_default_profile, constant.ACCESS_KEY_OPTION) os.environ[ "AWS_SECRET_KEY"] = args.aws_secret_key if args.aws_secret_key else credentials.get( args.profile if args.profile else context.config.user_default_profile, constant.SECRET_KEY_OPTION) os.environ[c.ENV_LAMBDA_PRODUCER] = resources[c.RES_LAMBDA_FIFOPRODUCER] os.environ[c.ENV_DEPLOYMENT_STACK_ARN] = resources[c.ENV_STACK_ID] generate_threads(resources[c.RES_LAMBDA_FIFOPRODUCER], args.threads, args.iterations_per_thread, args.events_per_iteration, args.sleep_duration_between_jobs, args.use_lambda, args.event_type, args.sensitivity_type, args.compression_type)
def cli(context, args): util.set_logger(args.verbose) from resource_manager_common import constant credentials = context.aws.load_credentials() resources = util.get_resources(context) os.environ[c.ENV_REGION] = context.config.project_region os.environ[c.ENV_DB_TABLE_CONTEXT] = resources[c.RES_DB_TABLE_CONTEXT] os.environ["AWS_ACCESS_KEY"] = args.aws_access_key if args.aws_access_key else credentials.get(args.profile if args.profile else context.config.user_default_profile, constant.ACCESS_KEY_OPTION) os.environ["AWS_SECRET_KEY"] = args.aws_secret_key if args.aws_secret_key else credentials.get(args.profile if args.profile else context.config.user_default_profile, constant.SECRET_KEY_OPTION) print eval(args.function)({}, args.param )
def cli(context, args): #this import is only available when you execute via cli import util util.set_logger(args.verbose) from resource_manager_common import constant credentials = context.aws.load_credentials() resources = util.get_resources(context) os.environ[c.ENV_DB_TABLE_CONTEXT] = resources[c.RES_DB_TABLE_CONTEXT] os.environ[c.ENV_REGION] = context.config.project_region os.environ["AWS_ACCESS_KEY"] = args.aws_access_key if args.aws_access_key else credentials.get(args.profile if args.profile else context.config.user_default_profile, constant.ACCESS_KEY_OPTION) os.environ["AWS_SECRET_KEY"] = args.aws_secret_key if args.aws_secret_key else credentials.get(args.profile if args.profile else context.config.user_default_profile, constant.SECRET_KEY_OPTION) print eval(args.function)( type('obj', (object,), {c.ENV_STACK_ID: resources[c.ENV_STACK_ID]}))
def __init__( self, filename_prefix="output", filename_suffix="raw", rate=44100, channels=2, card="default", log_level=logging.INFO, ): threading.Thread.__init__(self) self.daemon = True self.filename_prefix = filename_prefix self.filename_suffix = filename_suffix self.filename = None self.rate = rate self.channels = channels self.card = card self.recording = False self.finish = False # set logger self.logger = util.set_logger(level=log_level) # setup input self.input = alsaaudio.PCM(alsaaudio.PCM_CAPTURE, alsaaudio.PCM_NONBLOCK, self.card) self.input.setchannels(self.channels) self.input.setrate(self.rate) self.input.setformat(alsaaudio.PCM_FORMAT_S16_LE) self.input.setperiodsize(self.PERIOD_SIZE) self.file = None
def __init__(self, filename_prefix='output', filename_suffix='h264', log_level=logging.INFO, format='h264', horizontal_resolution=640, vertical_resolution=480, rotation=0, snapshot_filename="snapshot.png" ): threading.Thread.__init__(self) self.daemon = True self.filename_prefix = filename_prefix self.filename_suffix = filename_suffix self.filename = "%s.%s" % (self.filename_prefix, self.filename_suffix) self.recording = False self.format = format self.finish = False self.snapshot_filename = snapshot_filename # initialize camera self.horizontal_resolution = horizontal_resolution self.vertical_resolution = vertical_resolution self.rotation = rotation self.camera = picamera.PiCamera() self.camera.resolution = (self.horizontal_resolution, self.vertical_resolution) self.camera.rotation = self.rotation # set logger self.logger = util.set_logger(level=log_level)
def __init__(self, endpoint, log_level=logging.INFO, port=8883, keepalive=60): # set logger self.logger = util.set_logger(level=log_level) # tls self.ca_file = None self.client_cert_file = None self.client_key_file = None self.client_id = ''.join([random.choice(string.ascii_letters + string.digits) for n in range(16)]) self._client = mqtt.Client(client_id=self.client_id) self.endpoint = endpoint self.port = port self.keepalive = keepalive # set callbacks self._client.on_connect = self.mqtt_on_connect self._client.on_disconnect = self.mqtt_on_disconnect self._client.on_message = self.mqtt_on_message self._client.on_publish = self.mqtt_on_publish self._client.on_subscribe = self.mqtt_on_subscribe self._client.on_log = self.mqtt_on_log # subscriptions self.subscriptions = []
def __init__(self, pin, name='sensor', log_level=logging.INFO): threading.Thread.__init__(self) self.name = name self.pin = pin self.finish = False self.daemon = True self.logger = util.set_logger(level=log_level) self.last_reading = None GPIO.setwarnings(False) GPIO.setmode(GPIO.BCM) GPIO.setup(self.pin, GPIO.IN, pull_up_down=GPIO.PUD_UP)
def __init__(self, pin, name='relay', pulse_duration=1, log_level=logging.INFO): threading.Thread.__init__(self) self.name = name self.pin = pin self.pulse_duration = pulse_duration self.finish = False self.daemon = True self.logger = util.set_logger(level=log_level) self.last_reading = None # initialize relay wiringpi.wiringPiSetup()
def __init__(self, dir): super(CodiseaseProbExp, self).__init__(dir) # Set the logger set_logger(os.path.join(self.dir, 'experiment.log'), level=logging.INFO, console=True) # Log title logging.info("Co-disease probability in the PPI Network") logging.info("Sabri Eyuboglu -- SNAP Group") logging.info("======================================") logging.info("Loading Network...") self.ppi_networkx, self.ppi_adj, self.protein_to_node = load_network( self.params.ppi_network) logging.info("Loading Disease Associations...") self.diseases = load_diseases(self.params.diseases_path, self.params.disease_subset) # unpack params self.ppi_matrices = { name: np.load(file) for name, file in self.params.ppi_matrices.items() } self.top_k = self.params.top_k self.n_buckets = self.params.n_buckets self.window_length = self.params.window_length self.smooth = self.params.smooth self.plots = self.params.plots if hasattr(self.params, 'codisease_matrix'): logging.info("Loading Codisease Matrix...") self.codisease_matrix = np.load(self.params.codisease_matrix) else: logging.info("Building Codisease Matrix...") self.codisease_matrix = self.build_codisease_matrix()
def __init__(self, pin, name='pir', log_level=logging.INFO): threading.Thread.__init__(self) self.name = name self.pin = pin self.finish = False self.daemon = True self.logger = util.set_logger(level=log_level) self.last_reading = None # for GPIO numbering, choose BCM GPIO.setmode(GPIO.BCM) GPIO.setwarnings(False) # initialize motion sensor GPIO.setup(self.pin, GPIO.IN, GPIO.PUD_DOWN)
def __init__(self): self.movie_df = bf.get_dataframe() self.episode_df = ef.get_episodes() self.df = None self.logger = set_logger('recommendation') self.matrix = None self.random_array = None self.create_df() self.preproccessing() self.clustering() self.create_matrix() self.create_array()
def __init__(self, name, pin, log_level=logging.INFO): threading.Thread.__init__(self) self.name = name self.pin = pin self.finish = False self.daemon = True # set logger self.logger = util.set_logger(level=log_level) # for GPIO numbering, choose BCM GPIO.setmode(GPIO.BCM) GPIO.setwarnings(False) # set pin GPIO.setup(self.pin, GPIO.OUT) GPIO.output(self.pin, False) # on start , turn LED off
def _zh_experiment0_helper(vec_type, num_units, projection, num_hidden_layers, exp_name, relation_list, label_vector, label_alphabet): params = [vec_type, str(num_units), projection, str(num_hidden_layers)] file_name = '%s-%s' % (experiment_name, '-'.join(params)) json_file = util.set_logger(file_name) word2vec_ff = util._get_zh_word2vec_ff(num_units, vec_type, projection) data_matrix_pair = word2vec_ff(relation_list) learning_rate = 0.001 lr_smoother = 0.01 num_folds = 7 for fold_index in xrange(num_folds): data_triplet = get_xvalidated_datatriplet(data_matrix_pair, label_vector, label_alphabet, num_folds, fold_index) num_reps = 15 num_hidden_units_list = [50, 200, 300, 400] for num_hidden_units in num_hidden_units_list: rng = np.random.RandomState(100) X_list = [T.matrix(), T.matrix()] net, layers = make_multilayer_net(rng, n_in_list=data_triplet.input_dimensions(), X_list=X_list, Y=T.lvector(), use_sparse=False, num_hidden_layers=num_hidden_layers, num_hidden_units=num_hidden_units, num_output_units=data_triplet.output_dimensions()[0], output_activation_fn=T.nnet.softmax, dropout=False) trainer = AdagradTrainer(net, net.crossentropy, learning_rate, lr_smoother, data_triplet, _make_givens) for rep in xrange(num_reps): random_seed = rep rng = np.random.RandomState(random_seed) net.reset(rng) trainer.reset() minibatch_size = np.random.randint(20, 60) n_epochs = 50 start_time = timeit.default_timer() best_iter, best_dev_acc, best_test_acc = \ trainer.train_minibatch_triplet(minibatch_size, n_epochs) end_time = timeit.default_timer() print 'Training process takes %s seconds' % (end_time - start_time) print 'Best iteration is %s;' % best_iter + \ 'Best dev accuracy = %s' % best_dev_acc + \ 'Test accuracy =%s' % best_test_acc
parser.add_argument('--experiment_dir', default='experiments/base_model', help="Directory containing params.json") if __name__ == '__main__': # Load the parameters from the experiment params.json file in model_dir args = parser.parse_args() json_path = os.path.join(args.experiment_dir, 'params.json') assert os.path.isfile( json_path), "No json configuration file found at {}".format(json_path) params = Params(json_path) params.update(json_path) # Set the logger set_logger(os.path.join(args.experiment_dir, 'experiment.log'), level=logging.INFO, console=True) # Log Title logging.info("DPP-Diff Generator") logging.info("Sabri Eyuboglu -- SNAP Group") logging.info("======================================") prepare_sns(sns, params) diseases_dict = load_diseases(params.diseases_path, params.disease_subset) method_to_scores = {} for method_name, method_exp_dir in params.method_exp_dirs.items(): method_to_scores[method_name] = {} with open(os.path.join(method_exp_dir, 'metrics.csv'),
def main(): """Create the model and start the training.""" if not os.path.exists(args.snapshot_dir): os.makedirs(args.snapshot_dir) logger = util.set_logger(args.snapshot_dir, args.log_file, args.debug) logger.info('start with arguments %s', args) h, w = map(int, args.input_size.split(',')) input_size = (h, w) lscale, hscale = map(float, args.train_scale.split(',')) train_scale = (lscale, hscale) cudnn.enabled = True # Create network. model = Res_Deeplab(num_classes=args.num_classes) #saved_state_dict = torch.load(args.restore_from) #new_params = model.state_dict().copy() #for i in saved_state_dict: # #Scale.layer5.conv2d_list.3.weight # i_parts = i.split('.') # # print i_parts # if not args.num_classes == 21 or not i_parts[1]=='layer5': # new_params['.'.join(i_parts[1:])] = saved_state_dict[i] model_urls = { 'resnet18': 'https://download.pytorch.org/models/resnet18-5c106cde.pth', 'resnet34': 'https://download.pytorch.org/models/resnet34-333f7ec4.pth', 'resnet50': 'https://download.pytorch.org/models/resnet50-19c8e357.pth', 'resnet101': 'https://download.pytorch.org/models/resnet101-5d3b4d8f.pth', 'resnet152': 'https://download.pytorch.org/models/resnet152-b121ed2d.pth', } saved_state_dict = torch.utils.model_zoo.load_url(model_urls['resnet101']) # coco pretrained parameters: # saved_state_dict = torch.load(args.restore_from) new_params = model.state_dict().copy() for i in saved_state_dict: #Scale.layer5.conv2d_list.3.weight i_parts = str(i).split('.') # print i_parts if not i_parts[0] == 'fc': new_params['.'.join(i_parts[0:])] = saved_state_dict[i] model.load_state_dict(new_params) #model.float() model.eval() # use_global_stats = True #model.train() device = torch.device("cuda:" + str(args.gpu)) model.to(device) cudnn.benchmark = True trainloader = data.DataLoader(GTA5DataSet(args.data_dir, args.data_list, max_iters=args.num_steps * args.batch_size, crop_size=input_size, train_scale=train_scale, scale=args.random_scale, mirror=args.random_mirror, mean=IMG_MEAN, std=IMG_STD), batch_size=args.batch_size, shuffle=True, num_workers=5, pin_memory=args.pin_memory) optimizer = optim.SGD([{ 'params': get_1x_lr_params_NOscale(model), 'lr': args.learning_rate }, { 'params': get_10x_lr_params(model), 'lr': 10 * args.learning_rate }], lr=args.learning_rate, momentum=args.momentum, weight_decay=args.weight_decay) optimizer.zero_grad() interp = nn.Upsample(size=input_size, mode='bilinear', align_corners=True) for i_iter, batch in enumerate(trainloader): images, labels, _, _ = batch images = images.to(device) labels = labels.long().to(device) optimizer.zero_grad() adjust_learning_rate(optimizer, i_iter) pred = interp(model(images)) loss = loss_calc(pred, labels) loss.backward() optimizer.step() # print('iter = ', i_iter, 'of', args.num_steps,'completed, loss = ', loss.data.cpu().numpy()) logger.info('iter = {} of {} completed, loss = {:.4f}'.format( i_iter, args.num_steps, loss.data.cpu().numpy())) if i_iter >= args.num_steps - 1: print('save model ...') torch.save( model.state_dict(), osp.join(args.snapshot_dir, 'VOC12_scenes_' + str(args.num_steps) + '.pth')) break if i_iter % args.save_pred_every == 0 and i_iter != 0: print('taking snapshot ...') torch.save( model.state_dict(), osp.join(args.snapshot_dir, 'VOC12_scenes_' + str(i_iter) + '.pth')) end = timeit.default_timer() print(end - start, 'seconds')
df = pd.read_csv(os.path.join(path, 'rates.csv'))[:n_sample] train_df, val_df = train_test_split(df, test_size=.2, random_state=42) user_to_index = {user: id for id, user in enumerate(df['user'].unique())} movie_to_index = { movie: id for id, movie in enumerate(df['movie'].unique()) } return train_df, val_df, user_to_index, movie_to_index train_df, val_df, user_to_index, movie_to_index = read_data(DATA_DIR) if __name__ == '__main__': logger = set_logger(name='dataloader', file_path=LOGGER_PATH) train_df, val_df, user_to_index, movie_to_index = read_data(DATA_DIR) train_dataset = KMRDataset(train_df, user_to_index, movie_to_index) val_dateset = KMRDataset(val_df, user_to_index, movie_to_index) train_dataloader = DataLoader(train_dataset, batch_size=64, shuffle=True) val_dataloader = DataLoader(val_dateset, batch_size=16, shuffle=True) logger.info(f"Load data from {DATA_DIR}") logger.info(f"Train_df: {train_df.shape}, {train_dataset.data[0].size()}") logger.info( f"Validaetion df: {val_df.shape}, {val_dateset.data[0].size()}") save_pickle(os.path.join(DATA_LOADER_DIR, 'dataloader_train.pickle'), train_dataloader) save_pickle(os.path.join(DATA_LOADER_DIR, 'dataloader_val.pickle'),
%load_ext autoreload %autoreload 2 """ import json from datetime import datetime from typing import Dict from pathlib import Path import torch import torch.optim as optim import matplotlib.pyplot as plt from data_processor import ConllDataSet import mst_lstm from util import set_logger logger = set_logger("train.py") torch.manual_seed(0) def train(config_dict:Dict): # Directory for storing result result_dir_path = Path("result",str(datetime.now())) result_dir_path.mkdir() # Store the config in json config_path = result_dir_path / "config.json" with config_path.open(mode="w") as fp: json.dump(config_dict, fp) # Read data train_path = Path("data","en-universal-train.conll") train_data = ConllDataSet(conll_path=train_path,
def main(): """Create the model and start the evaluation process.""" device = torch.device("cuda:" + str(args.gpu)) if not os.path.exists(args.save): os.makedirs(args.save) logger = util.set_logger(args.save, args.log_file, args.debug) logger.info('start with arguments %s', args) x_num = 0 with open(args.data_list) as f: for _ in f.readlines(): x_num = x_num + 1 sys.path.insert(0, 'dataset/helpers') if args.data_src == 'gta' or args.data_src == 'cityscapes': from labels import id2label, trainId2label elif args.data_src == 'synthia': from labels_cityscapes_synthia import id2label, trainId2label # label_2_id = 255 * np.ones((256, )) for l in id2label: if l in (-1, 255): continue label_2_id[l] = id2label[l].trainId id_2_label = np.array( [trainId2label[_].id for _ in trainId2label if _ not in (-1, 255)]) valid_labels = sorted(set(id_2_label.ravel())) scorer = ScoreUpdater(valid_labels, args.num_classes, x_num, logger) scorer.reset() if args.model == 'DeeplabRes': model = Res_Deeplab(num_classes=args.num_classes) # elif args.model == 'DeeplabVGG': # model = DeeplabVGG(num_classes=args.num_classes) # if args.restore_from == RESTORE_FROM: # args.restore_from = RESTORE_FROM_VGG if args.restore_from[:4] == 'http': saved_state_dict = model_zoo.load_url(args.restore_from) new_params = model.state_dict().copy() for i in saved_state_dict: # Scale.layer5.conv2d_list.3.weight i_parts = str(i).split('.') # print i_parts if not i_parts[0] == 'fc': new_params['.'.join(i_parts[0:])] = saved_state_dict[i] else: loc = "cuda:" + str(args.gpu) saved_state_dict = torch.load(args.restore_from, map_location=loc) new_params = saved_state_dict.copy() model.load_state_dict(new_params) #model.train() model.eval() model.to(device) testloader = data.DataLoader(GTA5TestDataSet(args.data_dir, args.data_list, test_scale=1.0, test_size=(1024, 512), mean=IMG_MEAN, std=IMG_STD, scale=False, mirror=False), batch_size=1, shuffle=False, pin_memory=True) test_scales = [float(_) for _ in str(args.test_scale).split(',')] h, w = map(int, args.test_image_size.split(',')) if version.parse(torch.__version__) >= version.parse('0.4.0'): interp = nn.Upsample(size=(h, w), mode='bilinear', align_corners=True) else: interp = nn.Upsample(size=(h, w), mode='bilinear') test_image_size = (h, w) mean_rgb = IMG_MEAN[::-1].copy() std_rgb = IMG_STD[::-1].copy() with torch.no_grad(): for index, batch in enumerate(testloader): image, label, _, name = batch img = image.clone() num_scales = len(test_scales) # output_dict = {k: [] for k in range(num_scales)} for scale_idx in range(num_scales): if version.parse(torch.__version__) > version.parse('0.4.0'): image = F.interpolate(image, scale_factor=test_scales[scale_idx], mode='bilinear', align_corners=True) else: test_size = (int(h * test_scales[scale_idx]), int(w * test_scales[scale_idx])) interp_tmp = nn.Upsample(size=test_size, mode='bilinear', align_corners=True) image = interp_tmp(img) if args.model == 'DeeplabRes': output2 = model(image.to(device)) coutput = interp(output2).cpu().data[0].numpy() if args.test_flipping: output2 = model( torch.from_numpy( image.numpy()[:, :, :, ::-1].copy()).to(device)) coutput = 0.5 * ( coutput + interp(output2).cpu().data[0].numpy()[:, :, ::-1]) if scale_idx == 0: output = coutput.copy() else: output += coutput output = output / num_scales output = output.transpose(1, 2, 0) output = np.asarray(np.argmax(output, axis=2), dtype=np.uint8) pred_label = output.copy() label = label_2_id[np.asarray(label.numpy(), dtype=np.uint8)] scorer.update(pred_label.flatten(), label.flatten(), index) output_col = colorize_mask(output) output = Image.fromarray(output) name = name[0].split('/')[-1] output.save('%s/%s' % (args.save, name)) output_col.save('%s/%s_color.png' % (args.save, name.split('.')[0]))
json_path), "No json configuration file found at {}".format(json_path) params = util.Params(json_path) # use GPU if available params.cuda = torch.cuda.is_available() if params.ngpu > 0 and params.cuda: params.device = torch.device('cuda') else: params.device = torch.device('cpu') print(params.device) # Set the random seed for reproducible experiments torch.manual_seed(42) if params.cuda: torch.cuda.manual_seed(42) # Set the logger util.set_logger(os.path.join(args.model_dir, 'train.log')) # Create the input data pipeline logging.info("Loading the datasets...") # fetch dataloaders train_dl = data_loader.fetch_dataloader(args.data_dir, 'train', params, shuffle=True) val_dl = data_loader.fetch_dataloader(args.data_dir, 'valid', params, shuffle=False) logging.info("- done.")
def __init__(self, dir): """ Initialize the disease protein prediction experiment Args: dir (string) The directory where the experiment should be run """ super(DPPExperiment, self).__init__(dir) # Set the logger set_logger(os.path.join(args.dir, 'experiment.log'), level=logging.INFO, console=True) # Log Title logging.info("Disease Protein Prediction in the PPI Network") logging.info("Sabri Eyuboglu -- SNAP Group") logging.info("======================================") # Load data from params file logging.info("Loading PPI Network...") self.ppi_networkx, self.ppi_network_adj, self.protein_to_node = load_network(self.params.ppi_network) self.node_to_protein = {node: protein for protein, node in self.protein_to_node.items()} logging.info("Loading Disease Associations...") self.diseases_dict = load_diseases(self.params.diseases_path, self.params.disease_subset, ['none']) # Load method specific data # TODO: Build class for each method if(self.params.method == "ppi_matrix"): logging.info("Loading PPI Matrix...") self.ppi_matrix = np.load(self.params.ppi_matrix) # normalize columns of ppi_matrix if(self.params.normalize): if hasattr(self.params, "norm_type"): if self.params.norm_type == "frac": self.ppi_matrix = self.ppi_matrix / np.sum(self.ppi_matrix, axis=0) elif self.params.norm_type == "zscore": self.ppi_matrix = (self.ppi_matrix - np.mean(self.ppi_matrix, axis=0)) / np.std(self.ppi_matrix, axis=0) else: self.ppi_matrix = (self.ppi_matrix - np.mean(self.ppi_matrix, axis=0)) / np.std(self.ppi_matrix, axis=0) # zero out the diagonal np.fill_diagonal(self.ppi_matrix, 0) elif (self.params.method == 'lr'): logging.info("Loading Feature Matrices...") self.feature_matrices = [] for features_filename in self.params.features: self.feature_matrices.append( build_embedding_feature_matrix(self.protein_to_node, features_filename)) elif (self.params.method == 'l2_rw'): self.method = L2RandomWalk(self.params) elif (self.params.method == 'pathway_expansion'): self.method = PathwayExpansion(self.params, self.ppi_networkx, self.ppi_network_adj) elif (self.params.method == "learned_cn"): self.method = LearnedCN(self.dir, self.params, self.ppi_network_adj, self.diseases_dict, self.protein_to_node) elif (self.params.method == 'gcn'): self.method = GCN(self.params, self.ppi_network_adj)
from db.factory import BasicFactory from elasticsearch import Elasticsearch, helpers, exceptions from util import get_params, set_logger import json bf = BasicFactory() config = get_params() logger = set_logger('elastic') es = Elasticsearch(hosts=[{ 'host': config['elastic']['host'], 'port': config['elastic']['port'] }]) class Elastic: INDEX = config['elastic']['index'] def search(self, name): """ Autocomplete for movies :param str name: prefix from search :return: movies :rtype: dict """ message = {'status': False} query = { 'suggest': { 'movie': { 'prefix': name,
import store import prep # Twitter consumer_key = os.environ['CONSUMER_KEY'] consumer_secret = os.environ['CONSUMER_SECRET'] access_token = os.environ['ACCESS_TOKEN'] access_token_secret = os.environ['ACCESS_TOKEN_SECRET'] # Set location # Bounding boxes for geolocations # Online-Tool to create boxes (c+p as raw CSV): http://boundingbox.klokantech.com/ GEOBOX_NETHERLANDS = [3.0761845666, 51.0227615064, 7.288878522, 53.9033167283] # Set log util.set_logger(True) def parse_args(): parser = argparse.ArgumentParser(description='BI Specialist Assignment', formatter_class=RawTextHelpFormatter) parser.add_argument('-t', '--track', type=str, help='python main.py --track word-to-track') return parser.parse_args() if __name__ == '__main__': args = parse_args()
config = util.load_module(cnf).config if weights_from is None: weights_from = config.weights_file else: weights_from = str(weights_from) files = data.get_image_files(config.get('train_dir')) names = data.get_names(files) labels = data.get_labels(names).astype(np.float32) net = create_net(config) try: net.load_params_from(weights_from) print("loaded weights from {}".format(weights_from)) except IOError: print("couldn't load weights starting from scratch") print("Shape of files: " + str(files.shape)) print("Shape of labels: " + str(labels.shape)) start = time.time() print("fitting ...") net.fit(files, labels) end = time.time() print("Time elapsed for fitting: " + str(end - start)) if __name__ == '__main__': set_logger('train.log') main()
from db import connect from db.factory import TableFactory from util import get_params, set_logger import numpy as np import pandas as pd import re config = get_params() tf = TableFactory() logger = set_logger('imdb data') path_config = { 'basics': f"{config['crawler']['path']}/title.basics.tsv.gz", 'episodes': f"{config['crawler']['path']}/title.episode.tsv.gz", 'ratings': f"{config['crawler']['path']}/title.ratings.tsv.gz" } mapping = { 'tconst': 'title_id', 'runtimeMinutes': 'runtime', 'parentTconst': 'parent_id' } title_types = { 'movie': 1, 'tvSeries': 2, 'tvMiniSeries': 3, 'tvMovie': 4, 'short': 5, 'tvShort': 6, 'tvSpecial': 7
Script to confirm model correctness by observing overfitting to small data For Hydrogen; %load_ext autoreload %autoreload 2 """ from pathlib import Path import torch import torch.optim as optim import matplotlib.pyplot as plt from data_processor import ConllDataSet from mst_lstm import BiLSTM_Parser, margin_based_loss from util import set_logger # Set logger logger = set_logger(__name__) # Read data train_path = Path("data", "en-universal-train.conll") train_data = ConllDataSet(train_path) # Init model def init_model(): model = BiLSTM_Parser(vocab_size=train_data.vocab_size, pos_size=train_data.pos_size, word_embed_dim=100, pos_embed_dim=25, lstm_hidden_dim=250, mlp_hidden_dim=100, num_layers=2)
json_path = os.path.join(args.model_dir, 'params.json') assert os.path.isfile( json_path), "No json configuration file found at {}".format(json_path) params = util.Params(json_path) # use GPU if available params.cuda = torch.cuda.is_available() if params.ngpu > 0 and params.cuda: params.device = torch.device('cuda') else: params.device = torch.device('cpu') # Set the random seed for reproducible experiments torch.manual_seed(42) if params.cuda: torch.cuda.manual_seed(42) # Get the logger util.set_logger(os.path.join(args.model_dir, 'evaluate.log')) # Create the input data pipeline logging.info("Creating the dataset...") # fetch dataloaders test_dl = data_loader.fetch_dataloader('data', 'test', params, shuffle=False) logging.info("- done.") # Define the model g = began.BeganGenerator(params).to(params.device) d = began.BeganDiscriminator(params).to(params.device)
def main(): setup_seed(666) device = torch.device("cuda") save_path = args.save save_pseudo_label_path = osp.join( save_path, 'pseudo_label') # in 'save_path'. Save labelIDs, not trainIDs. save_stats_path = osp.join(save_path, 'stats') # in 'save_path' save_lst_path = osp.join(save_path, 'list') if not os.path.exists(save_path): os.makedirs(save_path) if not os.path.exists(save_pseudo_label_path): os.makedirs(save_pseudo_label_path) if not os.path.exists(save_stats_path): os.makedirs(save_stats_path) if not os.path.exists(save_lst_path): os.makedirs(save_lst_path) cudnn.enabled = True cudnn.benchmark = True logger = util.set_logger(args.save, args.log_file, args.debug) logger.info('start with arguments %s', args) model = DeeplabMulti(num_classes=args.num_classes) saved_state_dict = torch.load(args.restore_from) model.load_state_dict(saved_state_dict) model.train() model.to(device) # init D num_class_list = [2048, 19] model_D = nn.ModuleList([ FCDiscriminator(num_classes=num_class_list[i]).train().to(device) if i < 1 else OutspaceDiscriminator( num_classes=num_class_list[i]).train().to(device) for i in range(2) ]) saved_state_dict_D = torch.load(args.restore_from_D) model_D.load_state_dict(saved_state_dict_D) if not os.path.exists(args.snapshot_dir): os.makedirs(args.snapshot_dir) image_src_list, _, src_num = parse_split_list(args.data_src_list) image_tgt_list, image_name_tgt_list, tgt_num = parse_split_list( args.data_tgt_train_list) # portions tgt_portion = args.init_tgt_port # training crop size w, h = map(int, args.input_size.split(',')) input_size = (w, h) w, h = map(int, args.input_size_target.split(',')) input_size_target = (w, h) bce_loss1 = torch.nn.MSELoss() bce_loss2 = torch.nn.MSELoss(reduce=False, reduction='none') seg_loss = torch.nn.CrossEntropyLoss(ignore_index=255) round_idx = 3 save_round_eval_path = osp.join(args.save, str(round_idx)) save_pseudo_label_color_path = osp.join(save_round_eval_path, 'pseudo_label_color') if not os.path.exists(save_round_eval_path): os.makedirs(save_round_eval_path) if not os.path.exists(save_pseudo_label_color_path): os.makedirs(save_pseudo_label_color_path) ########## pseudo-label generation # evaluation & save confidence vectors test(model, model_D, device, save_round_eval_path, round_idx, 500, args, logger) conf_dict, pred_cls_num, save_prob_path, save_pred_path = val( model, model_D, device, save_round_eval_path, round_idx, tgt_num, args, logger) # class-balanced thresholds cls_thresh = kc_parameters(conf_dict, pred_cls_num, tgt_portion, round_idx, save_stats_path, args, logger) # pseudo-label maps generation label_selection(cls_thresh, tgt_num, image_name_tgt_list, round_idx, save_prob_path, save_pred_path, save_pseudo_label_path, save_pseudo_label_color_path, save_round_eval_path, args, logger) src_train_lst, tgt_train_lst, src_num_sel = savelst_SrcTgt( image_tgt_list, image_name_tgt_list, image_src_list, save_lst_path, save_pseudo_label_path, src_num, tgt_num, args) ########### model retraining # dataset srctrainset = SrcSTDataSet(args.data_src_dir, src_train_lst, max_iters=args.num_steps * args.batch_size, crop_size=input_size, scale=False, mirror=False, mean=IMG_MEAN) tgttrainset = TgtSTDataSet(args.data_tgt_dir, tgt_train_lst, pseudo_root=save_pseudo_label_path, max_iters=args.num_steps * args.batch_size, crop_size=input_size_target, scale=False, mirror=False, mean=IMG_MEAN, set='train') trainloader = torch.utils.data.DataLoader(srctrainset, batch_size=args.batch_size, shuffle=True, num_workers=8, pin_memory=True) trainloader_iter = enumerate(trainloader) targetloader = torch.utils.data.DataLoader(tgttrainset, batch_size=args.batch_size, shuffle=True, num_workers=8, pin_memory=True) targetloader_iter = enumerate(targetloader) optimizer = optim.SGD(model.parameters(), lr=args.learning_rate, momentum=args.momentum, weight_decay=args.weight_decay) optimizer.zero_grad() optimizer_D = optim.Adam(model_D.parameters(), lr=args.learning_rate_D, betas=(0.9, 0.99)) optimizer_D.zero_grad() logger.info( '###### Start model retraining dataset in round {}! ######'.format( round_idx)) start = timeit.default_timer() # start training interp = nn.Upsample(size=(input_size[1], input_size[0]), mode='bilinear', align_corners=True) interp_target = nn.Upsample(size=(input_size_target[1], input_size_target[0]), mode='bilinear', align_corners=True) # labels for adversarial training source_label = 0 target_label = 1 for i_iter in range(args.num_steps): lamb = 1 optimizer.zero_grad() adjust_learning_rate(optimizer, i_iter) optimizer_D.zero_grad() adjust_learning_rate_D(optimizer_D, i_iter) # train G # don't accumulate grads in D for param in model_D.parameters(): param.requires_grad = False # train with source _, batch = trainloader_iter.__next__() images, labels, _, _ = batch images = images.to(device) labels = labels.long().to(device) feat_source, pred_source = model(images, model_D, 'source') pred_source = interp(pred_source) loss_seg = seg_loss(pred_source, labels) loss_seg.backward() # train with target _, batch = targetloader_iter.__next__() images, labels, _, _ = batch images = images.to(device) labels = labels.long().to(device) feat_target, pred_target = model(images, model_D, 'target') pred_target = interp_target(pred_target) # atten_target = F.interpolate(atten_target, size=(16, 32), mode='bilinear', align_corners=True) loss_seg_tgt = seg_loss(pred_target, labels) * lamb D_out1 = model_D[0](feat_target) loss_adv1 = bce_loss1( D_out1, torch.FloatTensor( D_out1.data.size()).fill_(source_label).to(device)) D_out2 = model_D[1](F.softmax(pred_target, dim=1)) loss_adv2 = bce_loss2( D_out2, torch.FloatTensor( D_out2.data.size()).fill_(source_label).to(device)) loss_adv = loss_adv1 * 0.01 + loss_adv2.mean() * 0.01 loss = loss_seg_tgt + loss_adv loss.backward() optimizer.step() # train D # bring back requires_grad for param in model_D.parameters(): param.requires_grad = True # train with source D_out_source1 = model_D[0](feat_source.detach()) loss_D_source1 = bce_loss1( D_out_source1, torch.FloatTensor( D_out_source1.data.size()).fill_(source_label).to(device)) D_out_source2 = model_D[1](F.softmax(pred_source.detach(), dim=1)) loss_D_source2 = bce_loss1( D_out_source2, torch.FloatTensor( D_out_source2.data.size()).fill_(source_label).to(device)) loss_D_source = loss_D_source1 + loss_D_source2 loss_D_source.backward() # train with target D_out_target1 = model_D[0](feat_target.detach()) loss_D_target1 = bce_loss1( D_out_target1, torch.FloatTensor( D_out_target1.data.size()).fill_(target_label).to(device)) D_out_target2 = model_D[1](F.softmax(pred_target.detach(), dim=1)) weight_target = bce_loss2( D_out_target2, torch.FloatTensor( D_out_target2.data.size()).fill_(target_label).to(device)) loss_D_target2 = weight_target.mean() loss_D_target = loss_D_target1 + loss_D_target2 loss_D_target.backward() optimizer_D.step() if i_iter % 10 == 0: print( 'iter={0:8d}/{1:8d}, seg={2:.3f} seg_tgt={3:.3f} adv={4:.3f} adv1={5:.3f} adv2={6:.3f} src1={7:.3f} src2={8:.3f} tgt1={9:.3f} tgt2={10:.3f} D1={11:.3f} D2={12:.3f}' .format(i_iter, args.num_steps, loss_seg.item(), loss_seg_tgt.item(), loss_adv.item(), loss_adv1.item(), loss_adv2.mean().item(), loss_D_source1.item(), loss_D_source2.item(), loss_D_target1.item(), loss_D_target2.item(), loss_D_source.item(), loss_D_target.item())) if i_iter >= args.num_steps_stop - 1: print('save model ...') torch.save( model.state_dict(), osp.join(args.snapshot_dir, 'GTA5_' + str(args.num_steps_stop) + '.pth')) torch.save( model_D.state_dict(), osp.join(args.snapshot_dir, 'GTA5_' + str(args.num_steps_stop) + '_D.pth')) break if i_iter % args.save_pred_every == 0 and i_iter != 0: print('taking snapshot ...') test(model, model_D, device, save_round_eval_path, round_idx, 500, args, logger) torch.save( model.state_dict(), osp.join(args.snapshot_dir, 'GTA5_' + str(i_iter) + '.pth')) torch.save( model_D.state_dict(), osp.join(args.snapshot_dir, 'GTA5_' + str(i_iter) + '_D.pth')) end = timeit.default_timer() logger.info( '###### Finish model retraining dataset in round {}! Time cost: {:.2f} seconds. ######' .format(round_idx, end - start)) # test self-trained model in target domain test set test(model, model_D, device, save_round_eval_path, round_idx, 500, args, logger)
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/4.3.1/css/bootstrap.min.css"> </head> <body> <div class="container"> <h1>Upload CSV for NLPQL Form Parsing</h1> <br> <div> <a href="https://github.com/ClarityNLP/Utilities/blob/master/custom_query/afib.csv" target="_blank"> (See sample CSV) </a> </div> <form method=post enctype=multipart/form-data> <br> <input type=text name="formname" class="form-control" placeholder="Form Name"> <br> <input type=file name=file class="form-control-file"> <br> <input type=submit value=Upload class="btn btn-primary"> </form> </div> </body> ''' if __name__ == '__main__': util.app_token() application.run(host='0.0.0.0', port=5000, debug=True) util.set_logger(application.logger)