def generate(args): devices = utils.get_devices(args.gpu) if args.seed is not None: utils.manual_seed(args.seed) logging.info("Loading data...") vocab = utils.load_pkl(args.vocab) logging.info("Initializing generation environment...") model = prepare_model(args, vocab) model = utils.to_device(model, devices) generator = Generator( model=model, device=devices[0], batch_size=args.batch_size, vocab=vocab, bos=args.bos, eos=args.eos, unk=args.unk, max_len=args.max_length ) logging.info("Commencing generation...") samples = generator.generate(args.z_samples) if args.nearest_neighbors is not None: dataset = prepare_dataset(args, vocab) neighbors = nearest_neighbors(args, samples, dataset) else: neighbors = None save(args, samples, neighbors) logging.info("Done!")
def get_summary(self): res = [] path_dir = self.mars_config.get_base_path() snap_times_list = os.listdir(path_dir) snap_times_list.sort(reverse=True) for snap_time in snap_times_list: try: time_stamp = int(snap_time) time_str = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time_stamp)) cur_summary = { 'time': time_str, GROUPS_NAME: len(get_group(self.mars_config, snap_time)), DEVICE_NAME: len(get_devices(self.mars_config, snap_time)), DEVICE_CONFIG_NAME: len(get_devices_configs(self.mars_config, snap_time)), HOSTS_NAME: len(get_host(self.mars_config, snap_time)), LINKS_NAME: len(get_link(self.mars_config, snap_time)), FLOW_NAME: len(get_flow(self.mars_config, snap_time)), } res.append(cur_summary) except ValueError as e: pass return res
def main(): # Setup argument parsing parser = argparse.ArgumentParser(description='Monitor bluetooth igrill devices, and export to MQTT') parser.add_argument('-c', '--config', action='store', dest='config_directory', default='.', help='Set config directory, default: \'.\'') parser.add_argument('-l', '--log-level', action='store', dest='log_level', default='INFO', help='Set log level, default: \'info\'') parser.add_argument('-d', '--log-destination', action='store', dest='log_destination', default='', help='Set log destination (file), default: \'\' (stdout)') options = parser.parse_args() config = read_config(options.config_directory) # Setup logging log_setup(options.log_level, options.log_destination) # Get device list devices = get_devices(config['devices']) # Connect to MQTT client = mqtt_init(config['mqtt']) base_topic = config['mqtt']['base_topic'] polling_interval = config['interval'] if 'interval' in config else 15 while True: for device in devices: publish(device.read_temperature(), device.read_battery(), client, base_topic, device.name) time.sleep(polling_interval)
def train(args): devices = utils.get_devices(args.gpu) if args.seed is not None: utils.manual_seed(args.seed) logging.info("Loading data...") dataloader = create_dataloader(args) vocab = dataloader.dataset.vocab utils.save_pkl(vocab, os.path.join(args.save_dir, "vocab.pkl")) logging.info("Initializing training environment...") mdl = prepare_model(args, dataloader) optimizer_cls = get_optimizer_cls(args) trainer = Trainer(model=utils.to_device(mdl, devices), device=devices[0], vocab=vocab, epochs=args.epochs, save_dir=args.save_dir, save_period=args.save_period, optimizer_cls=optimizer_cls, tensor_key="tensor", samples=args.samples, show_progress=args.show_progress, kld_annealing=args.kld_annealing, dynamic_rnn=mdl.encoder.rnn.dynamic or mdl.decoder.rnn.dynamic) report_model(trainer) logging.info("Commecing training...") trainer.train(dataloader) logging.info("Done!")
def initialize(param_runDir, cuda_id): runtime_path = Path(param_runDir).resolve() logger.handlers.clear() # set logger log_file = f"demo_log.log" logger.set_log_to_stream() logger.set_log_to_file(runtime_path.joinpath(log_file)) # print versions after logger.set_log_to_file() to log them into file print_versions() #logger.info(f"runtime commit: {get_commit()}") logger.info(f"runtime path: {runtime_path}") random_seed = 20 #for Pytorch 1.2.0 + DenseNet121 #basic logger.info(f"random seed for reproducible: {random_seed}") torch.manual_seed(random_seed) np.random.seed(random_seed) torch.backends.cudnn.deterministic = True torch.backends.cudnn.benchmark = False # check gpu device device = get_devices(cuda_id) return runtime_path, device
def check_distributed(args): devices = get_devices(args.cuda) if args.local_rank is None: return False, devices[0] else: device = devices[args.local_rank % len(devices)] torch.cuda.set_device(device) logger.info(f"waiting other ranks ...") dist.init_process_group(backend="nccl", init_method="env://") logger.info(f"set world_size to {dist.get_world_size()}") return True, device
def generate_commands(self, rebalance=True, meta=None): commands = [] ring_disks = get_devices(self.zones, metadata=meta) for ringtype in RING_TYPES: builder_present = os.path.exists("%s/%s.builder" % (self.workspace, ringtype)) if not builder_present: commands.append(self._ring_create_command(ringtype)) for zone, devices in ring_disks[ringtype].iteritems(): for device in devices: port = self.ports[ringtype] weight = device['weight'] disk = device['device'] node = device['ip'] metadata = device['metadata'] # When rings are not present or if device does not # exist in ring, add it to the ring # Else if the weight is to be set to 0 remove # the device eor just update the weight if not builder_present or \ not self._is_devpresent(ringtype, zone, node, port, disk): cmd = self._ring_add_command(ringtype, zone, node, port, disk, metadata, weight) else: if int(weight) == 0: cmd = self._ring_remove_command(ringtype, zone, node, port, disk) else: # Always set the weight of device # Verified that setting weight (to same) # value doesnt cause partitions to reassign cmd = self._ring_setweight_command(ringtype, zone, node, port, disk, weight) commands.append(cmd) if rebalance: commands.append(self._ring_rebalance_command(ringtype)) return commands
def main(): apk = sys.argv[1] package = sys.argv[2] instructions_file = sys.argv[3] print(instructions_file) print('Checking adb devices') devices = get_devices() print('Devices found: '+str(devices)) if not len(devices): print('No devices found, setting up emulator') run_emulator() print('Emulator boot completed.. proceding..') device = MonkeyRunner.waitForConnection() easy_device = EasyMonkeyDevice(device) print('Connected\nInstalling package..') device.installPackage(apk) print('Installed!') print('Checking all activities..\nThis may take a while..') f = open(instructions_file, 'r') instructions = json.load(f) for activity in instructions: print(activity) runComponent = package + '/' + activity for button in instructions[activity]: device.startActivity(component=runComponent) time.sleep(1) if easy_device.visible(By.id('id/'+button)): easy_device.touch(By.id('id/'+button), MonkeyDevice.DOWN_AND_UP) time.sleep(1) else: device.press("KEYCODE_BACK", MonkeyDevice.DOWN_AND_UP) time.sleep(1) if easy_device.visible(By.id('id/'+button)): easy_device.touch(By.id('id/'+button), MonkeyDevice.DOWN_AND_UP) result = device.takeSnapshot() result_path = os.path.join(os.path.abspath('monkeyresult/'), package) if not os.path.exists(result_path): os.makedirs(result_path) result.writeToFile( os.path.join(result_path, activity+'.png'), 'png' ) f.close() print('Saved some snapshots to\n'+result_path)
def generate_commands(self, rebalance=True, meta=None): commands = [] ring_disks = get_devices(self.zones, metadata=meta) for ringtype in RING_TYPES: builder_present = os.path.exists("%s/%s.builder" % (self.workspace, ringtype)) if not builder_present: commands.append(self._ring_create_command(ringtype)) for zone, devices in ring_disks[ringtype].iteritems(): for device in devices: port = self.ports[ringtype] weight = device['weight'] disk = device['device'] node = device['ip'] metadata = device['metadata'] # When rings are not present or if device does not # exist in ring, add it to the ring # Else if the weight is to be set to 0 remove # the device eor just update the weight if not builder_present or \ not self._is_devpresent(ringtype, zone, node, port, disk): cmd = self._ring_add_command(ringtype, zone, node, port, disk, metadata, weight) else: if int(weight) == 0: cmd = self._ring_remove_command( ringtype, zone, node, port, disk) else: # Always set the weight of device # Verified that setting weight (to same) # value doesnt cause partitions to reassign cmd = self._ring_setweight_command( ringtype, zone, node, port, disk, weight) commands.append(cmd) if rebalance: commands.append(self._ring_rebalance_command(ringtype)) return commands
def predict(args): devices = utils.get_devices(args.gpu) if args.seed is not None: utils.manual_seed(args.seed) logging.info("Loading data...") vocab_paths = [getattr(args, f"{mode}_vocab") for mode in MODES] vocabs = [utils.load_pkl(v) for v in vocab_paths] test_dataset = prepare_dataset(args, vocabs[0]) test_dataloader = td.DataLoader( dataset=test_dataset, batch_size=args.batch_size, num_workers=args.data_workers, collate_fn=dataset.TextSequenceBatchCollator( pad_idxs=[len(v) for v in vocabs])) logging.info("Initializing generation environment...") model, vocabs[0] = prepare_model(args, vocabs) model = utils.to_device(model, devices) predictor = Predictor( model=model, device=devices[0], batch_size=args.batch_size, sent_vocab=vocabs[0], label_vocab=vocabs[1], intent_vocab=vocabs[2], bos=args.bos, eos=args.eos, unk=args.unk, beam_size=args.beam_size, ) logging.info("Commencing prediction...") with torch.no_grad(): (labels, intents), (pl, pi) = predictor.predict(test_dataloader) report_stats(args, labels, intents, pl, pi) save(args, labels, intents, pl, pi) logging.info("Done!")
def generate(args): devices = utils.get_devices(args.gpu) if args.seed is not None: utils.manual_seed(args.seed) logging.info("Loading data...") vocab_paths = [getattr(args, f"{mode}_vocab") for mode in MODES] vocabs = [utils.load_pkl(v) for v in vocab_paths] test_dataset = prepare_dataset(args, vocabs[0]) test_dataloader = td.DataLoader( dataset=test_dataset, batch_size=args.batch_size, num_workers=args.data_workers, collate_fn=dataset.TextSequenceBatchCollator( pad_idxs=[len(v) for v in vocabs])) logging.info("Initializing generation environment...") model, vocabs[0] = prepare_model(args, vocabs) model.beam_size = args.beam_size model = utils.to_device(model, devices) predictor = PredictorWithProgress(model=model, device=devices[0], vocabs=vocabs, progress=args.show_progress, bos=args.bos, eos=args.eos, unk=args.unk, topk=args.top_k) logging.info("Commencing prediction...") with torch.no_grad(): (labels, pl), (intents, pi) = predictor.predict(test_dataloader) labels, intents = [l[0] for l in labels], [i[0] for i in intents] pl, pi = [p[0] for p in pl], [p[0] for p in pi] report_stats(args, labels, intents, pl, pi) save(args, labels, intents, pl, pi) logging.info("Done!")
def __init__(self, run_callback): super().__init__() self.left = 200 self.top = 200 self.width = 400 self.height = 120 self.run_callback = run_callback self.custom_state_rows = [] self.input_dropdown = None self.output_dropdown = None self.playback_dropdown = None self.devices = get_devices() self.layout = QFormLayout() self.selected_devices = { "input_device_index": None, "output_device_index": None, "playback_device_index": None } self.init_ui()
def __init__(self, config, train_loader, valid_loader, test_loader): """ Construct a new Trainer instance. Args ---- - config: object containing command line arguments. - data_loader: data iterator """ # DATA PARAMS self.train_loader = train_loader self.valid_loader = valid_loader self.test_loader = test_loader self.test_script = config.test_script if self.test_script: try: if config.train: self.train_loader = DataLoader( self.train_loader.sampler.data_source. data[:250 * config.batch_size], batch_size=config.batch_size) self.valid_loader = DataLoader( self.valid_loader.sampler.data_source. data[:3 * config.batch_size], batch_size=config.batch_size) except AttributeError: if config.train: self.train_loader = DataLoader( self.train_loader.sampler.data_source.dataset. data[:10 * config.batch_size], batch_size=config.batch_size) self.valid_loader = DataLoader( self.valid_loader.sampler.data_source.dataset. data[:3 * config.batch_size], batch_size=config.batch_size) self.test_loader = self.valid_loader self.num_train = len(self.train_loader.dataset) if train_loader else 0 self.num_valid = len(self.valid_loader.dataset) if valid_loader else 0 self.num_test = len(self.test_loader.dataset) if test_loader else 0 self.num_classes = config.num_classes _batch = next(iter(self.test_loader)) self.input_size = utils.infer_input_size(_batch) # TRAINING PARAMS self.epochs = config.epochs self.start_epoch = 0 self.momentum = config.momentum self.lr = config.init_lr self.weight_decay = config.weight_decay self.nesterov = config.nesterov self.gamma = config.gamma self.lr_step = config.lr_step self.lambda_a = config.lambda_a self.lambda_b = config.lambda_b self.temperature = config.temperature self.scale_dml = config.scale_dml # MISCELLANEOUS PARAMS self.model_num = len(config.model_names) self.counter = 0 self.use_wandb = config.use_wandb self.use_sweep = config.use_sweep self.progress_bar = config.progress_bar self.experiment_level = config.experiment_level self.experiment_name = config.experiment_name self.level_name = config.level_name self.unlabelled = bool(config.unlabel_split) self.discard_unlabelled = config.discard_unlabelled # MODELS AND MODEL ATTRIBUTES self.model_names = config.model_names self.indexed_model_names = [ f'({i})_{model_name}' for i, model_name in enumerate(self.model_names, 1) ] self.use_gpu = (not config.disable_cuda and torch.cuda.is_available()) self.devices = utils.get_devices(self.model_num, self.use_gpu) self.nets = utils.model_init_and_placement(self.model_names, self.devices, self.input_size, self.num_classes) # LOSS FUNCTIONS self.loss_kl = nn.KLDivLoss(reduction='batchmean') self.loss_ce = nn.CrossEntropyLoss(reduction='none') # KEEP TRACK OF THE VALIDATION ACCURACY self.best_valid_accs = [-0.1] * self.model_num self.best_mean_valid_acc = -0.1 # LEARNING SIGNAL CONDITIONNS ADN FRACTIONS # if lambda b = 1. (which it always should be for the first level) # the kd part is disabled and should therefore not be logged self.kd_condition = all( [bool(1 - self.lambda_b), self.experiment_level > 1]) self.kd_fraction = (1 - self.lambda_b) * self.lambda_a # deep mutual learning can only be done on a set of models # the dml is therefore disabled for model_num <= 1 (for a kd experiment for instance) self.dml_condition = any( [self.model_num >= 2, bool(self.lambda_b), bool(self.lambda_a)]) self.dml_fraction = self.lambda_a * self.lambda_b # if both previous conditions are False, the sl_signal is equal to the overall loss, # therefore we do not need the sl_signal explicitely self.sl_condition = any([self.dml_condition, self.kd_condition]) self.sl_fraction = 1 - self.lambda_a # CONFIGURE WEIGHTS & BIASES LOGGING AND SAVE DIR self.experiment_dir = utils.prepare_dirs(self.experiment_name, self.level_name) if self.use_wandb: wandb.init(name=self.level_name, project=self.experiment_name, dir=self.experiment_dir, config=config, id=self.level_name, tags=list(set(config.model_names))) wandb.log({ 'sl fraction': self.sl_fraction, 'kd fraction': self.kd_fraction, 'dml fraction': self.dml_fraction }) if self.use_sweep: _config = wandb.config for param in config.hp_search: print(param, getattr(_config, param)) setattr(self, param, getattr(_config, param)) # INITIALIZE OPTIMIZER & SCHEDULER AND LOG THE MODEL DESCRIPTIONS model_stats = [] self.optimizers = [] self.schedulers = [] for i, net in enumerate(self.nets): optimizer = torch.optim.SGD(net.parameters(), lr=self.lr, momentum=self.momentum, weight_decay=self.weight_decay, nesterov=self.nesterov) self.optimizers.append(optimizer) # set learning rate decay scheduler = torch.optim.lr_scheduler.StepLR(self.optimizers[i], step_size=self.lr_step, gamma=self.gamma, last_epoch=-1) self.schedulers.append(scheduler) model_name = self.model_names[i] architecture, size_indicator = utils.infer_model_desc(model_name) params = sum([p.data.nelement() for p in net.parameters()]) print('[*] Number of parameters of {} model: {:,}'.format( model_name, params)) model_stats.append([ model_name, architecture, size_indicator, f'{params:,}'.replace(',', '.') ]) if self.use_wandb: wandb.log({ "examples": wandb.Table(data=model_stats, columns=[ "Model name", "Architecture", "Size indicator", "# params" ]) })
def __init__(self, dataset, num_classes, batch_size, train, model_num, teachers=[], unlabel_split=.0, use_gpu=False, progress_bar=True): self.data = [] self.teacher_num = len(teachers) devices = get_devices(model_num, use_gpu) if unlabel_split: len_ds = len(dataset) all_indices = range(len_ds) subset_indices = set(random.sample( all_indices, int(len_ds * unlabel_split))) counter = 0 if progress_bar: pbar = tqdm(total=len(dataset), smoothing=.005) if teachers: accuracies = [] # prepare and assert the teachers behaviour _batch = dataset[0][0].unsqueeze(0) for teacher, device in zip(teachers, devices): teacher = teacher.to(device) batch = _batch.clone().to(device) teacher.eval() sample_classes = teacher(batch).shape[1] assert sample_classes == num_classes, f"Num classes of the output is {sample_classes}, {num_classes} required" accuracies.append(RunningAverageMeter()) # create the psuedo labels with torch.no_grad(): for image, label in dataset: unlabelled = 1 if unlabel_split: if counter in subset_indices: unlabelled = 0 counter += 1 _psuedo_labels = [] for i, (teacher, device) in enumerate(zip(teachers, devices)): if use_gpu: image = image.to(device) # add dimension to comply with the desired input dimension (batch of single image) pred = teacher(image.unsqueeze(0)).cpu() _psuedo_labels.append(pred) # keep track of the accuracy of the teacher model acc_at_1 = accuracy( pred, torch.tensor([[label]]), topk=(1,))[0] accuracies[i].update(acc_at_1.item()) image = image.cpu() psuedo_labels = torch.stack(_psuedo_labels, -1).squeeze(0) self.data.append((image, label, psuedo_labels, unlabelled)) if progress_bar: pbar.update(1) if progress_bar: pbar.close() print( f"Accurcies of loaded models are {' ,'.join([str(round(acc.avg, 2))+'%' for acc in accuracies])}, respectively") else: dummy_psuedo_label = torch.empty(num_classes, model_num) for image, label in dataset: unlabelled = 1 if unlabel_split: if counter in subset_indices: unlabelled = 0 counter += 1 self.data.append( (image, label, dummy_psuedo_label, unlabelled) ) if progress_bar: pbar.update(1) if progress_bar: pbar.close()
def generate(args): devices = utils.get_devices(args.gpu) if args.seed is not None: utils.manual_seed(args.seed) logging.info("Loading data...") vocab_paths = [args.word_vocab, args.label_vocab, args.intent_vocab] vocabs = [utils.load_pkl(v) for v in vocab_paths] dataloader = None logging.info("Initializing generation environment...") model, vocabs[0] = prepare_model(args, vocabs) model = utils.to_device(model, devices) encoder = encode.Encoder(model=model, device=devices[0], batch_size=args.batch_size) generator = Generator(model=model, device=devices[0], batch_size=args.batch_size, sent_vocab=vocabs[0], label_vocab=vocabs[1], intent_vocab=vocabs[2], bos=args.bos, eos=args.eos, unk=args.unk, max_len=args.max_length, beam_size=args.beam_size, beam_topk=args.beam_sample_topk, validate=args.validate) logging.info("Commencing generation...") if args.generation_type in {"posterior", "uniform"}: if dataloader is None: dataloader = create_dataloader(args, vocabs) sampler = utils.map_val(args.generation_type, { "gaussian": lambda: None, "posterior": lambda: MultivariateGaussianMixtureSampler( *encoder.encode(dataloader), scale=args.posterior_sampling_scale), "uniform": lambda: UniformNoiseSampler(encoder.encode(dataloader)[0], pa=args.uniform_sampling_pa, pm=args.uniform_sampling_pm) }, name="sampler")() with torch.no_grad(): gens, probs = generator.generate(args.samples, sampler) if args.nearest_neighbors is not None: if dataloader is None: dataloader = create_dataloader(args, vocabs) sents = [data["string"][0] for data in dataloader.dataset] searcher = neighbor.PyTorchPCASearcher( pca_dim=100, sents=sents, num_neighbors=args.nearest_neighbors, batch_size=args.nearest_neighbors_batch_size, device=devices[0]) neighbors = searcher.search(gens[0]) else: neighbors = None report_stats(args, gens[0], neighbors) save(args, gens, probs, neighbors) logging.info("Done!")
def graph(args): flnA,flnB,flnC = args['inputA'],args['inputB'],args['inputC'] typeA,wA= args['pA'][0],float(args['pA'][1]) typeB,wB= args['pB'][0],float(args['pB'][1]) if flnC is not None: typeC,wC= args['pC'][0],args['pC'][1] inputA, sr = librosa.load(args['audio_path']+flnA, sr=args['sr'], mono=True) inputB, sr = librosa.load(args['audio_path']+flnB, sr=args['sr'], mono=True) if flnC is not None: inputC, sr = librosa.load(args['audio_path']+flnC, sr=args['sr'], mono=True) else: inputC = None # segment beats proc = Segmentation(args,inputA,flnA,inputB,flnB,inputC,flnC) inputA, inputB, inputC, xA_seg, xB_seg, xC_seg, x_seg, beat_channels = proc.get_segments() if len(x_seg[0]) // args['hoplen'] < args['k_h']: args['k_h'] = int(round(len(x_seg[0]) // args['hoplen'])) n_channels = args['nfft'] // 2 + 1 kernel_seg = utils.weight_fn( (1, args['k_h'], n_channels, args['n_filters_stft'])) kernel_odf = utils.weight_fn( (1, 1, n_channels, args['n_filters_odf']) ) dft_real_kernels, dft_imag_kernels = utils.get_stft_kernels(args['nfft']) # allocate cpu/gpu for compute config, devices, input_dev, seg_dev, odf_dev, output_dev = utils.get_devices() segcount=0 result,res=[],[] for xA_beat_channel,xB_beat_channel,xC_beat_channel,x_beat_channel in beat_channels[:args['num_beat_segs']]: seglen =min(len(xA_beat_channel),len(xB_beat_channel),len(xC_beat_channel),len(x_beat_channel)) print 'Transforming segment {}/{}'.format(segcount,len(beat_channels)) segcount+=1 xA_beat_channel=xA_beat_channel[:seglen] xB_beat_channel=xB_beat_channel[:seglen] xC_beat_channel=xC_beat_channel[:seglen] x_beat_channel = x_beat_channel[:seglen] # initalize segments g=tf.Graph() with g.as_default(), g.device(input_dev), tf.Session(config=config) as sess: beta_stft = tf.placeholder(tf.float32,shape=(),name='beta_stft') dft_real_kernels_tf = tf.constant(dft_real_kernels, name="dft_real_kernels", dtype='float32') dft_imag_kernels_tf = tf.constant(dft_imag_kernels, name="dft_imag_kernels", dtype='float32') # input A xA_seg_raw = np.ascontiguousarray(xA_beat_channel[None,:,None,None]) xA_seg_raw = tf.constant(xA_seg_raw, name='xA_seg_raw', dtype='float32') _, xA_seg_mag, xA_seg = utils.get_logmagnitude_STFT(xA_seg_raw, dft_real_kernels_tf, dft_imag_kernels_tf, args['hoplen']) # input B xB_seg_raw = np.ascontiguousarray(xB_beat_channel[None,:,None,None]) xB_seg_raw = tf.constant(xB_seg_raw, name='xB_seg_raw', dtype='float32') _, xB_seg_mag, xB_seg = utils.get_logmagnitude_STFT(xB_seg_raw, dft_real_kernels_tf, dft_imag_kernels_tf, args['hoplen']) # input C xC_seg_raw = np.ascontiguousarray(xC_beat_channel[None,:,None,None]) xC_seg_raw = tf.constant(xC_seg_raw, name='xC_seg_raw', dtype='float32') _, xC_seg_mag, xC_seg = utils.get_logmagnitude_STFT(xC_seg_raw, dft_real_kernels_tf, dft_imag_kernels_tf, args['hoplen']) # optimizable variable x x_beat_channel = np.ascontiguousarray(x_beat_channel[None,:,None,None]).astype(np.float32) x_seg_raw = tf.Variable(x_beat_channel,name='x_seg_raw') _, x_seg_mag, x_seg = utils.get_logmagnitude_STFT(x_seg_raw, dft_real_kernels_tf, dft_imag_kernels_tf, args['hoplen']) # compute features with g.device(seg_dev): kernel_seg_tf = tf.constant(kernel_seg, name='kernel', dtype='float32') # optimizable var net conv = tf.nn.conv2d(x_seg,kernel_seg_tf,padding="VALID",strides=[1,1,1,1],name="conv_x_") net = utils.selu(conv) # net A segA_conv = tf.nn.conv2d(xA_seg,kernel_seg_tf,padding="VALID",strides=[1,1,1,1],name="convA_"+typeA) segA_net = utils.selu(segA_conv) if typeA == 'style': gramA, style_gramA = compute_gram(net, segA_net) segA_loss = compute_style_loss(gramA, style_gramA) elif typeA == 'content': segA_loss = compute_content_loss(net, segA_net) # net B seg_B_conv =tf.nn.conv2d(xB_seg,kernel_seg_tf,padding='VALID',strides=[1,1,1,1],name='convB_'+typeB) segB_net = utils.selu(seg_B_conv) if typeB == 'style': gramB, style_gramB = compute_gram(net, segB_net) segB_loss = compute_style_loss(gramB, style_gramB) elif typeB == 'content': segB_loss = compute_content_loss(net, segB_net) # net C if inputC is not None: segC_conv=tf.nn.conv2d(xC_seg,kernel_seg_tf,padding="VALID",strides=[1,1,1,1],name="convC_"+typeC) segC_net = utils.selu(segC_conv) if typeC == 'style': gramC, style_gramC = compute_gram(net, segC_net) segC_loss = compute_style_loss(gramC, style_gramC) elif typeC == 'content': segC_loss = compute_content_loss(net, segC_net) else: segC_loss=None grads_segA = tf.gradients(segA_loss, x_seg_raw)[0] norm_segA = tf.norm(grads_segA) grads_segB = tf.gradients(segB_loss, x_seg_raw)[0] norm_segB = tf.norm(grads_segB) if inputC is not None: grads_segC = tf.gradients(segC_loss, x_seg_raw)[0] norm_segC = tf.norm(grads_segC) if args['mode']=='ODF': # cosine distance loss from spectral difference evelopes with g.device(odf_dev): sdif_odf_loss =tf.ones(()) odf_conv = tf.nn.conv2d(x_seg,kernel_odf,padding="VALID",strides=[1,1,1,1],name="conv_"+'odf') odf_net = utils.selu(odf_conv) x_odf,x_sdif = compute_sdif_odf(odf_net) if args['target_odf_pattern'] == 'A': _odf,_spec =compute_sdif_odf(xA_seg) elif args['target_odf_pattern'] == 'B': _odf,_spec =compute_sdif_odf(xB_seg) elif args['target_odf_pattern'] == 'C': _odf,_spec =compute_sdif_odf(xC_seg) else: _odf,_spec =compute_sdif_odf(xA_seg) sdif_odf_loss = tf.losses.cosine_distance(x_odf,_odf,dim=0) else: sdif_odf_loss =0 # compute outputs with g.device(output_dev): total_inputA_loss=0 total_inputB_loss=0 total_inputC_loss=0 total_odf_loss = sdif_odf_loss total_inputA_loss = wA * segA_loss total_inputB_loss = wB * segB_loss if inputC is not None: total_inputC_loss = float(wC) *segC_loss total_loss = total_inputA_loss + total_inputB_loss + total_inputC_loss + total_odf_loss with tf.Session(config=config) as sess: sess.run(tf.global_variables_initializer()) feed_dict ={beta_stft: 1.0} segA = norm_segA.eval(feed_dict=feed_dict) segB = norm_segB.eval(feed_dict=feed_dict) if inputC is not None: segC = norm_segC.eval(feed_dict=feed_dict) # normalize loss gradient beta_vals =1.0 beta_vals =segA/segB if inputC is not None: beta_vals =abs(segA+segB+segC)/3.0 feed_dict[beta_stft]=beta_vals # train optimizer opt = tf.contrib.opt.ScipyOptimizerInterface(total_loss,var_list=[x_seg_raw],method='L-BFGS-B',options={'maxiter': args['iterations'],'ftol': args['factor']* np.finfo(float).eps,'gtol': args['factor']* np.finfo(float).eps},tol=args['factor']) opt.minimize(sess, feed_dict=feed_dict) res = x_seg_raw.eval() print 'Segment loss:', total_loss.eval(feed_dict=feed_dict) result.append(res.squeeze()) return result
if opt.plot_online: line.set_data(dataQ[:, 0].detach().cpu().numpy(), dataQ[:, 1].detach().cpu().numpy()) plt.pause(0.01) return dataQ, collQ, coll_mmd if __name__ == "__main__": FILENAME = opt.expDirName + '/' + opt.expName + '_' FILENAME += opt.src.split('/')[-1][:3] + '-' + opt.target.split( '/')[-1][:3] device, num_gpus = get_devices( "cuda:0" if not opt.no_cuda and torch.cuda.is_available() else "cpu", seed=opt.seed) # Load data dataQ0 = load_data(opt.src, opt.nPointsMax_src).to(device) dataP, wP = load_data_weights(opt.target, opt.target_weights, opt.nPointsMax_target) dataP = dataP.to(device) if wP is not None: wP = wP.to(device) n_samples, n_features = dataQ0.shape print(opt) # Discriminator
from airqo_batch_insert import get_device_measurements from config import configuration from kcca_batch_insert import ProcessMeasurements from utils import get_devices, filter_valid_devices, filter_valid_kcca_devices if __name__ == "__main__": tenant = configuration.TENANT if not tenant: print("Tenant not specified") exit() if tenant.strip().lower() == "airqo": airqo_devices = get_devices(configuration.AIRQO_BASE_URL, "airqo") filtered_devices = filter_valid_devices(airqo_devices) if len(filtered_devices) > 0: get_device_measurements(filtered_devices) else: print("No valid devices") elif tenant.strip().lower() == "kcca": kcca_devices = get_devices(configuration.AIRQO_BASE_URL, "kcca") filtered_devices = filter_valid_kcca_devices(kcca_devices) if len(filtered_devices) > 0: process_measurements = ProcessMeasurements(filtered_devices) process_measurements.begin_fetch() else: print("No valid devices") else: print("Error", "Invalid Tenant", sep=" : ")
def page3_prepare(self): self.page3_ls.clear() for dev,_type in utils.get_devices(): self.page3_ls.append([False, dev, _type, False, ""])
default=None, type=str, help="use GPUs with its device ids, separated by commas") parser.add_argument('--model', default=None, type=str, help="pretrained model to be used in prediction") parser.add_argument('input_csv', type=str, help="input csv filepath for test") parser.add_argument('output_csv', type=str, help="output csv filepath for test") args = parser.parse_args() device = get_devices(args.cuda)[0] print_versions() mode = Mode.PER_STUDY LABELS = [ 'Atelectasis', 'Cardiomegaly', 'Consolidation', 'Edema', 'Pleural Effusion' ] model_path = Path(args.model).resolve() #model_path = Path("train_20190526_per_study/model_epoch_030.pth.tar").resolve() env = PredictEnvironment(5, device, mode, model_path=model_path) p = Predictor(env) input_path = Path(args.input_csv).resolve() entries = load_manifest(input_path, mode) base_path = input_path.parent
def train(args): devices = utils.get_devices(args.gpu) if args.seed is not None: utils.manual_seed(args.seed) logging.info("Loading data...") dataloader = create_dataloader(args) vocabs = dataloader.dataset.vocabs if args.validate: val_dataloader = create_dataloader(args, vocabs, True) else: val_dataloader = None fnames = [f"{mode}.vocab" for mode in MODES] for vocab, fname in zip(vocabs, fnames): utils.save_pkl(vocab, os.path.join(args.save_dir, fname)) logging.info("Initializing training environment...") resume_from = dict() if args.resume_from is not None: resume_from = torch.load(args.resume_from) mdl = prepare_model(args, vocabs, resume_from) mdl = utils.to_device(mdl, devices) optimizer_cls = get_optimizer_cls(args) validator = None if args.validate: validator = Validator( model=mdl, device=devices[0], vocabs=vocabs, bos=args.bos, eos=args.eos, unk=args.unk, alpha=args.loss_alpha, beta=args.loss_beta, progress=args.show_progress, batch_stats=args.log_stats ) trainer = Trainer( model=mdl, model_path=args.model_path, alpha=args.loss_alpha, beta=args.loss_beta, device=devices[0], vocabs=vocabs, epochs=args.epochs, save_dir=args.save_dir, save_period=args.save_period, optimizer_cls=optimizer_cls, samples=args.samples, tensorboard=args.tensorboard, progress=args.show_progress, validator=validator, batch_stats=args.log_stats, early_stop=args.early_stop, early_stop_criterion=args.early_stop_criterion, early_stop_patience=args.early_stop_patience ) trainer.load_snapshot(resume_from) report_model(trainer) logging.info("Commencing training joint-lu...") trainer.train(dataloader, val_dataloader) logging.info("Done!")