def bulk_clone(self, count, attrs=None, batch_size=None, auto_commit=False): ops = connections[self.__class__._default_manager.db].ops objs = range(count) clones = [] batch_size = (batch_size or max(ops.bulk_batch_size([], list(objs)), 1)) with conditional( auto_commit, transaction_autocommit( using=self.__class__._default_manager.db), ): # If count exceeds the MAX_UNIQUE_DUPLICATE_QUERY_ATTEMPTS with conditional( self.MAX_UNIQUE_DUPLICATE_QUERY_ATTEMPTS < count, context_mutable_attribute( self, 'MAX_UNIQUE_DUPLICATE_QUERY_ATTEMPTS', count, ), ): if not self.MAX_UNIQUE_DUPLICATE_QUERY_ATTEMPTS >= count: raise AssertionError( 'An Unknown error has occured: Expected ({}) >= ({})'. format(self.MAX_UNIQUE_DUPLICATE_QUERY_ATTEMPTS, count), ) clones = list(repeat(self.make_clone(attrs=attrs), batch_size)) return clones
def test_returning_true_from_exit_handles_exception(self): cm = self.make_one(__exit__=lambda x,y,z:True) flexmock(cm).should_call('__enter__').once flexmock(cm).should_call('__exit__').once with conditional(True, cm): raise RuntimeError()
def deploy(no_push, skip_crd_check, interactive, extra_config_args, retries=5, template='test', logs=False, verbose=False, catch_exception=None): deploy = DeployCommand({ 'deploy': True, '--no-push': no_push, '--skip-crd-check': skip_crd_check, '--interactive': interactive, '--retries': retries, '--logs': logs, '--verbose': verbose }) deploy.config = { 'name': 'app', 'namespace': 'namespace', 'template': template } deploy.config.update(extra_config_args) with catch_stdout() as caught_output: with conditional(catch_exception, pytest.raises(catch_exception)): deploy.action() output = caught_output.getvalue() return output
def test_false_condition_returns_None(self): cm = self.make_one() flexmock(cm).should_call('__enter__').never flexmock(cm).should_call('__exit__').never with conditional(False, cm) as value: self.assertEqual(value, None)
def test_false_condition_does_not_enter_context_manager(self): cm = self.make_one() flexmock(cm).should_call('__enter__').never flexmock(cm).should_call('__exit__').never with conditional(False, cm): pass
def test_true_condition_returns_enter_result(self): cm = self.make_one(__enter__=lambda:42) flexmock(cm).should_call('__enter__').once flexmock(cm).should_call('__exit__').once with conditional(True, cm) as value: self.assertEqual(value, 42)
def test_true_condition_enters_context_manager(self): cm = self.make_one() flexmock(cm).should_call('__enter__').once flexmock(cm).should_call('__exit__').once with conditional(True, cm): pass
def main(): # Get parameters bundle_topic = rospy.get_param('~bundle_topic', '/bc/bundle') src_bag_path = rospy.get_param('~source_bag_path', None) backchannel_topic = rospy.get_param('~backchannel_topic', '/bc/backchannels') sink_bag_path = rospy.get_param('~sink_bag_path', None) # Instantiate source if src_bag_path: bundle_source = BagSource(src_bag_path, bundle_topic) else: bundle_source = TopicSource(bundle_topic, Bundle) # Instantiate sinks if sink_bag_path: bag = rosbag.Bag(sink_bag_path, 'w') backchannel_sink = BagSink(bag, backchannel_topic, Backchannel) else: bag = None backchannel_sink = TopicSink(backchannel_topic, Backchannel) with conditional(bag, bag): with bundle_source, backchannel_sink: for msg, t in bundle_source: if random.random() > 0.2: backchannel_msg = Backchannel() backchannel_msg.header.stamp = t pids = [1,2,3] to_pid = random.choice(pids) pids.remove(to_pid) from_pid = random.choice(pids) backchannel_msg.to_pid = to_pid backchannel_msg.from_pid = from_pid backchannel_sink.put(backchannel_msg, t)
def test_returning_None_from_exit_lets_exception_propagate(self): cm = self.make_one() flexmock(cm).should_call('__enter__').once flexmock(cm).should_call('__exit__').once with self.assertRaises(RuntimeError): with conditional(True, cm): raise RuntimeError()
def launch_benchmark(mock_platform_util, request, mock_isdir, mock_isfile, mock_islink, mock_stat, mock_path_exists): """sets up launch_benchmark obj for every test case and handles catching errors if we wanna test that To catch errors called when running launch_benchmark, use something like: ['catch_error', SystemExit, [{args}], {error_message}] in parametrize where args are args to pass to the benchmark creation and error_message is an optional error message to check for otherwise just pass in the req args you'd like to run with via [] catch_error_override_all_params will not use any example_req_args when creating benchmark Sample request.params: ['catch_error', SystemExit, []] ['catch_error_override_all_params', SystemExit, []] ['catch_error', SystemExit, ['--framework', 'foo'], "The specified framework is not supported"]] """ catch_error = False error = None error_message = '' # deleting from this sometimes so need to redeclare it, probably can do that differently... example_req_args = [ "--model-name", test_model_name, "--framework", test_framework, "--mode", test_mode, "--precision", test_precision, "--docker-image", test_docker_image, "--batch-size", test_batch_size, "--num-cores", test_num_cores ] if hasattr(request, 'param'): if 'catch_error' in request.param[0]: catch_error = True error = request.param[1] if request.param[0] != 'catch_error_override_all_params': # TODO: make more efficient! Want to get rid of any example_req_args that exist in request.param[2] # using safe deletion from the back for idx in range(len(example_req_args) - 1, -1, -1): arg = example_req_args[idx] if not arg.startswith('--'): continue if arg in request.param[2]: # flags are always followed by their value in example_req_args, so delete both arg and its value del example_req_args[idx] del example_req_args[idx] req_args = request.param[2] + example_req_args else: req_args = request.param[2] error_message = request.param[3] if len(request.param) == 4 else '' else: # add extra arguments to the default ones when calling LaunchBenchmark req_args = request.param + example_req_args else: # only use default arguments when calling LaunchBenchmark req_args = example_req_args with mock_patch.object(sys, "argv", ['run_tf_benchmark.py'] + req_args): with conditional(catch_error, pytest.raises(error)) as e: obj = LaunchBenchmark(mock_platform_util) if error_message: assert error_message in str(e.value) return obj
def get_events(catch_exception=None, job_name=None): events_command = EventsCommand({'events': True, '--job-name': job_name}) events_command.config = {'name': 'app', 'namespace': 'namespace'} with catch_stdout() as caught_output: with conditional(catch_exception, pytest.raises(catch_exception)): events_command.action() output = caught_output.getvalue().strip() return output
def status(count=1, catch_exception=None): status_cmd = StatusCommand({'<count>': count}) status_cmd.config = {'name': 'app', 'namespace': 'namespace'} with catch_stdout() as caught_output: with conditional(catch_exception, pytest.raises(catch_exception)): status_cmd.action() output = caught_output.getvalue() return output
def run(project): config_path = project.get_property("conf") + "/" config.load(config_path) with conditional(config.PIPELINE_CONFIGS['code_profiling'], Profiler()): dataset_path = project.get_property("dataset") logging_config.load() start.create_output_dir(dataset_path, project.get_property("output")) start.snapshot_config(config_path) from main import main main(dataset_path) start.clean()
def test_returning_None_from_exit_lets_exception_propagate(self): cm = self.make_one() flexmock(cm).should_call('__enter__').once flexmock(cm).should_call('__exit__').once try: with conditional(True, cm): raise RuntimeError() except RuntimeError: pass # success else: self.fail('RuntimeError not raised')
def GetCards(setid, showBar): set_url = "https://www.neonmob.com/api/setts/" + str(setid) + "/" data = requests.request('GET', set_url).json() set_name = data['name'] total = 0 for cat in range(len(data['core_stats'])): total += data['core_stats'][cat]['total'] for cat in range(len(data['special_stats'])): total += data['special_stats'][cat]['total'] print("\nGetting cards from series \"" + set_name + "\"...") cards = [] nxt = "/api/sets/" + str(setid) + "/pieces/" with conditional(showBar, alive_bar(total, bar='smooth', spinner='dots_recur')) as bar: first = True while True: raw = requests.request('GET', "https://www.neonmob.com" + nxt) if raw.status_code == 500 and first: print("Using fallback card endpoint...") raw = requests.request( 'GET', "https://www.neonmob.com/api/sets/" + str(setid) + "/piece-names") data = raw.json() for card in data: cards.append({ 'name': card['name'], 'id': card['id'], 'setName': set_name }) if showBar: bar() if not showBar: print('...', end="", flush=True) break else: data = raw.json() nxt = data['payload']['metadata']['resultset']['link']['next'] for card in data['payload']['results']: cards.append({ 'name': card['name'], 'id': card['id'], 'setName': set_name }) if showBar: bar() if not showBar: print(". ", end="", flush=True) first = False if not nxt: break return cards
def build_graph(self, *args): costs = [] for i, name in enumerate(self.agent_names): joint_state, action, reward, isOver, comb_mask, joint_fine_mask = args[i * 6:(i + 1) * 6] with tf.variable_scope(name): with conditional(name is None, varreplace.freeze_variables()): state = tf.identity(joint_state[:, 0, :, :, :], name='state') fine_mask = tf.identity(joint_fine_mask[:, 0, :], name='fine_mask') self.predict_value = self.get_DQN_prediction(state, comb_mask, fine_mask) if not get_current_tower_context().is_training: continue # reward = tf.clip_by_value(reward, -1, 1) next_state = tf.identity(joint_state[:, 1, :, :, :], name='next_state') next_fine_mask = tf.identity(joint_fine_mask[:, 1, :], name='next_fine_mask') action_onehot = tf.one_hot(action, self.num_actions, 1.0, 0.0) pred_action_value = tf.reduce_sum(self.predict_value * action_onehot, 1) # N, max_pred_reward = tf.reduce_mean(tf.reduce_max( self.predict_value, 1), name='predict_reward') summary.add_moving_summary(max_pred_reward) with tf.variable_scope('target'), varreplace.freeze_variables(skip_collection=True): # we are alternating between comb and fine states targetQ_predict_value = self.get_DQN_prediction(next_state, tf.logical_not(comb_mask), next_fine_mask) # NxA if self.method != 'Double': # DQN best_v = tf.reduce_max(targetQ_predict_value, 1) # N, else: # Double-DQN next_predict_value = self.get_DQN_prediction(next_state, tf.logical_not(comb_mask), next_fine_mask) self.greedy_choice = tf.argmax(next_predict_value, 1) # N, predict_onehot = tf.one_hot(self.greedy_choice, self.num_actions, 1.0, 0.0) best_v = tf.reduce_sum(targetQ_predict_value * predict_onehot, 1) target = reward + (1.0 - tf.cast(isOver, tf.float32)) * self.gamma * tf.stop_gradient(best_v) # target = tf.Print(target, [target], summarize=100) # tf.assert_greater(target, -100., message='target error') # tf.assert_greater(pred_action_value, -100., message='pred value error') # pred_action_value = tf.Print(pred_action_value, [pred_action_value], summarize=100) l2_loss = tensorpack.regularize_cost(name + '/dqn.*W{1}', l2_regularizer(1e-3)) # cost = tf.losses.mean_squared_error(target, pred_action_value) with tf.control_dependencies([tf.assert_greater(target, -100., message='target error'), tf.assert_greater(pred_action_value, -100., message='pred value error')]): cost = tf.losses.huber_loss( target, pred_action_value, reduction=tf.losses.Reduction.MEAN) summary.add_param_summary((name + '.*/W', ['histogram', 'rms'])) # monitor all W summary.add_moving_summary(cost) costs.append(cost) if not get_current_tower_context().is_training: return return tf.add_n([costs[i] * self.cost_weights[i] for i in range(3)])
def call_logs(catch_exception=None, job_name=None): logs_command = LogsCommand( {'logs': True, '--since': '1m', '--retries': 5, '--job-name': job_name }) logs_command.config = {'name': 'app', 'namespace': 'namespace'} with catch_stdout() as caught_output: with conditional(catch_exception, pytest.raises(catch_exception)): logs_command.action() output = caught_output.getvalue().strip() return output
def GetSeekers(card, showBar): if card['id'] == -1: print("\nCouldn't find card " + card['name'] + " in set " + card['setName']) return [] print("\nGetting seekers of " + card['name'] + " [" + str(card['id']) + "]...") seekers = [] data = requests.request( 'GET', "https://www.neonmob.com/api/pieces/" + str(card['id']) + "/needers/?completion=desc&grade=desc&wishlist=desc").json() total = data['count'] with conditional(showBar, alive_bar(total, bar='smooth', spinner='dots_recur')) as bar: while True: nxt = data['next'] for seeker in data['results']: seekers.append({ 'id': seeker['id'], 'name': seeker['name'], 'trader_score': seeker['trader_score'], 'wishlisted': seeker['wishlisted'], 'needs_special_piece_count': seeker['special_piece_count'], 'needs_owned_special_piece_count': seeker['owned_special_piece_count'], 'needs_owned_percentage': seeker['owned_percentage'], 'needs_card_name': card['name'], 'needs_card_set_name': card['setName'] }) if showBar: bar() if not showBar: print(". ", end="", flush=True) if not nxt: break data = requests.request('GET', "https://www.neonmob.com" + nxt).json() return seekers
def run(self, end=None, timeout=0, **kwargs): if end: end = callbacks.set_end_of_execution_callback(self.ql, end) self._enable_sanitizers() try: # Don't collect coverage information unless explicitly requested by the user. with conditional( self.coverage_file, cov_utils.collect_coverage(self.ql, 'drcov_exact', self.coverage_file)): self.ql.run(end=end, timeout=timeout) except fault.ExitEmulation: # Exit cleanly. pass
def main(): # Get parameters probability_topic = rospy.get_param('~probability_topic', '/bc/probabilities') backchannel_topic = rospy.get_param('~backchannel_topic', '/bc/backchannels') threshold = rospy.get_param('~threshold', 0.9) src_bag_path = rospy.get_param('~source_bag_path', None) sink_bag_path = rospy.get_param('~sink_bag_path', None) # Instantiate source if src_bag_path: probability_source = BagSource(src_bag_path, probability_topic) else: probability_source = TopicSource(probability_topic, BackchannelProbability) # Instatiate bag, if necessary if sink_bag_path: bag = rosbag.Bag(sink_bag_path, 'w') bc_sink = BagSink(bag, backchannel_topic, Backchannel) else: bag = None bc_sink = TopicSink(backchannel_topic, Backchannel) predictors = {1: {}, 2: {}, 3: {}} for actor_pid in [1, 2, 3]: for receiver_pid in [1, 2, 3]: if actor_pid == receiver_pid: continue predictors[actor_pid][receiver_pid] = Predictor( actor_pid, receiver_pid, threshold) with conditional(bag, bag): with probability_source, bc_sink: for msg, t in probability_source: predictors[msg.from_pid][msg.to_pid].put(msg.probability, t) for bc_t in predictors[msg.from_pid][msg.to_pid]: new_msg = Backchannel() new_msg.header.stamp = bc_t new_msg.from_pid = msg.from_pid new_msg.to_pid = msg.to_pid bc_sink.put(new_msg, bc_t)
def GetOwners(card, showBar, force=False): global OCACHE purgeCache() if card['id'] == -1: print("\nCouldn't find card " + card['name'] + " in set " + card['setName']) return [] currentMillis = int(round(time.time() * 1000)) if str(card['id']) in OCACHE.keys() and not force: print("Card is in cache") if currentMillis - OCACHE[str(card['id'])]['time'] < (keepalivemins * 60 * 1000): print("Time is under 10 minutes") return OCACHE[str(card['id'])]['owners'] print("\nGetting owners of " + card['name'] + " [" + str(card['id']) + "]...") owners = [] data = requests.request( 'GET', "https://www.neonmob.com/api/pieces/" + str(card['id']) + "/owners/?completion=asc&grade=desc&owned=desc").json() total = data['count'] with conditional(showBar, alive_bar(total, bar='smooth', spinner='dots_recur')) as bar: while True: nxt = data['next'] for owner in data['results']: owners.append({ 'id': owner['id'], 'name': owner['name'], 'trader_score': owner['trader_score'], 'owns': [{ 'card_id': card['id'], 'card_name': card['name'], 'set_name': card['setName'], 'print_count': owner['print_count'], 'total_specials': owner['special_piece_count'], 'specials': owner['owned_special_piece_count'], 'percentage': owner['owned_percentage'] }], 'wants': [] }) if showBar: bar() if not showBar: print(". ", end="", flush=True) if not nxt: break data = requests.request('GET', "https://www.neonmob.com" + nxt).json() try: OCACHE.pop(str(card['id'])) except KeyError: print("Card is not in cache") currentMillis = int(round(time.time() * 1000)) OCACHE.update({ card['id']: { 'time': currentMillis, 'cardName': card['name'], 'setName': card['setName'], 'owners': owners } }) saveCache() return owners
else: # TEST if args.test: ids = glob.glob(join(TEST_FOLDER, '*.tif')) elif args.test_train: ids = glob.glob(join(TRAIN_FOLDER, '*/*.jpg')) else: assert False ids.sort() match = re.search(r'([^/]*)\.hdf5', args.model) model_name = match.group(1) + ('_tta_' + args.ensembling if args.tta else '') csv_name = 'submission_' + model_name + '.csv' with conditional(args.test, open(csv_name, 'w')) as csvfile: if args.test: csv_writer = csv.writer(csvfile, delimiter=',', quotechar='|', quoting=csv.QUOTE_MINIMAL) csv_writer.writerow(['fname', 'camera']) classes = [] else: correct_predictions = 0 for i, idx in enumerate(tqdm(ids)): img = np.array(Image.open(idx))
def __init__(self, embs_type, embs_shape, cpuembs, w_stddev, **kwargs): embs_dim = embs_shape[1] with conditional(cpuembs, tf.device('/cpu:0')): self.init_embs_subgraph(embs_type, embs_shape) # Inputs and vars with conditional(cpuembs, tf.device('/cpu:0')): x_vec_ph = tf.placeholder(embs_type, [None, embs_dim], name='x_vec_ph') y_ind_ph = tf.placeholder(tf.int32, [None, 1], name='y_ind_ph') x = x_vec_ph # n x dim #TODO: add x_ind placeholder and use EmbeddingsLookup(y_ind) instead of placeholder? y_ind = y_ind_ph[:, 0] # n y = tf.nn.embedding_lookup(self.embs_var, y_ind, name='y_embs_lookup') # print(self.embs_var.get_shape(), self.embs_var.dtype) print(y.get_shape(), y.dtype) # Predict hypernym vector from hyponym vector with tf.name_scope('xW'): W = tf.Variable(tf.random_normal([embs_dim, embs_dim], stddev=w_stddev), name='W') # dim x dim y_hat = tf.matmul(x, W) # n x dim # Dot similarity of predicted hypernym with true hypernym with tf.name_scope('score_yhat_y'): pos_logit = tf.reduce_sum(y_hat * y, axis=1) # n print(pos_logit.get_shape(), pos_logit.dtype) # Dot similarity of predicted hypernym with all words with tf.name_scope('score_yhat_embs'), conditional( cpuembs, tf.device('/cpu:0')): dot = tf.matmul(y_hat, self.embs_var, transpose_b=True) # n x V with tf.name_scope('negative_ind'): top_k_vals, top_k_inds = tf.nn.top_k(dot, k=2, sorted=True) # n x 2 print(top_k_vals.get_shape()) # takes all top_k y' which are not equal to true y # TODO: try also taking only first y' not equal to true y top_mask = tf.not_equal(y_ind, top_k_inds[:, 0], name='top_mask') # n print(top_mask.get_shape()) neg_ind = tf.where(top_mask, top_k_inds[:, 0], top_k_inds[:, 1]) # n # neg_logit = tf.where(top_mask, top_k_vals[:, 0], top_k_vals[:, 1]) # n # print(neg_logit.get_shape()) with tf.name_scope('score_yhat_yneg'): with conditional(cpuembs, tf.device('/cpu:0')): y_neg = tf.nn.embedding_lookup(self.embs_var, neg_ind, name='y_neg_embs_lookup') neg_logit = tf.reduce_sum(y_hat * tf.stop_gradient(y_neg), axis=1) # n print(neg_logit.get_shape()) # Loss with tf.name_scope('loss'): with tf.name_scope('pos_loss'): pos_loss = tf.nn.sigmoid_cross_entropy_with_logits( pos_logit, tf.constant(1, dtype=tf.float32, shape=[1])) with tf.name_scope('neg_loss'): neg_loss = tf.nn.sigmoid_cross_entropy_with_logits( neg_logit, tf.constant(0, dtype=tf.float32, shape=[1])) loss = tf.reduce_mean(pos_loss + neg_loss) print(pos_loss.get_shape(), neg_loss.get_shape(), loss.get_shape()) # Evaluation with tf.name_scope('evaluation'): tf.summary.histogram('W', W) tf.summary.histogram('neg_logit', neg_logit) tf.summary.histogram('pos_logit', pos_logit) tf.summary.scalar('avg_neg_logit', tf.reduce_mean(neg_logit)) tf.summary.scalar('avg_pos_logit', tf.reduce_mean(pos_logit)) tf.summary.scalar('avg_neg_LOSS', tf.reduce_mean(neg_loss)) tf.summary.scalar('avg_pos_LOSS', tf.reduce_mean(pos_loss)) tf.summary.scalar('LOSS', loss) acc_10 = tf.reduce_mean( tf.cast(tf.nn.in_top_k(dot, y_ind, k=10), tf.float32)) acc_2 = tf.reduce_mean( tf.cast(tf.nn.in_top_k(dot, y_ind, k=2), tf.float32)) tf.summary.scalar('acc_10', acc_10) tf.summary.scalar('acc_2', acc_2) # Summaries self.summary = tf.summary.merge_all() self.X = x_vec_ph self.Y = y_ind_ph self.Z = tf.placeholder(tf.float32, shape=[None, embs_dim], name='Z') # not used, for compatibility self.Y_hat = y_hat self.loss = loss
class_weight=class_weight) else: # TEST if args.test: ids = glob.glob(join(TEST_FOLDER, '*.tif')) elif args.test_train: ids = glob.glob(join(TRAIN_FOLDER, '*/*.jpg')) else: assert False ids.sort() from conditional import conditional with conditional(args.test, open('submission.csv', 'w')) as csvfile: if args.test: csv_writer = csv.writer(csvfile, delimiter=',', quotechar='|', quoting=csv.QUOTE_MINIMAL) csv_writer.writerow(['fname', 'camera']) else: correct_predictions = 0 for i, idx in enumerate(tqdm(ids)): img = np.array(Image.open(idx)) manipulated = np.float32([1. if idx.find('manip') != -1 else 0.])
def run(loader, model, loss_function, distances, all_soft_labels, classes, opts, epoch, prev_steps, optimizer=None, is_inference=True, corrector=lambda x: x): """ Runs training or inference routine for standard classification with soft-labels style losses """ max_dist = max(distances.distances.values()) # for each class, create the optimal set of retrievals (used to calculate hierarchical precision @k) best_hier_similarities = _make_best_hier_similarities( classes, distances, max_dist) # Using different logging frequencies for training and validation log_freq = 1 if is_inference else opts.log_freq # strings useful for logging descriptor = "VAL" if is_inference else "TRAIN" loss_id = "loss/" + opts.loss dist_id = "ilsvrc_dist" # Initialise TensorBoard with_tb = opts.out_folder is not None if with_tb: tb_writer = tensorboardX.SummaryWriter( os.path.join(opts.out_folder, "tb", descriptor.lower())) # Initialise accumulators to store the several measures of performance (accumulate as sum) num_logged = 0 loss_accum = 0.0 time_accum = 0.0 norm_mistakes_accum = 0.0 flat_accuracy_accums = np.zeros(len(topK_to_consider), dtype=np.float) hdist_accums = np.zeros(len(topK_to_consider)) hdist_top_accums = np.zeros(len(topK_to_consider)) hdist_mistakes_accums = np.zeros(len(topK_to_consider)) hprecision_accums = np.zeros(len(topK_to_consider)) hmAP_accums = np.zeros(len(topK_to_consider)) # Affects the behaviour of components such as batch-norm if is_inference: model.eval() else: model.train() with conditional(is_inference, torch.no_grad()): time_load0 = time.time() for batch_idx, (embeddings, target) in enumerate(loader): this_load_time = time.time() - time_load0 this_rest0 = time.time() assert embeddings.size( 0 ) == opts.batch_size, "Batch size should be constant (data loader should have drop_last=True)" if opts.gpu is not None: embeddings = embeddings.cuda(opts.gpu, non_blocking=True) target = target.cuda(opts.gpu, non_blocking=True) # get model's prediction output = model(embeddings) # for soft-labels we need to add a log_softmax and get the soft labels if opts.loss == "soft-labels": output = torch.nn.functional.log_softmax(output, dim=1) if opts.soft_labels: target_distribution = make_batch_soft_labels( all_soft_labels, target, opts.num_classes, opts.batch_size, opts.gpu) else: target_distribution = make_batch_onehot_labels( target, opts.num_classes, opts.batch_size, opts.gpu) loss = loss_function(output, target_distribution) else: loss = loss_function(output, target) if not is_inference: optimizer.zero_grad() loss.backward() optimizer.step() # start/reset timers this_rest_time = time.time() - this_rest0 time_accum += this_load_time + this_rest_time time_load0 = time.time() # only update total number of batch visited for training tot_steps = prev_steps if is_inference else prev_steps + batch_idx # correct output of the classifier (for yolo-v2) output = corrector(output) # if it is time to log, compute all measures, store in summary and pass to tensorboard. if batch_idx % log_freq == 0: num_logged += 1 # compute flat topN accuracy for N \in {topN_to_consider} topK_accuracies, topK_predicted_classes = accuracy( output, target, ks=topK_to_consider) loss_accum += loss.item() topK_hdist = np.empty([opts.batch_size, topK_to_consider[-1]]) for i in range(opts.batch_size): for j in range(max(topK_to_consider)): class_idx_ground_truth = target[i] class_idx_predicted = topK_predicted_classes[i][j] topK_hdist[i, j] = distances[( classes[class_idx_predicted], classes[class_idx_ground_truth])] # select samples which returned the incorrect class (have distance!=0 in the top1 position) mistakes_ids = np.where(topK_hdist[:, 0] != 0)[0] norm_mistakes_accum += len(mistakes_ids) topK_hdist_mistakes = topK_hdist[mistakes_ids, :] # obtain similarities from distances topK_hsimilarity = 1 - topK_hdist / max_dist # all the average precisions @k \in [1:max_k] topK_AP = [ np.sum(topK_hsimilarity[:, :k]) / np.sum(best_hier_similarities[:, :k]) for k in range(1, max(topK_to_consider) + 1) ] for i in range(len(topK_to_consider)): flat_accuracy_accums[i] += topK_accuracies[i].item() hdist_accums[i] += np.mean( topK_hdist[:, :topK_to_consider[i]]) hdist_top_accums[i] += np.mean([ np.min(topK_hdist[b, :topK_to_consider[i]]) for b in range(opts.batch_size) ]) hdist_mistakes_accums[i] += np.sum( topK_hdist_mistakes[:, :topK_to_consider[i]]) hprecision_accums[i] += topK_AP[topK_to_consider[i] - 1] hmAP_accums[i] += np.mean(topK_AP[:topK_to_consider[i]]) # Get measures print("**%8s [Epoch %03d/%03d, Batch %05d/%05d]\t" "Time: %2.1f ms | \t" "Loss: %2.3f (%1.3f)\t" % (descriptor, epoch, opts.epochs, batch_idx, len(loader), time_accum / (batch_idx + 1) * 1000, loss.item(), loss_accum / num_logged)) if not is_inference: # update TensorBoard with the current snapshot of the epoch's summary summary = _generate_summary( loss_accum, flat_accuracy_accums, hdist_accums, hdist_top_accums, hdist_mistakes_accums, hprecision_accums, hmAP_accums, num_logged, norm_mistakes_accum, loss_id, dist_id, ) if with_tb: _update_tb_from_summary(summary, tb_writer, tot_steps, loss_id, dist_id) # update TensorBoard with the total summary of the epoch summary = _generate_summary( loss_accum, flat_accuracy_accums, hdist_accums, hdist_top_accums, hdist_mistakes_accums, hprecision_accums, hmAP_accums, num_logged, norm_mistakes_accum, loss_id, dist_id, ) if with_tb: _update_tb_from_summary(summary, tb_writer, tot_steps, loss_id, dist_id) if with_tb: tb_writer.close() return summary, tot_steps
def main(): # listens for messages on 2 topics, combines them all into a single, flattens them into a joint message by time rospy.init_node('bundler', anonymous=True) # Get parameters audio_features_P1_bag_path = rospy.get_param('~audio_features_P1_bag_path', None) audio_features_P2_bag_path = rospy.get_param('~audio_features_P2_bag_path', None) audio_features_P3_bag_path = rospy.get_param('~audio_features_P3_bag_path', None) audio_features_P1_topic = rospy.get_param('~audio_features_P1_topic', '/pid1/audio/features') audio_features_P2_topic = rospy.get_param('~audio_features_P2_topic', '/pid2/audio/features') audio_features_P3_topic = rospy.get_param('~audio_features_P3_topic', '/pid3/audio/features') start_time_topic = rospy.get_param('~start_time_topic', '/bc/start_time') start_time_bag_path = rospy.get_param('~start_time_bag_path', None) window_duration = rospy.get_param('~window_duration', None) window_duration_topic = rospy.get_param('~window_duration_topic', '/bc/window_duration') window_duration_bag_path = rospy.get_param('~window_duration_bag_path', None) output_topic = rospy.get_param('~output_topic', '/bc/bundle') sink_bag_path = rospy.get_param('~sink_bag_path', None) # Instantiate sources if audio_features_P1_bag_path: audio_features_P1_src = BagSource(audio_features_P1_bag_path, audio_features_P1_topic) else: audio_features_P1_src = TopicSource(audio_features_P1_topic, AudioFeatures) if audio_features_P2_bag_path: audio_features_P2_src = BagSource(audio_features_P2_bag_path, audio_features_P2_topic) else: audio_features_P2_src = TopicSource(audio_features_P2_topic, AudioFeatures) if audio_features_P3_bag_path: audio_features_P3_src = BagSource(audio_features_P3_bag_path, audio_features_P3_topic) else: audio_features_P3_src = TopicSource(audio_features_P3_topic, AudioFeatures) # Instantiate sink if sink_bag_path: sink_bag = rosbag.Bag(sink_bag_path, 'w') sink = BagSink(sink_bag, output_topic, Bundle) else: sink_bag = None sink = TopicSink(output_topic, Bundle) # Find start time, window_duration if start_time_bag_path: start_time_src = BagSource(start_time_bag_path, start_time_topic) else: start_time_src = TopicSource(start_time_topic, Time) with start_time_src: rospy.loginfo('Finding start time.') msg, _ = next(start_time_src) start_time = msg.data rospy.loginfo('Found start time.') # Set window duration if window_duration: window_duration = rospy.Duration(window_duration) else: if window_duration_bag_path: window_duration_src = BagSource(window_duration_bag_path, window_duration_topic) else: window_duration_src = TopicSource(window_duration_topic, Int32) with window_duration_src: rospy.loginfo('Finding window duration') msg, _ = next(window_duration_src) rospy.loginfo('Found window duration') window_duration = rospy.Duration(msg.data / 1000.) # Set-up combiner combiner = Combiner(start_time, window_duration, ['audio_features_P1', 'audio_features_P2', 'audio_features_P3']) with conditional(sink_bag, sink_bag): with audio_features_P1_src, audio_features_P2_src, audio_features_P3_src, sink: sync = Synchronizer([audio_features_P1_src, audio_features_P2_src, audio_features_P3_src]) for (audio_features_P1, audio_features_P2, audio_features_P3) in sync: for (msg, t) in audio_features_P1: combiner.put('audio_features_P1', msg, t, t + window_duration) for (msg, t) in audio_features_P2: combiner.put('audio_features_P2', msg, t, t + window_duration) for (msg, t) in audio_features_P3: combiner.put('audio_features_P3', msg, t, t + window_duration) for bundle, start, end in combiner: h = Header() h.stamp = start bundle['header'] = h sink.put(bundle, start)
cstudent1 = student.conditional(x[0].reshape(1,1)) cstudent2 = student.conditional(x[1].reshape(1,1)) cstudent3 = student.conditional(x[2].reshape(1,1)) cpdf1 = np.asarray([cstudent1.pdf(i) for i in xx]).flatten() cpdf2 = np.asarray([cstudent2.pdf(i) for i in xx]).flatten() cpdf3 = np.asarray([cstudent3.pdf(i) for i in xx]).flatten() # Conditional models # To generate the conditionals for fixed x we first need to rotate the RTBM # model by a Pi/2 angle rM = ut.rotate(M) cM1 = con.conditional(rM, -x[0]) cM2 = con.conditional(rM, -x[1]) cM3 = con.conditional(rM, -x[2]) #~~~~~~~~~~~~~~~~~~~~~~ Plotting ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # Figure for the plot
import conditional as example print(example.conditional(3)) print(example.conditional(15)) print(example.conditional(10))
M.set_parameters( np.array([ 1.02735619e-05, -3.63860337e-04, 2.15571853e+00, -2.96561961e-01, 1.47561952e+01, 9.63622840e-01, -1.03625774e+01, -8.31445704e+00, -2.55729017e-01, -7.01183003e+00, 4.09970744e+00, -3.38018523e+00, 1.70457837e+00, 6.56773389e-01, 1.49908760e+01, 1.15771402e+00, 1.50000000e+01, 1.50000000e+01, 4.59256632e+00, 3.45065143e-01, 2.12470448e+00, 1.50000000e+01, 4.29228972e-02, -1.66464616e+00, 1.49999786e+01, 3.06820775e-01, 1.36879929e+01 ])) # Conditional models y_slice = np.array( [0.6, 0.15, -0.2]) #fixed y at which we calculate the conditional probability p(x|y) M1 = con.conditional(M, y_slice[0]) M2 = con.conditional(M, y_slice[1]) M3 = con.conditional(M, y_slice[2]) #~~~~~~~~~~~~~~~~~~~~~~ Plotting ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # Figure for the plot # Definitions for the axes left, width = 0.1, 0.65 bottom, height = 0.1, 0.65 bottom_h = left_h = left + width + 0.04 rect_scatter = [left, bottom, width, height] rect_histx1 = [left - 0.6, bottom + 0.6, width * 2 / 3, 0.2] rect_histx2 = [left - 0.6, bottom + 0.25, width * 2 / 3, 0.2]
def main(): # Get topic parameters audio_topic = rospy.get_param('~audio_topic') start_time = rospy.get_param('~start_time', None) window_duration = rospy.get_param('~window_duration', None) start_time_topic = rospy.get_param('~start_time_topic', '/bc/start_time') window_duration_topic = rospy.get_param('~window_duration_topic', '/bc/window_duration') features_topic = rospy.get_param('~features_topic', '/bc/audio_features') # Get source, sink parameters src_bag_path = rospy.get_param('~source_bag_path', None) sink_bag_path = rospy.get_param('~sink_bag_path', None) # Instantiate sources if src_bag_path: audio_src = BagSource(src_bag_path, audio_topic) start_time_src = BagSource(src_bag_path, start_time_topic) window_duration_src = BagSource(src_bag_path, window_duration_topic) else: audio_src = TopicSource(audio_topic, AudioData) start_time_src = TopicSource(start_time_topic, Time) window_duration_src = TopicSource(window_duration_topic, Int32) # Instantiate sinks if sink_bag_path: bag = rosbag.Bag(sink_bag_path, 'w') features_sink = BagSink(bag, features_topic, AudioFeatures) else: bag = None features_sink = TopicSink(features_topic, AudioFeatures) # Get the start time rospy.loginfo('Finding start time, window duration.') if not start_time: with start_time_src: msg, _ = next(start_time_src) start_time = msg.data # Get the window duration if window_duration: window_duration = rospy.Duration(window_duration) else: with window_duration_src: msg, _ = next(window_duration_src) window_duration = rospy.Duration(msg.data / 1000.) rospy.loginfo('Found start time, window duration.') with conditional(bag, bag): with audio_src, features_sink: def put_features(bundle, t): h = Header() h.stamp = t bundle['header'] = h features_sink.put(bundle, t) msg, t = next(audio_src) audio = ((msg.data, t) for (msg, t) in audio_src) compute_audio_features(audio=audio, put_features=put_features, start_time=start_time, window_duration=window_duration, get_duration=rospy.Duration, sample_rate=msg.sample_rate, sample_width=msg.sample_width, num_channels=msg.num_channels, is_bigendian=msg.is_bigendian)
def main(): # listens for messages on 2 topics, combines them all into a single, flattens them into a joint message by time rospy.init_node('bundler', anonymous=True) # Get parameters audio_topic = rospy.get_param('~audio_topic', '/bc/audio_features') audio_bag_path = rospy.get_param('~audio_bag_path', None) nod_topic = rospy.get_param('~nod_topic', '/bc/nod_features') nod_bag_path = rospy.get_param('~nod_bag_path', None) start_time_topic = rospy.get_param('~start_time_topic', '/bc/start_time') start_time_bag_path = rospy.get_param('~start_time_bag_path', None) window_duration_topic = rospy.get_param('~window_duration_topic', '/bc/window_duration') window_duration_bag_path = rospy.get_param('~window_duration_bag_path', None) output_topic = rospy.get_param('~output_topic', '/bc/bundle') sink_bag_path = rospy.get_param('~sink_bag_path', None) # Instantiate sources if audio_bag_path: audio_src = BagSource(audio_bag_path, audio_topic) else: audio_src = TopicSource(audio_topic, AudioFeatures) if nod_bag_path: # nod_src = BagSource(nod_bag_path, nod_topic) nod_src = BagSource(nod_bag_path, nod_topic) else: # nod_src = TopicSource(nod_topic, NodFeature) nod_src = TopicSource(nod_topic, NodFeatures) if start_time_bag_path: start_time_src = BagSource(start_time_bag_path, start_time_topic) else: start_time_src = TopicSource(start_time_topic, Time) if window_duration_bag_path: window_duration_src = BagSource(window_duration_bag_path, window_duration_topic) else: window_duration_src = TopicSource(window_duration_topic, Int32) # Instantiate sink if sink_bag_path: sink_bag = rosbag.Bag(sink_bag_path, 'w') sink = BagSink(sink_bag, output_topic, Bundle) else: sink_bag = None sink = TopicSink(output_topic, Bundle) # Set-up combiner rospy.loginfo('Finding start time, window duration.') with start_time_src: msg, _ = next(start_time_src) start_time = msg.data with window_duration_src: msg, _ = next(window_duration_src) window_duration = rospy.Duration(msg.data / 1000.) rospy.loginfo('Found start time, window duration.') combiner = Combiner(start_time, window_duration, ['audio', 'nod']) with conditional(sink_bag, sink_bag): with audio_src, nod_src, sink: sync = Synchronizer([audio_src, nod_src]) for (audio, nod) in sync: for (msg, t) in audio: combiner.put('audio', msg, t, t + window_duration) for (msg, t) in nod: combiner.put('nod', msg, t, t + window_duration) for bundle, start, end in combiner: h = Header() h.stamp = start bundle['header'] = h sink.put(bundle, start)