def get_kids_qso_map(qsos, nside): map = get_map(qsos['RAJ2000'].values, qsos['DECJ2000'].values, nside=nside) mask = get_map(qsos['RAJ2000'].values, qsos['DECJ2000'].values, nside=256) mask = hp.ud_grade(mask, nside) mask[mask.nonzero()] = 1 map = get_masked_map(map, mask) return map, mask
def eval_epoch_linear_probe(self, eval_list, epoch, vocab_dict, print_sample =False): label_binarizer = sklearn.preprocessing.LabelBinarizer() label_binarizer.fit(range(self.target_vocab_size)) vocab_dict_rev = {v: k for k, v in vocab_dict.items()} self.linear.eval() total_loss = list([]) map_list = list([]) ppl_list = list([]) query_map_list = list([]) query_ppl_list = list([]) target_map_list = list([]) target_ppl_list = list([]) with torch.no_grad(): for i, instance in enumerate(eval_list): labels_onehot, masks_onehot, labels, _ = self.get_training_labels(instance[self.query_indices], instance[self.fact_indices], instance[self.negative_indices]) output_ = self.linear(instance[self.input_type].to(self.device)) # output size is (6600) output = nn.functional.sigmoid(output_) loss = self.get_loss(output, torch.tensor(labels_onehot, dtype=torch.float32).to(self.device), torch.tensor(masks_onehot, dtype=torch.float32).to(self.device)) total_loss.append(loss.detach().cpu().numpy()) map_list.append(get_map(output.detach().cpu().numpy(), labels)) ppl_list.append(get_ppl(output.detach().cpu().numpy(), labels)) query_labels_eval = np.array(instance[self.query_indices]) target_labels_eval = np.array(list(set(instance[self.fact_indices])-set(instance[self.query_indices]))) if len(query_labels_eval)>0: query_map_list.append(get_map(output.detach().cpu().numpy(), query_labels_eval)) query_ppl_list.append(get_ppl(output.detach().cpu().numpy(), query_labels_eval)) if len(target_labels_eval)>0: target_map_list.append(get_map(output.detach().cpu().numpy(), target_labels_eval)) target_ppl_list.append(get_ppl(output.detach().cpu().numpy(), target_labels_eval)) result_dict = {"eval_loss":total_loss, "avg map":map_list, "avg ppl":ppl_list, "query map:": query_map_list, "query ppl:": query_ppl_list, "target map:": target_map_list, "target ppl:": target_ppl_list} print("-" * 20) result_summary = {x:sum(result_dict[x])/len(result_dict[x]) for x in result_dict.keys()} print(result_summary) return result_summary , result_dict
def train_epoch_linear_probe(self, train_list, epoch, save_folder_path): self.linear.train() total_loss = 0 random.shuffle(train_list) map_list = list([]) ppl_list = list([]) for i, instance in enumerate(train_list): self.optimizer.zero_grad() labels_onehot, masks_onehot, labels, label_masks = self.get_training_labels(instance[self.query_indices], instance[self.fact_indices], instance[self.negative_indices]) output_ = self.linear(instance[self.input_type].to(self.device)) # output size is (6600) output = nn.functional.sigmoid(output_) loss = self.get_loss(output, torch.tensor(labels_onehot, dtype = torch.float32).to(self.device), torch.tensor(masks_onehot, dtype = torch.float32).to(self.device)) loss.backward() self.optimizer.step() total_loss+=loss.detach().cpu().numpy() map_list.append(get_map(output.detach().cpu().numpy(), labels)) ppl_list.append(get_ppl(output.detach().cpu().numpy(), labels)) # if (i + 1) % 10 == 0: # print("\tsample ",i+1, " loss:", total_loss/(i+1)) print("epoch ", epoch,"\tbert total training loss:", total_loss/len(train_list)) return total_loss/len(train_list)
def main(): config_file_name = "/etc/openbaton/ems/conf.ini" log.debug(config_file_name) config = ConfigParser.ConfigParser() config.read(config_file_name) #read config file _map = get_map(section='ems', config=config) #get the data from map queue_type = _map.get("type") #get type of the queue hostname = _map.get("hostname") username = _map.get("username") password = _map.get("password") autodel = _map.get("autodelete") heartbeat = _map.get("heartbeat") exchange_name = _map.get("exchange") queuedel = True if autodel == 'false': queuedel = False if not heartbeat: heartbeat = '60' if not exchange_name: exchange_name = 'openbaton-exchange' rabbit_credentials = pika.PlainCredentials(username, password) connection = pika.BlockingConnection( pika.ConnectionParameters(host=_map.get("orch_ip"), credentials=rabbit_credentials, heartbeat_interval=int(heartbeat))) channel = connection.channel() channel.exchange_declare(exchange=exchange_name, type="topic", durable=True) channel.queue_declare(queue='ems.%s.register' % queue_type, auto_delete=queuedel) channel.queue_declare(queue='vnfm.%s.actions' % hostname, auto_delete=queuedel) channel.queue_bind(exchange=exchange_name, queue='ems.%s.register' % queue_type) channel.queue_bind(exchange=exchange_name, queue='vnfm.%s.actions' % hostname) channel.basic_publish( exchange='', routing_key='ems.%s.register' % queue_type, properties=pika.BasicProperties(content_type='text/plain'), body='{"hostname":"%s"}' % hostname) channel.basic_qos(prefetch_count=1) channel.basic_consume(thread_function, queue='vnfm.%s.actions' % hostname) print "Waiting for actions" channel.start_consuming()
def build_map_by_image(contents, filename): content_type, content_string = contents.split(',') decoded = base64.b64decode(content_string) try: image = Image.open(BytesIO(decoded)) except Exception as e: print(e) return html.Div([ 'Произошла ошибка при обработке файла.' ]) my_map = get_map(image) return my_map
def add_fields(self): for key in self.player_data: #If map if type(self.player_data[key]) == type(dict()): default_map_data = get_map(key) default_node_attributes = list(default_map_data.node(data=True))[0][-1] default_edge_attributes = list(default_map_data.edges(data=True))[0][-1] for affiliation in self.player_data[key]: player_node_attributes = list(self.player_data[key][affiliation].node(data=True))[0][-1] player_edge_attributes = list(self.player_data[key][affiliation].edges(data=True))[0][-1] #Nodes for attribute in list(default_node_attributes.keys()): if attribute not in list(player_node_attributes.keys()): nx.set_node_attributes(self.player_data[key][affiliation], nx.get_node_attributes(default_map_data, attribute), attribute) #Edges for attribute in list(default_edge_attributes.keys()): if attribute not in list(player_edge_attributes.keys()): nx.set_edge_attributes(self.player_data[key][affiliation], nx.get_edge_attributes(default_map_data, attribute), attribute)
def get_lotss_map(lotss_data, data_release, mask_filename=None, nside=2048, cut_pixels=True, masked=True): counts_map = get_map(lotss_data['RA'].values, lotss_data['DEC'].values, nside=nside) if masked: if data_release == 1: mask = get_lotss_dr1_mask(nside) elif data_release == 2: mask = get_lotss_dr2_mask(nside, filename=mask_filename, cut_pixels=cut_pixels) else: raise Exception('Wrong LoTSS data release number') else: mask = None # Get noise in larger bins noise_map = get_aggregated_map(lotss_data['RA'].values, lotss_data['DEC'].values, lotss_data['Isl_rms'].values, nside=256, aggregation='mean') if masked: noise_map = get_masked_map(noise_map, hp.ud_grade(mask, nside_out=256)) counts_map = get_masked_map(counts_map, mask) return counts_map, mask, noise_map
y_1 = 180 h_list = [1.0] polygon_list_list = utils.get_polygon_list_list( h_list, y_0, y_1, x_0, x_1) combined_normal_df_positive_snps = utils.get_df_positive_snps( child_snps, combined_original_df, new_json_tree_rows) combined_normal_df_without_other = utils.get_df_without_other( combined_normal_df_positive_snps) if len(combined_normal_df_without_other.index) > 0: final_df = combined_normal_df_without_other.copy() if is_extended: final_df = utils.get_df_extended( combined_normal_df_without_other, str_number, new_json_tree_rows, child_snps, combined_original_df) if final_df is None: continue utils.get_map(final_df, polygon_list_list, child_snps, target_snp, h_list, db, collection_name) snps_list.append(target_snp) utils.update_db_list(collection_name, db, snps_list) else: print( "В наборе данных combined_normal_df_without_other 0 строк!" ) else: print("У выбранного SNP нет дочерних SNP!") print("Завершена обработка SNP {}".format(target_snp)) print(datetime.datetime.now())
def main(): sleep_time = 1 logging_dir='/var/log/openbaton/' #logging_dir = 'log/openbaton/' if not os.path.exists(logging_dir): os.makedirs(logging_dir) logging.basicConfig(filename=logging_dir + '/ems-receiver.log', level=logging.INFO, format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s', datefmt='%m-%d %H:%M') config_file_name = "/etc/openbaton/openbaton-ems.properties" log.debug(config_file_name) config = ConfigParser.ConfigParser() config.read(config_file_name) # read config file _map = get_map(section='ems', config=config) # get the data from map queue_type = _map.get("type") # get type of the queue hostname = _map.get("hostname") username = _map.get("username") password = _map.get("password") autodel = _map.get("autodelete") heartbeat = _map.get("heartbeat") broker_port = _map.get("broker_port") exchange_name = _map.get("exchange") virtual_host = _map.get("virtual_host") queuedel = True if autodel == 'false': queuedel = False if not heartbeat: heartbeat = '60' if not exchange_name: exchange_name = 'openbaton-exchange' if not broker_port: broker_port = "5672" if not virtual_host: virtual_host = "/" if not queue_type: queue_type = "generic" log.info( "EMS configuration paramters are " "hostname: %s, username: %s, password: *****, autodel: %s, heartbeat: %s, exchange name: %s" % ( hostname, username, autodel, heartbeat, exchange_name)) rabbit_credentials = pika.PlainCredentials(username, password) while True: try: connection = pika.BlockingConnection( pika.ConnectionParameters(host=_map.get("broker_ip"), port=int(broker_port), virtual_host=virtual_host, credentials=rabbit_credentials, heartbeat_interval=int(heartbeat))) channel = connection.channel() #channel.exchange_declare(exchange=exchange_name, type="topic", durable=True) #channel.queue_declare(queue='ems.%s.register'%queue_type, auto_delete=queuedel) channel.queue_bind(exchange=exchange_name, queue='ems.%s.register' % queue_type) channel.queue_declare(queue='vnfm.%s.actions' % hostname, auto_delete=queuedel) channel.queue_bind(exchange=exchange_name, queue='ems.%s.register' % queue_type) channel.queue_bind(exchange=exchange_name, queue='vnfm.%s.actions' % hostname) channel.basic_publish(exchange='', routing_key='ems.%s.register' % queue_type, properties=pika.BasicProperties(content_type='text/plain'), body='{"hostname":"%s"}' % hostname) channel.basic_qos(prefetch_count=1) channel.basic_consume(thread_function, queue='vnfm.%s.actions' % hostname) channel.start_consuming() except Exception: # logging.exception('') time.sleep(sleep_time) if (sleep_time < 10): sleep_time = sleep_time + 1 else: sleep_time = sleep_time + 10
def number_counts_pixels(data, nside=58, x_lim=None, title=None, legend_loc='upper left', columns=BAND_COLUMNS): # Get mask for the whole dataset map, _, _ = get_map(data['RAJ2000'], data['DECJ2000'], nside=nside) mask_non_zero = np.nonzero(map) # Get x limit from all magnitudes if x_lim is None: m_min = int(math.floor(data[columns].values.min())) m_max = int(math.ceil(data[columns].values.max())) bins = np.arange(m_min, m_max + 1.0, 1.0) else: bins = np.arange(x_lim[0], x_lim[1] + 1.0, 1.0) bin_titles = [ '({}, {}]'.format(bins[i], bins[i + 1]) for i, _ in enumerate(bins[:-1]) ] # Plot for every magnitude pixel_densities = pd.DataFrame() for band in columns: # Bin magnitudes data.loc[:, 'bin'] = pd.cut(data[band], bins, labels=False) # For each bin for i in range(len(bins) - 1): data_bin = data.loc[data['bin'] == i] # Get map map, _, _ = get_map(data_bin['RAJ2000'], data_bin['DECJ2000'], nside=nside) map_masked = map[mask_non_zero] pixel_densities = pixel_densities.append(pd.DataFrame({ 'pixel density': map_masked, 'magnitude range': bin_titles[i], 'magnitude': band }), ignore_index=True) sns.catplot(x='magnitude range', y='pixel density', hue='magnitude', data=pixel_densities, kind='bar', aspect=1.7, height=5, legend_out=False, palette='cubehelix') plt.legend(loc=legend_loc, framealpha=1.0) plt.yscale('log') plt.title(title)
#! /usr/bin/python from objects import BigDir from utils import get_map if len(sys.argv) != 2: print("Usage: add_file <device>") exit(1) fd = open(sys.argv[1], "r+b") fs_map = get_map(fd) root_frag_id = fs_map.disc_record.root >> 8 root_sec_offset = fs_map.disc_record.root & 0xff root_locations = fs_map.find_fragment(root_frag_id, fs_map.disc_record.root_size) fd.seek((root_locations[0])[0]) root = BigDir(fd.read(fs_map.disc_record.root_size)) root.delete('Loader') root.add('Loader', 0xffffc856, 0xeadfc18c, 50331648, 3, 0x300) root.sequence += 1 root.show() root.data() fd.seek((root_locations[0])[0]) fd.write(root.data())
def number_counts(data_dict, linear_data, nside=128, step=.1, band_column='MAG_GAAP_r', legend_loc='upper left', legend_size=None): fig, ax = plt.subplots() to_plot_df = pd.DataFrame() x_col = pretty_print_magnitude(band_column) y_col = r'surface density (≤ m) [N / deg$^2$]' for i, (data_name, (data, map)) in enumerate(data_dict.items()): (ra_col, dec_col) = ('RAJ2000', 'DECJ2000') if 'RAJ2000' in data else ('RA', 'DEC') mask_non_zero = np.nonzero(map) print('{} area: {:.2f} deg^2'.format( data_name, len(mask_non_zero[0]) * hp.nside2pixarea(nside, degrees=True))) m_min = int(math.ceil(data[band_column].min())) m_max = int(math.ceil(data[band_column].max())) magnitude_arr = np.arange(m_min, m_max + step, step) density_mean_arr, density_error_arr = [], [] for m_max in magnitude_arr: data_m_max = data.loc[data[band_column] < m_max] map_m_max, _, _ = get_map(data_m_max[ra_col], data_m_max[dec_col], nside=nside) densities = map_m_max[mask_non_zero] / hp.nside2pixarea( nside, degrees=True) (mu, sigma) = stats.norm.fit(densities) density_mean_arr.append(mu) density_error_arr.append(sigma / math.sqrt(densities.shape[0])) to_plot_df = to_plot_df.append(pd.DataFrame({ x_col: magnitude_arr, y_col: density_mean_arr, 'error': density_error_arr, 'data name': [data_name] * len(magnitude_arr), }), ignore_index=True) color_palette = get_cubehelix_palette( len(data_dict), reverse=True) if len(data_dict) > 1 else [(0, 0, 0)] sns.lineplot(x=x_col, y=y_col, data=to_plot_df, hue='data name', palette=color_palette, style='data name', markers=True) plot_linear_data(linear_data) ax = plt.gca() for i, data_name in enumerate(to_plot_df['data name'].unique()): to_plot_single_data = to_plot_df.loc[to_plot_df['data name'] == data_name] lower = to_plot_single_data[ y_col].values - to_plot_single_data['error'].values / 2 upper = to_plot_single_data[ y_col].values + to_plot_single_data['error'].values / 2 ax.fill_between(to_plot_single_data[x_col], lower, upper, color=color_palette[i], alpha=0.2) plt.yscale('log') # handles, labels = ax.get_legend_handles_labels() prop = {'size': legend_size} if legend_size else {} ax.legend(loc=legend_loc, framealpha=1.0, prop=prop) # handles=handles[1:], labels=labels[1:], plt.setp(ax.get_legend().get_texts(), fontsize='9') plt.show()
def spatial_number_density(data_dict, nside=128, z_bin_step=0.5, z_bin_size=0.5, cosmo_model=cosmo_wmap9, z_max=None, legend_size=None): volume_proportion = (hp.nside2pixarea(nside, degrees=True) / 41253.0) fig, ax = plt.subplots() to_plot_df = pd.DataFrame() x_col = 'z' y_col = r'spatial density [N / comoving Mpc$^3$]' for data_name, (data, map) in data_dict.items(): z_column = 'Z' if 'Z' in data else 'Z_PHOTO' z_half_bin_size = z_bin_size / 2 steps = np.arange(data[z_column].min() + z_half_bin_size, data[z_column].max() + z_half_bin_size, z_bin_step) comoving_volumes = np.array([ (cosmo_model.comoving_volume(step + z_half_bin_size) - cosmo_model.comoving_volume(step - z_half_bin_size)).value for step in steps ]) mask_non_zero = np.nonzero(map) print('{} area: {:.2f} deg^2'.format( data_name, len(mask_non_zero[0]) * hp.nside2pixarea(nside, degrees=True))) density_v_max_mean, density_v_max_error = [], [] (ra_col, dec_col) = ('RAJ2000', 'DECJ2000') if ('RAJ2000' in data) else ('RA', 'DEC') for i, step in enumerate(steps): data_step = data.loc[(data[z_column] > step - z_half_bin_size) & (data[z_column] < step + z_half_bin_size)] step_map, _, _ = get_map(data_step[ra_col], data_step[dec_col], v=data_step['v_weight'].values, nside=nside) v_max_values = step_map[mask_non_zero] / comoving_volumes[ i] / volume_proportion (mu, sigma) = stats.norm.fit(v_max_values) density_v_max_mean.append(mu) density_v_max_error.append(sigma / math.sqrt(v_max_values.shape[0])) density_v_max_mean = np.array(density_v_max_mean) density_v_max_error = np.array(density_v_max_error) # comoving_v_max_densities = (density_v_max_mean / comoving_volumes / volume_proportion) to_plot_df = to_plot_df.append(pd.DataFrame({ x_col: steps, y_col: density_v_max_mean, 'error': density_v_max_error, 'data name': [data_name] * len(steps), }), ignore_index=True) color_palette = get_cubehelix_palette( len(data_dict), reverse=False) if len(data_dict) > 1 else [(0, 0, 0)] sns.lineplot(x=x_col, y=y_col, data=to_plot_df, hue='data name', palette=color_palette, style='data name', markers=True, dashes=False) ax = plt.gca() for i, data_name in enumerate(to_plot_df['data name'].unique()): to_plot_single_data = to_plot_df.loc[to_plot_df['data name'] == data_name] lower = to_plot_single_data[ y_col].values - to_plot_single_data['error'].values / 2 upper = to_plot_single_data[ y_col].values + to_plot_single_data['error'].values / 2 ax.fill_between(to_plot_single_data[x_col], lower, upper, color=color_palette[i], alpha=0.2) plt.xlim(right=z_max) plt.yscale('log') # handles, labels = ax.get_legend_handles_labels() prop = {'size': legend_size} if legend_size else {} ax.legend(loc='upper right', framealpha=1.0, prop=prop) # handles=handles[1:], labels=labels[1:], plt.setp(ax.get_legend().get_texts(), fontsize='9') plt.show()
def get_plan(initial_pose, goal_pose, counter): # Create a publisher to publish the initial pose init_pose_pub = rospy.Publisher( INIT_POSE_TOPIC, PoseWithCovarianceStamped, queue_size=1) # to publish init position x=2500, y=640 # Create a publisher to publish the goal pose goal_pose_pub = rospy.Publisher( GOAL_POSE_TOPIC, PoseStamped, queue_size=1) # create a publisher for goal pose map_img, map_info = utils.get_map(MAP_TOPIC) # Get and store the map PWCS = PoseWithCovarianceStamped( ) # create a PoseWithCovarianceStamped() msg PWCS.header.stamp = rospy.Time.now() # set header timestamp value PWCS.header.frame_id = "map" # set header frame id value temp_pose = utils.map_to_world(initial_pose, map_info) # init pose PWCS.pose.pose.position.x = temp_pose[0] PWCS.pose.pose.position.y = temp_pose[1] PWCS.pose.pose.position.z = 0 PWCS.pose.pose.orientation = utils.angle_to_quaternion( temp_pose[2] ) # set msg orientation to [converted to queternion] value of the yaw angle in the look ahead pose from the path print "Pubplishing Initial Pose to topic", INIT_POSE_TOPIC print "Initial ", PWCS.pose init_pose_pub.publish( PWCS ) # publish initial pose, now you can add a PoseWithCovariance with topic of "/initialpose" in rviz if counter == 0: for i in range(0, 5): init_pose_pub.publish(PWCS) rospy.sleep(0.5) print "Initial Pose Set." # raw_input("Press Enter") PS = PoseStamped() # create a PoseStamped() msg PS.header.stamp = rospy.Time.now() # set header timestamp value PS.header.frame_id = "map" # set header frame id value temp_pose = utils.map_to_world(goal_pose, map_info) # init pose PS.pose.position.x = temp_pose[ 0] # set msg x position to value of the x position in the look ahead pose from the path PS.pose.position.y = temp_pose[ 1] # set msg y position to value of the y position in the look ahead pose from the path PS.pose.position.z = 0 # set msg z position to 0 since robot is on the ground PS.pose.orientation = utils.angle_to_quaternion(temp_pose[2]) print "Pubplishing Goal Pose to topic", GOAL_POSE_TOPIC print "\nGoal ", PS.pose goal_pose_pub.publish(PS) print "Goal Pose Set" # raw_input("Press Enter") print "\nwaiting for plan ", str(counter), "\n" raw_plan = rospy.wait_for_message(PLAN_POSE_ARRAY_TOPIC, PoseArray) print "\nPLAN COMPUTED! of type:", type(raw_plan) path_part_pub = rospy.Publisher( "/CoolPlan/plan" + str(counter), PoseArray, queue_size=1) # create a publisher for plan lookahead follower for i in range(0, 4): path_part_pub.publish(raw_plan) rospy.sleep(0.4) return raw_plan
# set up waypoints and poses automatically; this does not always give # the most straightforward paths, so customized poses were used # instead #waypoints = calc_waypoint_orientation(order_waypoints(start_path, good_waypoints_path)) start_pose = np.array([[2500, 640, 6.0]]) waypoints = np.array([[2600, 660, 6.5], [2600, 450, 2.8], [1880, 440, 3.0], [1699, 450, 4.5], [1590, 670, 3.3], [1490, 570, 1.85], [1430, 490, 2.7], [1250, 460, 3.0], [1150, 460, 3.2], [950, 480, 3.6], [600, 700, 3.9], [540, 835, 4.8]]) plan = [] map_img, map_info = utils.get_map(MAP_TOPIC) map_to_world(start_pose, map_info) map_to_world(waypoints, map_info) pp = PathPlanner(plan, waypoints) # offline plan location offline_plan_path = '/home/car-user/anptaszn/src/final/offline_data/output_plan.npy' # check if offline plan already exists. if yes, do not # continue if os.path.isfile(offline_plan_path): print "offline plan found" offline_plan = np.load(offline_plan_path)
st.markdown(''' ## Situazione Zone di Rischio (Colori ) Regionale * [Dowload Dataset](https://github.com/visiont3lab/project-work-ifoa/blob/main/data/dpc-covid19-ita-regioni-zone.csv) Il dataset è stato ottenuto utilizzando il notebook [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)] (https://colab.research.google.com/github/visiont3lab/project-work-ifoa/blob/main/colab/AnalisiCovidRegioni.ipynb) e partendo dai dati delle [aree della protezione civile](https://github.com/pcm-dpc/COVID-19/tree/master/aree) ''') regione = st.selectbox("Seleziona la Regione", nomi_regioni) st.dataframe(utils.get_zone_table(regione)) map_btn = st.button("Visualizza Mappa") if map_btn: map_zone = utils.get_map() st.plotly_chart(map_zone) st.markdown(''' Obbiettivo: * Raccogliere, in modo automatico, i dati corrispondenti al colore delle regioni. * Visualizzare attraverso una mappa il cambiamento di colore delle diverse regioni italiane in data specifica. Il colore delle regione ad oggi è visibile a nel sito del ministero della satute [Classificazione Regioni e Province autonome aggiornamento all'8 marzo](http://www.salute.gov.it/portale/nuovocoronavirus/dettaglioContenutiNuovoCoronavirus.jsp?area=nuovoCoronavirus&id=5351&lingua=italiano&menu=vuoto) ''') # -------------------------------------------------------------------- # --------------------------------------------------------------------
def experiments_squad_manual_check(device, data_partition="train", print_text=False, embd_type="useqa", label_type="gold", seed=0, epoch=1): def get_training_labels(label_binarizer, query_indices, fact_indices, negative_indices): label_masks = list(set(query_indices + fact_indices + negative_indices)) label_masks_onehot = np.sum(label_binarizer.transform(label_masks), axis=0) labels = list(set(query_indices + fact_indices)) labels_onehot = np.sum(label_binarizer.transform(labels), axis=0) return labels_onehot, label_masks_onehot, np.array(labels), np.array( label_masks) def get_loss(criterion, prediction, target, mask): loss = torch.sum( criterion(prediction, target) * mask) / torch.sum(mask) return loss probe_model_root_path = "data_generated/squad/probe_experiment_2020-05-30_215643/" input_type = "query_" + embd_type + "_embd" probe_model_path = probe_model_root_path + "query_" + embd_type + "_embd_" + label_type + "_result_seed_" + str( seed) + "/best_linear_prober" saved_data_folder = 'data_generated/squad/' train_list, dev_list, kb = utils_dataset_squad.load_squad_probe_raw_data() vocab_dict, tfidf_vectorizer = utils_probe_squad.get_vocabulary( train_list, kb, saved_data_folder + "squad_vocab_dict.pickle", saved_data_folder + "squad_tfidf_vectorizer.pickle") instances_all_seeds = utils_probe_squad.get_probe_dataset( train_list, dev_list, kb, "", vocab_dict, tfidf_vectorizer, saved_data_folder, "squad_probe.pickle") linear_probe = torch.load(probe_model_path).to(device) linear_probe.eval() criterion = nn.BCELoss(reduction="none") target_vocab_size = len(vocab_dict) label_binarizer = sklearn.preprocessing.LabelBinarizer() label_binarizer.fit(range(target_vocab_size)) vocab_dict_rev = {v: k for k, v in vocab_dict.items()} query_indices = "lemma_query_indices_" + label_type fact_indices = "lemma_fact_indices_" + label_type negative_indices = "lemma_negative_indices_" + label_type data_list = instances_all_seeds[seed][data_partition] total_loss = 0 map_list = list([]) ppl_list = list([]) query_map_list = list([]) query_ppl_list = list([]) target_map_list = list([]) target_ppl_list = list([]) pred_score_dict = {} target_occur_dict = {} with torch.no_grad(): for i, instance in enumerate(data_list): labels_onehot, masks_onehot, labels, label_masks = get_training_labels( label_binarizer, instance[query_indices], instance[fact_indices], instance[negative_indices]) output_ = linear_probe( instance[input_type].to(device)) # output size is (6600) output = nn.functional.sigmoid(output_) if print_text: output_numpy = output.detach().cpu().numpy() top_preds = np.flip(np.argsort(output_numpy)) print("=" * 20) print("\tquery:", instance["lemmas_query"]) print("\tfact:", instance["lemmas_fact"]) print('\ttop pred lemma:', [vocab_dict_rev[idx] for idx in top_preds[:20]]) input("A") loss = get_loss( criterion, output, torch.tensor(labels_onehot, dtype=torch.float32).to(device), torch.tensor(masks_onehot, dtype=torch.float32).to(device)) total_loss += loss.detach().cpu().numpy() map_list.append(get_map(output.detach().cpu().numpy(), labels)) ppl_list.append(get_ppl(output.detach().cpu().numpy(), labels)) query_map_list.append( get_map(output.detach().cpu().numpy(), np.array(instance[query_indices]))) query_ppl_list.append( get_ppl(output.detach().cpu().numpy(), np.array(instance[query_indices]))) if len(set(instance[fact_indices]) - set(instance[query_indices])) > 0: target_map_list.append( get_map( output.detach().cpu().numpy(), np.array(list( set(instance[fact_indices]) - set(instance[query_indices])), dtype=np.int64))) target_ppl_list.append( get_ppl( output.detach().cpu().numpy(), np.array(list( set(instance[fact_indices]) - set(instance[query_indices])), dtype=np.int64))) for pred_lemma_indices in list( set(instance[fact_indices]) - set(instance[query_indices])): pred_lemma = vocab_dict_rev[pred_lemma_indices] if pred_lemma not in pred_score_dict: pred_score_dict[pred_lemma] = 0 target_occur_dict[pred_lemma] = 0 target_occur_dict[pred_lemma] += 1 pred_score_dict[pred_lemma] += output[pred_lemma_indices].item( ) if print_text: print("=" * 20) print("query:", instance["lemmas_query"]) print("fact", instance["lemmas_fact"]) print("negative", instance["lemmas_negative"]) print("positive token reconstructed:", [vocab_dict_rev[lemma_idx] for lemma_idx in labels]) print("negative token reconstructed:", [ vocab_dict_rev[lemma_idx] for lemma_idx in list(set(label_masks) - set(labels)) ]) print("query reconstructed", [ vocab_dict_rev[lemma_idx] for lemma_idx in instance[query_indices] ]) print("fact alone reconstructed:", [ vocab_dict_rev[lemma_idx] for lemma_idx in instance[fact_indices] ]) input("--------") result_dict = { "eval_loss": total_loss / len(dev_list), "avg map": sum(map_list) / len(map_list), "avg ppl": sum(ppl_list) / len(ppl_list), "query map:": sum(query_map_list) / len(query_map_list), "query ppl:": sum(query_ppl_list) / len(query_ppl_list), "target map:": sum(target_map_list) / len(target_map_list), "target ppl:": sum(target_ppl_list) / len(target_ppl_list) } print("-" * 20) print(result_dict) print("-" * 20) pred_freq_dict_avg = {} for k in pred_score_dict.keys(): pred_freq_dict_avg[k] = pred_score_dict[k] / target_occur_dict[k] tokens_sorted_by_occur = sorted(target_occur_dict.items(), key=lambda kv: kv[1]) for histo_tuple in list(reversed(tokens_sorted_by_occur)): print("token:", histo_tuple[0], "\tn occur:", histo_tuple[1], "\tavg prob:", pred_freq_dict_avg[histo_tuple[0]]) return 0
def __init__(self): ''' constructor ''' print('Driver constructor') self.scale = 0.05 self.offset_x = 200 # 400 self.offset_y = 200 # 400 #Initialize ros node rospy.init_node('turtlebot_driver') #Starting position: in pixels self.x_start = rospy.get_param('cur_pos_x', self.offset_x) self.y_start = rospy.get_param('cur_pos_y', self.offset_y) self.theta_start = rospy.get_param('cur_pos_theta', 0) #Initialize goals self.x = np.array([]) self.y = np.array([]) self.theta = np.array([]) #Threshold for distance to goal self.goal_th_xy = rospy.get_param('goal_thershold_xy',0.1) #Position threshold self.goal_th_ang = rospy.get_param('goal_threshold_ang',0.001) #Orientation threshold #Point to the first goal self.active_goal = 0 #Initialize number of goals self.num_goals = 1 #Has the goal been loaded? self.params_loaded = False # For rviz plotting lines self.tfBroad = tf.TransformBroadcaster() self.i = 0 #Some parameters for LaserScan information self.sensor_spacing = 5 self.sensor_max_range = 10 # Define publisher self.pub = rospy.Publisher("/cmd_vel_mux/input/teleop", Twist, queue_size=100) self.pub_line = rospy.Publisher("lines", Marker,queue_size=0) self.pub_map = rospy.Publisher("linesekf", Marker, queue_size=2) self.pub_traj = rospy.Publisher("trajectory", Marker, queue_size=2) # Define subscriber ################################################################## self.sub_sensor = rospy.Subscriber("/scan", LaserScan, self.scan_callback) self.sub_odom = rospy.Subscriber("odom", Odometry, self.odom_callback) #define the velocity message self.vmsg = Twist() # Initialize robot's position (in meters) self.position_x = rospy.get_param('cur_pos_world_x', 0) self.position_y = rospy.get_param('cur_pos_world_y', 0) self.position_theta = rospy.get_param('cur_pos_world_theta', 0) #Controller parameters self.kp_v = 0.1 self.kp_w = 1.5 self.kd_v = 0 self.kd_w = 0 # self.ki_w = 0.001 #Controller variables self.d_prev = 0 self.dt_prev = 0 # Obstacle avoidance factors self.oa_v = 1 self.oa_w = 0 #self.counter = 0 # Publish map lines self.map = utils.get_map() # get_map_udg() self.trajectory = np.zeros((0, 4))