def restrict_by_people_pic(hpic_tbl, gpic_tbl, hosts, listings, reviews, guests): # Reduced Hosts new_host_ids = set(utils.convert_to_str(hpic_tbl[hpic_tbl['num_of_people_in_pic'] == 1]['id'].unique())) # Reduced Guests new_guest_ids = set(utils.convert_to_str(gpic_tbl[gpic_tbl['num_of_people'] == 1]['id'].unique())) # Reduced Reviews new_reviews = reviews[(reviews['recipient_id'].isin(new_host_ids)) & (reviews['reviewer_id'].isin(new_guest_ids))] print("Revised number of Reviews: %d (decreased %.2f %%)" % (len(new_reviews), utils.get_decreased_percent(new_reviews, reviews))) # Reduced Hosts new_hosts = hosts[hosts['id'].isin(set(new_reviews['recipient_id']))] print("Revised number of Hosts: %d (decreased %.2f %%)" % (len(new_hosts), utils.get_decreased_percent(new_hosts, hosts))) # Reduced Listings new_listings = listings[listings['id'].isin(set(new_reviews['listing_id']))] print("Revised number of Listings: %d (decreased %.2f %%)" % (len(new_listings), utils.get_decreased_percent(new_listings, listings))) # Reduced Overall Guests overall_guests = reviews['reviewer_id'].unique() new_overall_guests = new_reviews['reviewer_id'].unique() print("Revised number of Overall Guests: %d (decreased %.2f %%)" % (len(new_overall_guests), utils.get_decreased_percent(new_overall_guests, overall_guests))) # Reduced Retrieved Guests new_retrieved_guests = guests[guests['id'].isin(set(utils.convert_to_str(new_overall_guests)))] print("Revised number of Retrieved Guests: %d (decreased %.2f %%)" % (len(new_retrieved_guests), utils.get_decreased_percent(new_retrieved_guests, guests))) return (new_hosts, new_listings, new_reviews, new_overall_guests, new_retrieved_guests)
def get_item_info(this_item): pref_label = convert_to_str(this_item["item_pref_label"]) description = convert_to_str(this_item["item_description"]) item_name = set_item_name(this_item) question = convert_to_str(this_item["question"]) question = question.replace("\n", "") field_type = convert_to_str(this_item["field_type"]) if field_type == "integer": field_type = "int" choices = convert_to_str(this_item["choices"]) if type(choices) == str: choices = choices.split(" | ") visibility = get_visibility(this_item) mandatory = get_mandatory(this_item) return { "name": item_name, "pref_label": pref_label, "description": description, "question": question, "field_type": field_type, "choices": choices, "visibility": visibility, "mandatory": mandatory, }
def eval_bleu( train_loader: d.BatchedIterator, valid_loader: d.BatchedIterator, model: nn.Module, en_vocab: Vocabulary, fr_vocab: Vocabulary, device: str, multi_gpu: bool, ) -> None: model = model.to(device) if multi_gpu and device == 'cuda': print('Using multi gpu training') model = torch.nn.DataParallel(model, device_ids=[0, 1]).cuda() for e in range(1): count = 0 with tqdm(train_loader, total=len(train_loader)) as pbar: for i, data in enumerate(pbar): if i == 0: continue src, trg, src_lengths, trg_lengths, prev_tokens, prev_lengths = data src = src.to(device).long() trg = trg.to(device).long() src_lengths = src_lengths.to(device).long() trg_lengths = trg_lengths.to(device) predicted = model.generate_max(src, src_lengths, 100, device) # predicted = model.generate_beam(src, src_lengths, 100, 5, device) output = ' '.join( utils.convert_to_str(predicted.cpu().numpy(), fr_vocab)[0]) actual_out = ' '.join( utils.convert_to_str(trg.cpu().numpy(), fr_vocab)[0]) src = ' '.join( utils.convert_to_str(src.cpu().numpy(), en_vocab)[0]) print('src\n', src) print('') print('out\n', output) print('') print('trg\n', actual_out) if (i >= 2): return count += 1 # pbar.set_postfix( # loss_avg=total_loss/(count), # epoch="{}/{}".format(e + 1, epochs), # curr_loss=loss.item(), # nan_count=nan_count, # ) pbar.refresh() train_loader.reset() valid_loader.reset()
def restrict_number_of_reviews(hosts, listings, reviews, guests): # Reduced Hosts host_review_count = reviews.groupby('recipient_id').count()[['id']].rename(columns={'id': 'num_of_reviews'}) new_host_ids = set([str(i) for i in host_review_count[host_review_count['num_of_reviews'] >4].index]) # Reduced Hosts new_hosts = hosts[hosts['id'].isin(new_host_ids)] print("Revised number of Hosts: %d (decreased %.2f %%)" % (len(new_hosts), utils.get_decreased_percent(new_hosts, hosts))) # Reduced Listings new_listings = listings[listings['host_id'].isin(new_host_ids)] print("Revised number of Listings: %d (decreased %.2f %%)" % (len(new_listings), utils.get_decreased_percent(new_listings, listings))) # Reduced Reviews new_reviews = reviews[reviews['recipient_id'].isin(new_host_ids)] print("Revised number of Reviews: %d (decreased %.2f %%)" % (len(new_reviews), utils.get_decreased_percent(new_reviews, reviews))) # Reduced Overall Guests overall_guests = reviews['reviewer_id'].unique() new_overall_guests = new_reviews['reviewer_id'].unique() print("Revised number of Overall Guests: %d (decreased %.2f %%)" % (len(new_overall_guests), utils.get_decreased_percent(new_overall_guests, overall_guests))) # Reduced Retrieved Guests new_retrieved_guests = guests[guests['id'].isin(set(utils.convert_to_str(new_overall_guests)))] print("Revised number of Retrieved Guests: %d (decreased %.2f %%)" % (len(new_retrieved_guests), utils.get_decreased_percent(new_retrieved_guests, guests))) return (new_hosts, new_listings, new_reviews, new_overall_guests, new_retrieved_guests)
def get_visibility(this_item): visibility = convert_to_str(this_item["visibility"]) if visibility in ["1", 1]: visibility = True elif visibility in ["0", 0]: visibility = False # TODO # help with javascript expression input and validation return visibility
def _generate_id(cls, exp_id, exp_version, state_name): """Generates a unique ID for the Classifier Exploration Mapping of the form {{exp_id}}.{{exp_version}}.{{utf8_encoded_state_name}} Args: exp_id: str. ID of the exploration. exp_version: int. The exploration version at the time this training job was created. state_name: unicode. The name of the state to which the classifier belongs. Returns: str. ID of the new Classifier Exploration Mapping instance. """ new_id = '%s.%s.%s' % (exp_id, exp_version, state_name) return utils.convert_to_str(new_id)
def read_from_meter(self, meter_id, base_register, block_size, params_indices): """Reads data from meter correpsonding to the param indices specified Parameters ----------- meter_id : ID set on the meter (eg. 1, 2), int base_register : Base register for block of registers to read, int block_size : Number of register bytes in this block, int params_indices : List of indices relative to base_register, list Returns ------- data: Comma separated values correpsonding to parameters whose indices were specified """ try: binary_data = self.client.read_holding_registers(base_register, block_size, unit=meter_id) except Exception as e: # Sleep for some time and again try to connect time.sleep(0.5) self.logger.exception(e) self.logger.info('Will now try to reconnect') self.client = self.connect(vendor=self.vendor, product=self.product) binary_data = self.client.read_holding_registers(base_register, block_size, unit=meter_id) data = "" for i in range(0, (block_size - 1), 2): for j in params_indices: if (j == i): data = data + "," + convert_to_str( (binary_data.registers[i + 1] << 16) + binary_data.registers[i]) data = data[:-1] + "\n" data = str(time.time()) + data return data
def set_item_name(this_item): if "item" not in this_item.keys(): item_name = convert_to_str(this_item["item_pref_label"]) elif isinstance(convert_to_str(this_item["item"]), float): item_name = convert_to_str(this_item["item_pref_label"]) elif convert_to_str(this_item["item"]) == "": item_name = convert_to_str(this_item["item_pref_label"]) else: item_name = convert_to_str(this_item["item"]) item_name = snake_case(item_name) return item_name
def read_from_meter(self, meter_id, base_register, block_size, params_indices): """Reads data from meter correpsonding to the param indices specified Parameters ----------- meter_id : ID set on the meter (eg. 1, 2), int base_register : Base register for block of registers to read, int block_size : Number of register bytes in this block, int params_indices : List of indices relative to base_register, list Returns ------- data: Comma separated values correpsonding to parameters whose indices were specified """ try: binary_data = self.client.read_holding_registers( base_register, block_size, unit=meter_id) except Exception as e: # Sleep for some time and again try to connect time.sleep(0.5) self.logger.exception(e) self.logger.info('Will now try to reconnect') self.client = self.connect( vendor=self.vendor, product=self.product) binary_data = self.client.read_holding_registers( base_register, block_size, unit=meter_id) data = "" for i in range(0, (block_size - 1), 2): for j in params_indices: if(j == i): data = data + "," + convert_to_str( (binary_data.registers[i + 1] << 16) + binary_data.registers[i]) data = data[:-1] + "\n" data = str(time.time()) + data return data
def test_convert_to_str(self): string1 = 'Home' string2 = u'Лорем' self.assertEqual(utils.convert_to_str(string1), string1) self.assertEqual(utils.convert_to_str(string2), string2.encode(encoding='utf-8'))