def restrict_by_people_pic(hpic_tbl, gpic_tbl, hosts, listings, reviews, guests):
# Reduced Hosts
new_host_ids = set(utils.convert_to_str(hpic_tbl[hpic_tbl['num_of_people_in_pic'] == 1]['id'].unique()))
# Reduced Guests
new_guest_ids = set(utils.convert_to_str(gpic_tbl[gpic_tbl['num_of_people'] == 1]['id'].unique()))
# Reduced Reviews
new_reviews = reviews[(reviews['recipient_id'].isin(new_host_ids)) & (reviews['reviewer_id'].isin(new_guest_ids))]
print("Revised number of Reviews: %d (decreased %.2f %%)" % (len(new_reviews), utils.get_decreased_percent(new_reviews, reviews)))
# Reduced Hosts
new_hosts = hosts[hosts['id'].isin(set(new_reviews['recipient_id']))]
print("Revised number of Hosts: %d (decreased %.2f %%)" % (len(new_hosts), utils.get_decreased_percent(new_hosts, hosts)))
# Reduced Listings
new_listings = listings[listings['id'].isin(set(new_reviews['listing_id']))]
print("Revised number of Listings: %d (decreased %.2f %%)" % (len(new_listings), utils.get_decreased_percent(new_listings, listings)))
# Reduced Overall Guests
overall_guests = reviews['reviewer_id'].unique()
new_overall_guests = new_reviews['reviewer_id'].unique()
print("Revised number of Overall Guests: %d (decreased %.2f %%)" % (len(new_overall_guests), utils.get_decreased_percent(new_overall_guests, overall_guests)))
# Reduced Retrieved Guests
new_retrieved_guests = guests[guests['id'].isin(set(utils.convert_to_str(new_overall_guests)))]
print("Revised number of Retrieved Guests: %d (decreased %.2f %%)" % (len(new_retrieved_guests), utils.get_decreased_percent(new_retrieved_guests, guests)))
return (new_hosts, new_listings, new_reviews, new_overall_guests, new_retrieved_guests)
def get_item_info(this_item):
pref_label = convert_to_str(this_item["item_pref_label"])
description = convert_to_str(this_item["item_description"])
item_name = set_item_name(this_item)
question = convert_to_str(this_item["question"])
question = question.replace("\n", "")
field_type = convert_to_str(this_item["field_type"])
if field_type == "integer":
field_type = "int"
choices = convert_to_str(this_item["choices"])
if type(choices) == str:
choices = choices.split(" | ")
visibility = get_visibility(this_item)
mandatory = get_mandatory(this_item)
return {
"name": item_name,
"pref_label": pref_label,
"description": description,
"question": question,
"field_type": field_type,
"choices": choices,
"visibility": visibility,
"mandatory": mandatory,
}
def eval_bleu(
train_loader: d.BatchedIterator,
valid_loader: d.BatchedIterator,
model: nn.Module,
en_vocab: Vocabulary,
fr_vocab: Vocabulary,
device: str,
multi_gpu: bool,
) -> None:
model = model.to(device)
if multi_gpu and device == 'cuda':
print('Using multi gpu training')
model = torch.nn.DataParallel(model, device_ids=[0, 1]).cuda()
for e in range(1):
count = 0
with tqdm(train_loader, total=len(train_loader)) as pbar:
for i, data in enumerate(pbar):
if i == 0:
continue
src, trg, src_lengths, trg_lengths, prev_tokens, prev_lengths = data
src = src.to(device).long()
trg = trg.to(device).long()
src_lengths = src_lengths.to(device).long()
trg_lengths = trg_lengths.to(device)
predicted = model.generate_max(src, src_lengths, 100, device)
# predicted = model.generate_beam(src, src_lengths, 100, 5, device)
output = ' '.join(
utils.convert_to_str(predicted.cpu().numpy(), fr_vocab)[0])
actual_out = ' '.join(
utils.convert_to_str(trg.cpu().numpy(), fr_vocab)[0])
src = ' '.join(
utils.convert_to_str(src.cpu().numpy(), en_vocab)[0])
print('src\n', src)
print('')
print('out\n', output)
print('')
print('trg\n', actual_out)
if (i >= 2):
return
count += 1
# pbar.set_postfix(
# loss_avg=total_loss/(count),
# epoch="{}/{}".format(e + 1, epochs),
# curr_loss=loss.item(),
# nan_count=nan_count,
# )
pbar.refresh()
train_loader.reset()
valid_loader.reset()
def restrict_number_of_reviews(hosts, listings, reviews, guests):
# Reduced Hosts
host_review_count = reviews.groupby('recipient_id').count()[['id']].rename(columns={'id': 'num_of_reviews'})
new_host_ids = set([str(i) for i in host_review_count[host_review_count['num_of_reviews'] >4].index])
# Reduced Hosts
new_hosts = hosts[hosts['id'].isin(new_host_ids)]
print("Revised number of Hosts: %d (decreased %.2f %%)" % (len(new_hosts), utils.get_decreased_percent(new_hosts, hosts)))
# Reduced Listings
new_listings = listings[listings['host_id'].isin(new_host_ids)]
print("Revised number of Listings: %d (decreased %.2f %%)" % (len(new_listings), utils.get_decreased_percent(new_listings, listings)))
# Reduced Reviews
new_reviews = reviews[reviews['recipient_id'].isin(new_host_ids)]
print("Revised number of Reviews: %d (decreased %.2f %%)" % (len(new_reviews), utils.get_decreased_percent(new_reviews, reviews)))
# Reduced Overall Guests
overall_guests = reviews['reviewer_id'].unique()
new_overall_guests = new_reviews['reviewer_id'].unique()
print("Revised number of Overall Guests: %d (decreased %.2f %%)" % (len(new_overall_guests), utils.get_decreased_percent(new_overall_guests, overall_guests)))
# Reduced Retrieved Guests
new_retrieved_guests = guests[guests['id'].isin(set(utils.convert_to_str(new_overall_guests)))]
print("Revised number of Retrieved Guests: %d (decreased %.2f %%)" % (len(new_retrieved_guests), utils.get_decreased_percent(new_retrieved_guests, guests)))
return (new_hosts, new_listings, new_reviews, new_overall_guests, new_retrieved_guests)
def get_visibility(this_item):
visibility = convert_to_str(this_item["visibility"])
if visibility in ["1", 1]:
visibility = True
elif visibility in ["0", 0]:
visibility = False
# TODO
# help with javascript expression input and validation
return visibility
def _generate_id(cls, exp_id, exp_version, state_name):
"""Generates a unique ID for the Classifier Exploration Mapping of the
form {{exp_id}}.{{exp_version}}.{{utf8_encoded_state_name}}
Args:
exp_id: str. ID of the exploration.
exp_version: int. The exploration version at the time
this training job was created.
state_name: unicode. The name of the state to which the classifier
belongs.
Returns:
str. ID of the new Classifier Exploration Mapping instance.
"""
new_id = '%s.%s.%s' % (exp_id, exp_version, state_name)
return utils.convert_to_str(new_id)
def read_from_meter(self, meter_id, base_register, block_size,
params_indices):
"""Reads data from meter correpsonding to the param indices
specified
Parameters
-----------
meter_id : ID set on the meter (eg. 1, 2), int
base_register : Base register for block of registers to read, int
block_size : Number of register bytes in this block, int
params_indices : List of indices relative to base_register, list
Returns
-------
data: Comma separated values correpsonding to parameters whose indices
were specified
"""
try:
binary_data = self.client.read_holding_registers(base_register,
block_size,
unit=meter_id)
except Exception as e:
# Sleep for some time and again try to connect
time.sleep(0.5)
self.logger.exception(e)
self.logger.info('Will now try to reconnect')
self.client = self.connect(vendor=self.vendor,
product=self.product)
binary_data = self.client.read_holding_registers(base_register,
block_size,
unit=meter_id)
data = ""
for i in range(0, (block_size - 1), 2):
for j in params_indices:
if (j == i):
data = data + "," + convert_to_str(
(binary_data.registers[i + 1] << 16) +
binary_data.registers[i])
data = data[:-1] + "\n"
data = str(time.time()) + data
return data
def set_item_name(this_item):
if "item" not in this_item.keys():
item_name = convert_to_str(this_item["item_pref_label"])
elif isinstance(convert_to_str(this_item["item"]), float):
item_name = convert_to_str(this_item["item_pref_label"])
elif convert_to_str(this_item["item"]) == "":
item_name = convert_to_str(this_item["item_pref_label"])
else:
item_name = convert_to_str(this_item["item"])
item_name = snake_case(item_name)
return item_name
def read_from_meter(self, meter_id, base_register, block_size,
params_indices):
"""Reads data from meter correpsonding to the param indices
specified
Parameters
-----------
meter_id : ID set on the meter (eg. 1, 2), int
base_register : Base register for block of registers to read, int
block_size : Number of register bytes in this block, int
params_indices : List of indices relative to base_register, list
Returns
-------
data: Comma separated values correpsonding to parameters whose indices
were specified
"""
try:
binary_data = self.client.read_holding_registers(
base_register, block_size, unit=meter_id)
except Exception as e:
# Sleep for some time and again try to connect
time.sleep(0.5)
self.logger.exception(e)
self.logger.info('Will now try to reconnect')
self.client = self.connect(
vendor=self.vendor, product=self.product)
binary_data = self.client.read_holding_registers(
base_register, block_size, unit=meter_id)
data = ""
for i in range(0, (block_size - 1), 2):
for j in params_indices:
if(j == i):
data = data + "," + convert_to_str(
(binary_data.registers[i + 1] << 16) + binary_data.registers[i])
data = data[:-1] + "\n"
data = str(time.time()) + data
return data
def test_convert_to_str(self):
string1 = 'Home'
string2 = u'Лорем'
self.assertEqual(utils.convert_to_str(string1), string1)
self.assertEqual(utils.convert_to_str(string2),
string2.encode(encoding='utf-8'))
