def predict(self,passage :str,question :str):
example = input_to_squad_example(passage,question)
features = squad_examples_to_features(example,self.tokenizer,self.max_seq_length,self.doc_stride,self.max_query_length)
all_input_ids = torch.tensor([f.input_ids for f in features], dtype=torch.long)
all_input_mask = torch.tensor([f.input_mask for f in features], dtype=torch.long)
all_segment_ids = torch.tensor([f.segment_ids for f in features], dtype=torch.long)
all_example_index = torch.arange(all_input_ids.size(0), dtype=torch.long)
dataset = TensorDataset(all_input_ids, all_input_mask, all_segment_ids,
all_example_index)
eval_sampler = SequentialSampler(dataset)
eval_dataloader = DataLoader(dataset, sampler=eval_sampler, batch_size=1)
all_results = []
for batch in eval_dataloader:
batch = tuple(t.to(self.device) for t in batch)
with torch.no_grad():
inputs = {'input_ids': batch[0],
'attention_mask': batch[1],
'token_type_ids': batch[2]
}
example_indices = batch[3]
outputs = self.model(**inputs)
for i, example_index in enumerate(example_indices):
eval_feature = features[example_index.item()]
unique_id = int(eval_feature.unique_id)
result = RawResult(unique_id = unique_id,
start_logits = to_list(outputs[0][i]),
end_logits = to_list(outputs[1][i]))
all_results.append(result)
answer = get_answer(example,features,all_results,self.n_best_size,self.max_answer_length,self.do_lower_case)
return answer
def parse_system_file_ext(file):
data_list = []
with open(file, 'r') as f:
header = f.readline()
header_list = utils.to_list(header)
while True:
line = f.readline()
if not line:
# print("Reach {} EOF".format(f))
break
info = utils.to_list(line)
# info = line.split(',')
# error check
if len(header_list) != len(info):
print(line, len(header_list), len(info))
for i in range(0, min(len(info), len(header_list))):
print(file, header_list[i], info[i])
exit(0)
info_dict = {}
for i in range(0, len(header_list)):
info_dict[header_list[i]] = info[i]
data_list.append(info_dict)
res = {'header': header_list, 'data': data_list}
# print(file, len(data_list))
return res
def __init__(self, experiment, target_disease):
super(PhyloDAP, self).__init__(experiment=experiment)
self._disease_name = target_disease
self.type_data = settings.TYPE_DATA
self.total_nb_samples = settings.NB_SAMPLES
self.nb_filters = to_list(settings.nb_convolutional_filters)
self.phylo_neighbours = to_list(settings.nb_phylo_neighbours)
self._do_serialisation = False # Serialization does not work with our Keras layer
def __init__(self, experiment, target_disease):
super(PhyloDAP, self).__init__(experiment=experiment)
self._disease_name = target_disease
self.type_data = settings.TYPE_DATA
self.total_nb_samples = settings.NB_SAMPLES
self.nb_filters = to_list(settings.nb_convolutional_filters)
self.phylo_neighbours = to_list(settings.nb_phylo_neighbours)
self._path_weights = os.path.join(os.path.abspath(os.path.curdir), 'weights',
'{}_training_weights.hdf5'.format(self._disease_name.lower()))
self._do_serialisation = False # Serialization does not work with our Keras layer
def __init__(self, scenarios, b, c, h, f=None, m=None, durability=None):
self.root = defaultdict(ScenarioTree.Node)
self.model = Model()
self.variables = {}
self.obj_sum = []
self.t = len(scenarios[0][0])
self.c = utils.to_list(c, self.t)
self.b = utils.to_list(b, self.t)
self.h = utils.to_list(h, self.t)
self.f = None if f is None else utils.to_list(f, self.t)
self.M = self.find_m(scenarios, durability) if m is None else m
self.durability = durability
self.start_build_tree(scenarios)
self.start_build_model()
def build_scenario_structs(scenarios, b, c, h, f, durability):
b = utils.to_list(b, len(scenarios[0][0]))
c = utils.to_list(c, len(scenarios[0][0]))
h = utils.to_list(h, len(scenarios[0][0]))
f = None if f is None else utils.to_list(f, len(scenarios[0][0]))
scenario_structs = []
for scenario, prob in scenarios:
scenario_structs.append(
ScenarioStruct(scenario=scenario,
prob=prob,
b=b,
c=c,
h=h,
f=f,
durability=durability))
return scenario_structs
def single_rel_sampler(self, batch_id, relation_id, link_info, pos_neg=1, remove_false_neg=True):
'''
it is okay not to remove the "false negative", for it is not really a large portion
'''
batch_size = self.batch_sizes[pos_neg][relation_id]
if pos_neg:
# pos_neg is 1, positibe sampling
positive_samples = self.link_info[relation_id][:2, batch_id*batch_size : (batch_id+1)*batch_size]
samples_from = positive_samples[0, :]
samples_to = positive_samples[1, :]
else:
# pos_neg is 0, negative sampling
known_set = self.known_positive_links[relation_id]
samples_from = np.random.choice(self.n_entities, batch_size)
samples_to = np.random.choice(self.n_entities, batch_size)
if remove_false_neg:
# get the pairs as set and remove "false negatives" (those that are positive)
raw_negative_pair_set = pair_set(list(samples_from), list(samples_to))
filtered_negative_pairs = list(raw_negative_pair_set - known_set - self.self_loop_links)
# extract the from and to infomation
samples_from = from_list(filtered_negative_pairs)
samples_to = to_list(filtered_negative_pairs)
i_label = 0 if self.separate_relations else relation_id
samples_relation = [i_label] * len(samples_from)
return samples_from, samples_relation, samples_to
def create_tokenizer():
"""
根据训练数据集中图像名，和其它对应的标题，生成一个tokenizer
:return: 生成的tokenizer
"""
train_image_names = utils.load_image_nmaes('Flickr_8k.trainTmages.txt')
train_descriptions = utils.load_clean_captions('descriptions.txt',
train_image_names)
lines = utils.to_list(train_descriptions)
def evaluate(real_scenario, xts, goods_info):
t = len(real_scenario)
b = utils.to_list(goods_info['b'], t)
c = utils.to_list(goods_info['c'], t)
h = utils.to_list(goods_info['h'], t)
f = utils.to_list(goods_info['f'],
t) if goods_info['f'] is not None else None
durability = goods_info['durability']
yt = 0
fixed_cost = 0
ordering_cost = np.sum(np.array(c) * (np.array(xts)))
holding_cost = 0
shortage_cost = 0
leftovers = np.zeros(durability)
if f is not None:
fixed_cost = np.sum(np.array(f) * (np.array(xts) > 0))
for i in range(0, t):
xt = xts[i]
dt = real_scenario[i]
holding_cost += h[i] * max(0, (xt + yt - dt))
shortage_cost += b[i] * max(0, (dt - xt - yt))
if durability is not None and i >= durability:
yt = max(xt + yt - dt - leftovers[-1], 0)
else:
yt = max(xt + yt - dt, 0)
if durability is not None:
lt = dt
leftovers = np.roll(leftovers, 1)
for i in range(durability - 1, 0, -1):
if lt == 0:
break
dec = min(lt, leftovers[i])
leftovers[i] -= dec
lt -= dec
leftovers[0] = 0
leftovers[0] = min(xt, max(yt - np.sum(leftovers), 0))
yt = max(0, np.sum(leftovers))
cost = fixed_cost + ordering_cost + holding_cost + shortage_cost
return cost, fixed_cost, ordering_cost, holding_cost, shortage_cost
def __init__(self, state, num_steps, **kwargs):
state_list = utils.to_list(state)
#print state_list
self.num_steps = num_steps
self.range_step = range(num_steps)
self.num_states = len(state_list)
self.range_state = range(self.num_states)
self.current_state = [deque(maxlen=num_steps) for _ in self.range_state]
self.old_state = [deque(maxlen=num_steps) for _ in self.range_state]
for _ in self.range_step:
for qidx in self.range_state:
self.current_state[qidx].append(state_list[qidx])
self.old_state[qidx].append(state_list[qidx])
#self.current_state.append(state_list)
#print self.current_state
#self.num_states = len(current_state[-1]) if utils.multi_state(state) else 1
self.num_states = len(self.current_state[-1])
self.state_dims = [len(s.shape) for s in state_list]
def __init__(self, state, **kwargs):
self.current_state = utils.to_list(state)
self.old_state = utils.to_list(state)
def get_countries_by_ids(countries: Countries, country_ids: str) -> pd.DataFrame:
ids = utils.to_list(country_ids)
return countries.last_day.iloc[ids] # last_df[last_day.index.isin(df_def_countries['Country'])]
def ward(ward_number):
headers = mongo.db.streetlights.aggregate(utils.ward_pipeline(ward_number))
streetlights = mongo.db.streetlights.find({"ward": ward_number})
return render_template('index.html', tittle = "Ward", headers = headers["result"][0], streetlights = utils.to_list(streetlights))
def dump_json(path_feature_map, filepath):
path_feature_map = to_list(path_feature_map)
json.dump(path_feature_map, open(filepath, 'w'))
def evaluate(args, model, tokenizer, prefix=""):
dataset, examples, features = load_and_cache_examples(args,
tokenizer,
evaluate=True,
output_examples=True)
if not os.path.exists(args.output_dir):
os.makedirs(args.output_dir)
args.eval_batch_size = args.per_gpu_eval_batch_size * max(1, args.n_gpu)
eval_sampler = SequentialSampler(dataset)
eval_dataloader = DataLoader(dataset,
sampler=eval_sampler,
batch_size=args.eval_batch_size)
# multi-gpu evaluate
if args.n_gpu > 1 and not isinstance(model, torch.nn.DataParallel):
model = torch.nn.DataParallel(model)
# Eval!
logger.info("***** Running evaluation {} *****".format(prefix))
logger.info(" Num examples = %d", len(dataset))
logger.info(" Batch size = %d", args.eval_batch_size)
all_results = []
start_time = timeit.default_timer()
for batch in tqdm(eval_dataloader, desc="Evaluating"):
model.eval()
batch = tuple(t.to(args.device) for t in batch)
with torch.no_grad():
inputs = {
"input_ids": batch[0],
"attention_mask": batch[1],
"token_type_ids": batch[2],
}
example_indices = batch[3]
outputs = model(**inputs)
for i, example_index in enumerate(example_indices):
eval_feature = features[example_index.item()]
unique_id = int(eval_feature.unique_id)
# [A] bug fix
output = [
to_list(output[i]) for output in (outputs[0], outputs[1])
]
start_logits, end_logits = output
result = SquadResult(unique_id, start_logits, end_logits)
all_results.append(result)
evalTime = timeit.default_timer() - start_time
logger.info(" Evaluation done in total %f secs (%f sec per example)",
evalTime, evalTime / len(dataset))
# Compute predictions
output_prediction_file = os.path.join(args.output_dir,
"predictions_{}.json".format(prefix))
output_nbest_file = os.path.join(
args.output_dir, "nbest_predictions_{}.json".format(prefix))
if args.version_2_with_negative:
output_null_log_odds_file = os.path.join(
args.output_dir, "null_odds_{}.json".format(prefix))
else:
output_null_log_odds_file = None
predictions = compute_predictions_logits(
examples,
features,
all_results,
args.n_best_size,
args.max_answer_length,
args.do_lower_case,
output_prediction_file,
output_nbest_file,
output_null_log_odds_file,
args.verbose_logging,
args.version_2_with_negative,
args.null_score_diff_threshold,
tokenizer,
)
# Compute the F1 and exact scores.
results = squad_evaluate(examples, predictions)
return results
def update(self, state):
state_list = utils.to_list(state)
for qidx, s in enumerate(state_list):
self.update_deque(qidx, state_list)
def district(district_name):
headers = mongo.db.streetlights.aggregate(utils.district_pipeline(district_name))
streetlights = mongo.db.streetlights.find({"district": district_name})
print headers
return render_template('index.html', title = "District" , headers = headers["result"][0], streetlights = utils.to_list(streetlights))
def predict(self, passages: list, question: str):
examples = input_to_squad_example(passages, question)
features = squad_examples_to_features(examples,
self.tokenizer,
self.max_seq_length,
self.doc_stride,
self.max_query_length,
vsl=self.vsl)
if not self.use_tf and not self.use_onnx_runtime:
torch_input_ids = torch.tensor([f.input_ids for f in features],
dtype=torch.long).to(self.device)
torch_input_mask = torch.tensor([f.input_mask for f in features],
dtype=torch.long).to(self.device)
torch_segment_ids = torch.tensor([f.segment_ids for f in features],
dtype=torch.long).to(self.device)
torch_example_index = torch.arange(torch_input_ids.size(0),
dtype=torch.long).to(
self.device)
all_results = []
if self.use_tf:
if self.tf_onnx:
# this is the TF graph converted from ONNX
inputs = {
'input_ids:0': [f.input_ids for f in features],
'attention_mask:0': [f.input_mask for f in features],
'token_type_ids:0': [f.segment_ids for f in features]
}
start_logits, end_logits = self.model.run(
['Squeeze_49:0', 'Squeeze_50:0'], feed_dict=inputs)
else:
# this is the original TF graph
inputs = {
'input_ids:0': [f.input_ids for f in features],
'input_mask:0': [f.input_mask for f in features],
'segment_ids:0': [f.segment_ids for f in features]
}
start_logits, end_logits = self.model.run(
['start_logits:0', 'end_logits:0'], feed_dict=inputs)
example_indices = np.arange(len(features))
outputs = [start_logits, end_logits]
elif self.use_onnx_runtime:
inputs = {
self.model.get_inputs()[0].name:
np.array([f.input_ids for f in features]),
self.model.get_inputs()[1].name:
np.array([f.input_mask for f in features]),
self.model.get_inputs()[2].name:
np.array([f.segment_ids for f in features])
}
output_names = [
self.model.get_outputs()[0].name,
self.model.get_outputs()[1].name
]
example_indices = np.arange(len(features))
outputs = self.model.run(output_names, inputs)
else:
example_indices = torch_example_index
if self.use_jit:
outputs = self.model(torch_input_ids, torch_input_mask,
torch_segment_ids)
else:
with torch.no_grad():
inputs = {
'input_ids': torch_input_ids,
'attention_mask': torch_input_mask,
'token_type_ids': torch_segment_ids
}
outputs = self.model(**inputs)
for i, example_index in enumerate(example_indices):
eval_feature = features[example_index.item()]
unique_id = int(eval_feature.unique_id)
result = RawResult(unique_id=unique_id,
start_logits=to_list(outputs[0][i]),
end_logits=to_list(outputs[1][i]))
all_results.append(result)
answers = get_answer(examples, features, all_results, self.n_best_size,
self.max_answer_length, self.do_lower_case)
return answers
from Khorus.Choori.decorators import privileges, retrieve
from Khorus.crud.order import bp, orders
from Khorus.crud.order.design.schema import schema, obj, status as statuses
from sanic.response import json
from bson import ObjectId
from datetime import datetime, timedelta
from temp import orders as orders_cache, users as users_cache
from utils import to_list, dot_notation
from copy import deepcopy
retrieval = to_list(schema)
minus = [(feature[1][0][1:], feature[1][1], feature[0])
for feature in retrieval if '-' in feature[1][0]]
retrieval = [(feature[1][0][1:], feature[1][1], feature[0])
for feature in retrieval if '$' in feature[1][0]]
@bp.route('/', methods=['POST'])
@bp.route('/<_id>', methods=['POST'])
@privileges('dev', 'operator', 'applicator')
@retrieve(*[
'<{key}:{type}:$form:k>'.format(key=feature[0], type=feature[1])
for feature in retrieval
])
async def _put(request, payload, _id=None, **kwargs):
options = []
d = deepcopy(obj)
for pay, _type, address in minus:
_d, key, flag = dot_notation(d, address)
_d[key] = payload[pay]
def update(self, state):
self.old_state = self.current_state
self.current_state = utils.to_list(state)
