def main():
if opt.truncate:
train_data = pickle.load(
open('../datasets/' + opt.dataset + '/train_shortonly.txt', 'rb'))
else:
train_data = pickle.load(
open('../datasets/' + opt.dataset + '/train.txt', 'rb'))
if opt.validation:
train_data, valid_data = split_validation(train_data,
opt.valid_portion)
test_data = valid_data
else:
if opt.truncate:
test_data = pickle.load(
open('../datasets/' + opt.dataset + '/test_shortonly.txt',
'rb'))
else:
test_data = pickle.load(
open('../datasets/' + opt.dataset + '/test.txt', 'rb'))
# all_train_seq = pickle.load(open('../datasets/' + opt.dataset + '/all_train_seq.txt', 'rb'))
# g = build_graph(all_train_seq)
train_data = Data(opt, train_data, shuffle=True)
test_data = Data(opt, test_data, shuffle=False)
# del all_train_seq, g
if opt.dataset == 'diginetica':
n_node = 43098
elif opt.dataset == 'yoochoose1_64' or opt.dataset == 'yoochoose1_4':
n_node = 37484
else:
n_node = 310
model = trans_to_cuda(SessionGraph(opt, n_node))
model = torch.nn.DataParallel(model, device_ids=[0, 1])
start = time.time()
best_result = [0, 0]
best_epoch = [0, 0]
bad_counter = 0
for epoch in range(opt.epoch):
print('-------------------------------------------------------')
print('epoch: ', epoch)
hit, mrr = train_test(model, train_data, test_data)
flag = 0
if (hit - best_result[0]) > 0.0001:
best_result[0] = hit
best_epoch[0] = epoch
flag = 1
if (mrr - best_result[1]) > 0.0001:
best_result[1] = mrr
best_epoch[1] = epoch
flag = 1
print('Best Result:')
print('\tRecall@20:\t%.4f\tMMR@20:\t%.4f\tEpoch:\t%d,\t%d' %
(best_result[0], best_result[1], best_epoch[0], best_epoch[1]))
bad_counter += 1 - flag
if bad_counter >= opt.patience:
break
print('-------------------------------------------------------')
end = time.time()
print("Run time: %f s" % (end - start))
def parse(self):
if self.annot_file.endswith('.gz'):
fh = gzip.open(self.annot_file, 'rt')
else:
fh = open(self.annot_file)
for line in fh:
if line[0] == '#': continue
cols = line.strip().split('\t')
record = Data(
seqid = cols[0],
feature = cols[2].upper(),
start = int(cols[3]),
end = int(cols[4]),
attrs = Data()
)
for item in cols[-1].split(';'):
if not item:
continue
#if _format == 'GFF':
# name, value = item.split('=')
#else:
# name, value = item.strip().strip('"').split('"')
name, value = self.split_val(item)
record.attrs[name.strip().upper()] = value
yield record
fh.close()
def on_join(self, c, e):
"""Hook for someone joining the channel (/join).
--> e.target() == channel
--> e.source() == nick!~user@hostname
"""
nick, channame, hostname = nm_to_n(
e.source()), e.target().lower(), nm_to_h(e.source())
if self.nick != nick:
chan = self.chans[channame]
self.users[nick] = user = User(nick, hostname, self.connection) if \
not nick in self.users else self.users[nick]
user.channels += [chan]
chan.users += [user]
data = Data(line_raw=nick,
reaction_type="join",
chan=chan,
user=user)
self.dispatch(data)
else:
self.chans[channame] = chan = Channel(channame, self.connection)
data = Data(line_raw=channame,
reaction_type="enter_channel",
chan=chan)
self.dispatch(data)
def main():
os.environ["CUDA_VISIBLE_DEVICES"] = "0"
train_data = pickle.load(open('./dataset/train.txt', 'rb'))
test_data = pickle.load(open('./dataset/test.txt', 'rb'))
fp = open("./dataset/statistic.txt", "rb+")
(all_user_count, all_item_count, all_cate_count) = pickle.load(fp)
print("训练集sess数量：" + str(len(train_data[0])))
print("测试集sess数量：" + str(len(test_data[0])))
# ===========attention==============
# train_data一开始只是一堆session + target，在Data这个类里才开始构建图之类的
train_data = Data(train_data, shuffle=True)
# print(train_data)
test_data = Data(test_data, shuffle=False)
# del all_train_seq, g
n_node = all_item_count + 1
model = trans_to_cuda(SessionGraph(opt, n_node))
start = time.time()
best_result = [0, 0]
best_epoch = [0, 0]
bad_counter = 0
for epoch in range(opt.epoch):
print('-------------------------------------------------------')
print('epoch: ', epoch)
# print("===param===")
# for param in model.named_parameters():
# print(param)
hit, mrr = train_test(model, train_data, test_data)
flag = 0
if hit >= best_result[0]:
best_result[0] = hit
best_epoch[0] = epoch
flag = 1
if mrr >= best_result[1]:
best_result[1] = mrr
best_epoch[1] = epoch
flag = 1
print('Best Result:')
print('\tRecall@20:\t%.4f\tMMR@20:\t%.4f\tEpoch:\t%d,\t%d' %
(best_result[0], best_result[1], best_epoch[0], best_epoch[1]))
bad_counter += 1 - flag
if bad_counter >= opt.patience:
break
print('-------------------------------------------------------')
end = time.time()
print("Run time: %f s" % (end - start))
def main():
train_data = pickle.load(
open('../datasets/' + opt.dataset + '/train.txt', 'rb'))
if opt.validation:
train_data, valid_data = split_validation(train_data,
opt.valid_portion)
test_data = valid_data
else:
test_data = pickle.load(
open('../datasets/' + opt.dataset + '/test.txt', 'rb'))
# all_train_seq = pickle.load(open('../datasets/' + opt.dataset + '/all_train_seq.txt', 'rb'))
# g = build_graph(all_train_seq)
train_data = Data(train_data, shuffle=True, opt=opt)
test_data = Data(test_data, shuffle=False, opt=opt)
# del all_train_seq, g
if opt.dataset == 'diginetica':
n_node = 43098
elif opt.dataset == 'yoochoose1_64' or opt.dataset == 'yoochoose1_4':
n_node = 37484
elif opt.dataset == 'diginetica_users':
n_node = 57070
else:
n_node = 310
model = trans_to_cuda(
SessionGraph(opt, n_node, max(train_data.len_max, test_data.len_max)))
start = time.time()
best_result = [0, 0]
best_epoch = [0, 0]
bad_counter = 0
for epoch in range(opt.epoch):
print('-------------------------------------------------------')
print('epoch: ', epoch)
hit, mrr = train_test(model, train_data, test_data)
flag = 0
if hit >= best_result[0]:
best_result[0] = hit
best_epoch[0] = epoch
flag = 1
if mrr >= best_result[1]:
best_result[1] = mrr
best_epoch[1] = epoch
flag = 1
print('Best Result:')
print('\tRecall@20:\t%.4f\tMMR@20:\t%.4f\tEpoch:\t%d,\t%d' %
(best_result[0], best_result[1], best_epoch[0], best_epoch[1]))
bad_counter += 1 - flag
if bad_counter >= opt.patience:
break
print('-------------------------------------------------------')
end = time.time()
print("Run time: %f s" % (end - start))
def main():
#Read and sort the data
train_data = os.path.join(opt.data_folder, opt.train_data)
x_train = pd.read_csv(train_data)
x_train.sort_values(opt.sessionid, inplace=True)
x_train = x_train.iloc[-int(len(x_train) / 4):]
valid_data = os.path.join(opt.data_folder, opt.valid_data)
x_valid = pd.read_csv(valid_data)
x_valid.sort_values(opt.sessionid, inplace=True)
#Extract Item Features column names (if exists)
feats_columns = []
ffeats = opt.item_feats.strip().split("#")
if ffeats[0] != '':
feats_columns.extend(ffeats)
train_data, test_data, n_node, item_features = preprocess(x_train, x_valid, feats_columns, opt)
del x_train
del x_valid
train_data = Data(train_data, shuffle=True)
test_data = Data(test_data, shuffle=False)
print('Number of Nodes:', n_node)
model = trans_to_cuda(SessionGraph(opt, n_node, item_features))
start = time.time()
best_result = [0, 0]
#bad_counter = 0
f=open("outputlogs.txt","a+")
print('START TRAINING........')
for epoch in range(opt.epoch):
print('-------------------------------------------------------')
print('Epoch: ', epoch)
hit, mrr = train_test(model, train_data, test_data, opt.K)
#flag = 0
if hit >= best_result[0] or mrr >= best_result[1]:
#flag = 1
best_result[0] = hit
best_result[1] = mrr
print('Epoch%d: \tRecall@%d:\t%.4f\tMMR@%d:\t%.4f'% (epoch, opt.K, hit, opt.K, mrr))
f.write('Epoch%d: \tRecall@%d:\t%.4f\tMMR@%d:\t%.4f'% (epoch, opt.K, hit, opt.K, mrr))
#bad_counter += 1 - flag
#if bad_counter >= opt.patience:
# break
print('-------------------------------------------------------')
end = time.time()
print("Run time: %f s" % (end - start))
f.write("\nRun time: %f s" % (end - start))
f.write('\n-----------------------------------------------------------')
f.close()
def test_model():
input_shape = (256, 256, 3)
local_shape = (128, 128, 3)
batch_size = 4
test_datagen = Data(test_data, input_shape[:2], local_shape[:2])
completion_model = load_model("model/completion.h5", compile=False)
#completion_model.summary()
cnt = 0
for inputs, points, masks in test_datagen.flow(batch_size):
completion_image = completion_model.predict([inputs, masks])
num_img = min(5, batch_size)
fig, axs = plt.subplots(num_img, 3)
for i in range(num_img):
axs[i, 0].imshow(inputs[i] * (1 - masks[i]))
axs[i, 0].axis('off')
axs[i, 0].set_title('Input')
axs[i, 1].imshow(completion_image[i])
axs[i, 1].axis('off')
axs[i, 1].set_title('Output')
axs[i, 2].imshow(inputs[i])
axs[i, 2].axis('off')
axs[i, 2].set_title('Ground Truth')
fig.savefig(os.path.join("result", "result_test_%d.png" % cnt))
plt.close()
cnt += 1
def main():
# generate some dummy data for testing
manager = Data()
num_examples = 10**4
max_val = 1
train_batch_size = 16
train_size = int(num_examples/2)
eval_size = int(num_examples/2)
X, y = manager.create_data_set(num_of_examples=num_examples, max_val=max_val,
discriminator=lambda x: max_val*(1/(1+np.exp(-x)) + 1/(1+np.exp(x**2)))-max_val/2,
one_hot=False, plot_data=False, load_saved_data=True,
filename='dataset.npy')
train_examples, train_labels = X[0:train_size, :], y[0:train_size]
eval_examples, eval_labels = X[train_size:, :], y[train_size:]
print('train examples = {}, train labels = {}, eval examples = {}, eval labels = {}'.format(train_examples.shape,
train_labels.shape,
eval_examples.shape,
eval_labels.shape))
# get some small train batch
indices = np.random.randint(low=0,high=train_size,size=train_batch_size)
train_batch_examples, train_batch_labels = X[indices,:], y[indices]
# start by defining your default graph
graph = GRAPH()
graph.getDefaultGraph()
# declare your placeholders, to provide your inputs
# print(int(train_batch_examples.shape[1]))
input_features = placeholder(shape=(train_batch_size, int(train_batch_examples.shape[1])))
input_labels = placeholder(shape=(train_batch_size))
"""
Method #3
"""
# this is defined using layers
features = fully_connected(features=input_features, units=32)
features = relu(features)
features = fully_connected(features=features, units=64)
features = relu(features)
features = fully_connected(features=features, units=128)
features = relu(features)
features = fully_connected(features=features, units=64)
features = relu(features)
features = fully_connected(features=features, units=32)
features = relu(features)
features = fully_connected(features=features, units=2)
logits = softmax_classifier(features)
loss = CrossEntropyLoss(softmax_logits=logits, labels=input_labels)
# compile and run
graph.graph_compile(function=loss, verbose=True)
loss = graph.run(input_matrices={input_features: train_batch_examples, input_labels: train_batch_labels})
print(loss, logits.output.shape)
def setUp(self):
self.starting_recipes = get_starting_recipes()
self.starting_recipes_plus_one = get_starting_recipes_plus_one(
self.starting_recipes)
self._filepath = "./tests/test_data/recipes.json"
create_recipe_file(self._filepath)
self.r = Recipe(name="peanut_butter_toast",
meal="breakfast",
servings=1,
category="veggie",
speed="very_fast",
ingredients=[{
"item": "bread",
"qty": 1,
"unit": "slice"
}, {
"item": "peanut_butter",
"qty": 1,
"unit": "tablespoon"
}, {
"item": "banana",
"qty": 1,
"unit": "amount"
}])
self.d = Data(filepath=self._filepath)
def setUp(self):
random.seed(1)
self.rules = {
"monday": {
"breakfast_categories": ["veggie", "vegan", "fish"],
"brunch_categories": [],
"lunch_categories": ["veggie", "vegan", "fish"],
"dinner_categories": ["veggie", "vegan", "fish"],
"breakfast_speed": ["very_fast"],
"brunch_speed": [],
"lunch_speed": ["very_fast"],
"dinner_speed": ["very_fast", "fast", "medium"]
},
"sunday": {
"breakfast_categories": ["veggie", "vegan", "fish", "meat"],
"brunch_categories": [],
"lunch_categories": ["veggie", "vegan", "fish", "meat"],
"dinner_categories": ["meat"],
"breakfast_speed": ["very_fast"],
"brunch_speed": [],
"lunch_speed": ["very_fast"],
"dinner_speed":
["very_fast", "fast", "medium", "slow", "very_slow"]
}
}
self._filepath = "./tests/test_data/recipes.json"
create_recipe_file(self._filepath)
self.d = Data(self._filepath)
self.p = Plan(self.d, self.rules, "2021-04-17", people=2)
self.maxDiff = None
def make_memory_dataset(self, aug_crop=False):
dataset = tf.data.Dataset.from_tensor_slices(Data(self.images_PH, self.labels_PH, self.nus_PH, self.singles_PH))
c_map_func = None
if aug_crop:
c_map_func = self.augment
self.dataset = self.shuf_rep_map_batch_fetch(dataset, self.shuffle_buff, self.epochs, self.batch_sz, c_map_func)
def main():
args = parse_args()
data = Data(np.array([]), args)
data.open_image()
data = process(data)
data.save_image()
def main():
train_data = pickle.load(open('./datasets/' + opt.dataset + '/train.txt', 'rb'))
if opt.validation:
train_data, valid_data = split_validation(train_data, opt.valid_portion)
test_data = valid_data
else:
test_data = pickle.load(open('./datasets/' + opt.dataset + '/test.txt', 'rb'))
train_data = Data(train_data, shuffle=True)
test_data = Data(test_data, shuffle=False)
if opt.dataset == 'diginetica':
n_node = 43098
else:
n_node = 37484
model = trans_to_cuda(SelfAttentionNetwork(opt, n_node))
start = time.time()
best_result = [0, 0]
best_epoch = [0, 0]
bad_counter = 0
for epoch in range(opt.epoch):
print('-------------------------------------------------------')
print('epoch: ', epoch)
hit, mrr = train_test(model, train_data, test_data)
flag = 0
if hit >= best_result[0]:
best_result[0] = hit
best_epoch[0] = epoch
flag = 1
if mrr >= best_result[1]:
best_result[1] = mrr
best_epoch[1] = epoch
flag = 1
print('Best Result:')
print('\tRecall@20:\t%.4f\tMMR@20:\t%.4f\tEpoch:\t%d,\t%d'% (best_result[0], best_result[1], best_epoch[0], best_epoch[1]))
bad_counter += 1 - flag
if bad_counter >= opt.patience:
break
print('-------------------------------------------------------')
end = time.time()
print("Run time: %f s" % (end - start))
def main():
data = Data('hyperspectral_data//san.mat') # load data
ecem = ECEM()
ecem.parmset(**{'windowsize': [1/4, 2/4, 3/4, 4/4], # window size
'num_layer': 10, # the number of detection layers
'num_cem': 6, # the number of CEMs per layer
'Lambda': 1e-6, # the regularization coefficient
'show_proc': True}) # show the process or not
result = ecem.detect(data, pool_num=4) # detection (we recomemend to use multi-thread processing to speed up detetion)
ecem.show([result], ['E-CEM']) # show
def main():
doc_content_list, doc_train_list, doc_test_list, vocab_dic, labels_dic, max_num_sentence, keywords_dic, class_weights = read_file(
args.dataset, args.use_LDA)
pre_trained_weight = []
if args.dataset == 'mr':
gloveFile = 'data/glove.6B.300d.txt'
if not os.path.exists(gloveFile):
print(
'Please download the pretained Glove Embedding from https://nlp.stanford.edu/projects/glove/'
)
return
pre_trained_weight = loadGloveModel(gloveFile, vocab_dic,
len(vocab_dic) + 1)
train_data, valid_data = split_validation(doc_train_list,
args.valid_portion, SEED)
test_data = split_validation(doc_test_list, 0.0, SEED)
num_categories = len(labels_dic)
train_data = Data(train_data, max_num_sentence, keywords_dic,
num_categories, args.use_LDA)
valid_data = Data(valid_data, max_num_sentence, keywords_dic,
num_categories, args.use_LDA)
test_data = Data(test_data, max_num_sentence, keywords_dic, num_categories,
args.use_LDA)
model = trans_to_cuda(
DocumentGraph(args, pre_trained_weight, class_weights,
len(vocab_dic) + 1, len(labels_dic)))
for epoch in range(args.epoch):
print('-------------------------------------------------------')
print('epoch: ', epoch)
train_model(model, train_data, args)
valid_detail, valid_acc = test_model(model, valid_data, args, False)
detail, acc = test_model(model, test_data, args, False)
print('Validation Accuracy:\t%.4f, Test Accuracy:\t%.4f' %
(valid_acc, acc))
def standardize_data(self, datatup):
images = datatup.images
labels = datatup.labels
nus = datatup.nus
singles = datatup.singles
print('Inception preprocessing')
images = (images - tf.reduce_min(images)) * (1.0 - 0.0) / (tf.reduce_max(images) - tf.reduce_min(images)) + 0.0
images = (images - 0.5) * 2 # get values [-1, 1]. Shift and scale. For Resnet V2 and all tf models.
return Data(images, labels, nus, singles)
def make_memory_feed_dict(self, in_memory_Data=None):
if in_memory_Data is not None:
c_data = in_memory_Data # if you just process the files and keep the Data tuple in memory
else:
with np.load(self.file_name) as c_file:
c_data = Data(c_file['images'], c_file['labels'], c_file['nus'], c_file['singles'])
self.feed_dict = {self.images_PH: c_data.images,
self.labels_PH: c_data.labels,
self.nus_PH: c_data.nus,
self.singles_PH: c_data.singles}
def train_model(result_dir="result", data=data_train):
train_datagen = Data(data_train, input_shape[:2], local_shape[:2])
for n in range(n_epoch):
progbar = generic_utils.Progbar(len(train_datagen))
for inputs, points, masks in train_datagen.flow(batch_size):
completion_image = completion_model.predict([inputs, masks])
valid = np.ones((batch_size, 1))
fake = np.zeros((batch_size, 1))
g_loss = 0.0
d_loss = 0.0
if n < tc:
g_loss = completion_model.train_on_batch([inputs, masks],
inputs)
else:
d_loss_real = d_model.train_on_batch([inputs, points], valid)
d_loss_fake = d_model.train_on_batch(
[completion_image, points], fake)
d_loss = 0.5 * np.add(d_loss_real, d_loss_fake)
if n >= tc + td:
g_loss = all_model.train_on_batch([inputs, masks, points],
[inputs, valid])
g_loss = g_loss[0] + alpha * g_loss[1]
progbar.add(inputs.shape[0],
values=[("D loss", d_loss), ("G mse", g_loss)])
num_img = min(5, batch_size)
fig, axs = plt.subplots(num_img, 3)
for i in range(num_img):
axs[i, 0].imshow(inputs[i] * (1 - masks[i]))
axs[i, 0].axis('off')
axs[i, 0].set_title('Input')
axs[i, 1].imshow(completion_image[i])
axs[i, 1].axis('off')
axs[i, 1].set_title('Output')
axs[i, 2].imshow(inputs[i])
axs[i, 2].axis('off')
axs[i, 2].set_title('Ground Truth')
fig.savefig(os.path.join(result_dir, "result_%d.png" % n))
plt.close()
# Save the checkpoints every 10 epochs
if (n + 1) % 10 == 0:
checkpoint.save(file_prefix=checkpoint_prefix)
# save model
generator.save(os.path.join("model", "generator.h5"))
completion_model.save(os.path.join("model", "completion.h5"))
discriminator.save(os.path.join("model", "discriminator.h5"))
def create_execution() -> jsonify:
service_type = request.args.get(Constants.TYPE_PARAM_NAME)
filename = request.json[Constants.NAME_FIELD_NAME]
description = request.json[Constants.DESCRIPTION_FIELD_NAME]
module_path = request.json[Constants.MODULE_PATH_FIELD_NAME]
class_name = request.json[Constants.CLASS_FIELD_NAME]
class_parameters = request.json[Constants.CLASS_PARAMETERS_FIELD_NAME]
class_method_name = request.json[Constants.METHOD_FIELD_NAME]
method_parameters = request.json[Constants.METHOD_PARAMETERS_FIELD_NAME]
request_errors = analyse_post_request_errors(request_validator, filename,
module_path, class_name,
class_parameters,
class_method_name,
method_parameters)
if request_errors is not None:
return request_errors
storage = None
if service_type == Constants.EXPLORE_TENSORFLOW_TYPE or \
service_type == Constants.EXPLORE_SCIKITLEARN_TYPE:
storage = explore_storage
elif service_type == Constants.TRANSFORM_TENSORFLOW_TYPE or \
service_type == Constants.TRANSFORM_SCIKITLEARN_TYPE:
storage = transform_storage
data = Data(database, storage)
parameters = Parameters(database, data)
metadata_creator = Metadata(database)
execution = Execution(database, filename, service_type, storage,
metadata_creator, module_path, class_name,
class_parameters, parameters)
execution.create(class_method_name, method_parameters, description)
response_params = None
if service_type == Constants.TRANSFORM_TENSORFLOW_TYPE or \
service_type == Constants.TRANSFORM_SCIKITLEARN_TYPE:
response_params = Constants.MICROSERVICE_URI_GET_PARAMS
return (
jsonify({
Constants.MESSAGE_RESULT:
f'{Constants.MICROSERVICE_URI_SWITCHER[service_type]}'
f'{filename}{response_params}'
}),
Constants.HTTP_STATUS_CODE_SUCCESS_CREATED,
)
def train_bbox_regressor(X, bbox, gt):
config = Data()
config.min_overlap = 0.6
config.delta = 1000
config.method = 'ridge_reg_chol'
# get training groundtruth
Y, O = get_examples(bbox, gt)
X = X[O > config.min_overlap]
Y = Y[O > config.min_overlap]
# add bias
X = np.c_[X, np.ones([X.shape[0], 1])]
# center and decorrelate targets
mu = np.mean(Y, axis=0).reshape(1, -1)
Y = Y - mu
S = dot(Y.T, Y) / Y.shape[0]
D, V = eig(S)
T = dot(dot(V, diag(1.0 / sqrt(D + 0.001))), V.T)
T_inv = dot(dot(V, diag(sqrt(D + 0.001))), V.T)
Y = dot(Y, T)
model = Data()
model.mu = mu
model.T = T
model.T_inv = T_inv
model.Beta = np.c_[solve(X, Y[:, 0], config.delta, config.method),
solve(X, Y[:, 1], config.delta, config.method),
solve(X, Y[:, 2], config.delta, config.method),
solve(X, Y[:, 3], config.delta, config.method)]
# pack
bbox_reg = Data()
bbox_reg.model = model
bbox_reg.config = config
return bbox_reg
def on_privmsg(self, c, e):
"""Hook for private message written to bot directly (/msg).
--> e.target() == own_nick
--> e.source() == nick!~user@hostname
"""
nick, hostname = nm_to_n(e.source()), nm_to_h(e.source())
if not nick in self.users:
self.users[nick] = User(nick, hostname, self.connection)
user, line_raw = self.users[nick], e.arguments()[0]
data = Data(line_raw=line_raw, reaction_type="private", user=user)
# execute plugins
self.dispatch(data)
def on_nick(self, c, e):
"""Hook for someone changing his nickname (/nick).
--> e.target() == new_nick
--> e.source() == old_nick!~user@hostname
"""
old_nick, new_nick = nm_to_n(e.source()), e.target()
self.users[new_nick] = self.users[old_nick]
del self.users[old_nick]
self.users[new_nick].name = new_nick
# execute plugins
data = Data(line_raw=old_nick,
reaction_type="nick_change",
user=self.users[new_nick])
self.dispatch(data)
def on_part(self, c, e):
"""Hook for an user leaving the channel (/part).
--> e.target() == channel
--> e.source() == nick!~user@hostname
"""
nick, hostname = nm_to_n(e.source()), nm_to_h(e.source())
if not nick in self.users:
self.users[nick] = User(nick, hostname, self.connection)
chan, user = self.chans[e.target().lower()], self.users[nick]
if self.nick != user.name:
if user in chan.users:
chan.users.remove(user)
if chan in self.chans:
user.channels.remove(chan)
data = Data(line_raw=user.name,
reaction_type="leave",
chan=chan,
user=user)
self.dispatch(data)
if len(user.channels) == 0:
del self.users[user.name]
else:
data = Data(line_raw=user.name,
reaction_type="leave_channel",
chan=chan,
user=user)
self.dispatch(data)
del self.chans[chan.name]
for u in self.users.values():
if chan in u.channels:
u.channels.remove(chan)
def show_maps():
layer = model.h_conv
data = Data(1)
# print data._y.shape
# print data._X.shape
# layer = model.h_conv5
batch_size = 40 # image that will be used, taken from test_data/X.npy
batch = data.next_batch(batch_size) # jump batch_size images
batch = data.next_batch(batch_size) # use the first image of this batch
feature_maps = sess.run(layer,
feed_dict={
model.x: batch[0],
model.y_: batch[1],
model.keep_prob: 1.0
})
plotFeatureMaps(feature_maps)
def extractParam(self):
"""Turn muti part encoded form into params."""
params = []
try:
environ = {
'CONTENT_TYPE': self.headers['content-type'],
'CONTENT_LENGTH': self.headers['content-length'],
'REQUEST_METHOD': 'POST',
}
except KeyError:
trace('# Warning: missing header content-type or content-length'
' in file: %s not an http request ?\n' % self.file_path)
return params
form = FieldStorage(fp=StringIO(self.body),
environ=environ,
keep_blank_values=True)
try:
keys = form.keys()
except TypeError:
trace('# Using custom data for request: %s ' % self.file_path)
params = Data(self.headers['content-type'], self.body)
return params
for item in form.list:
key = item.name
value = item.value
filename = item.filename
if filename is None:
params.append([key, value])
else:
# got a file upload
filename = filename or ''
params.append([key, 'Upload("%s")' % filename])
if filename:
if os.path.exists(filename):
trace('# Warning: uploaded file: %s already'
' exists, keep it.\n' % filename)
else:
trace('# Saving uploaded file: %s\n' % filename)
f = open(filename, 'w')
f.write(str(value))
f.close()
return params
def main():
data = Data.Data(max_len=max_len,
max_len_target=max_len_target,
frac=data_frac)
x, y = data.get_data()
input_word_index = data.get_input_word_index()
target_word_index = data.get_target_word_index()
dim_size = data.get_embedding_dim()
embedding_matrix = data.load_embedding()
model = models.OneShotModel(x, y, num_epoch=num_epoch, max_len=max_len, \
max_len_target=max_len_target, batch_size=batch_size, \
input_word_index=input_word_index, \
target_word_index=target_word_index, \
dim_size=dim_size, embedding_matrix=embedding_matrix)
model.train()
def update_execution(filename: str) -> jsonify:
service_type = request.args.get(Constants.TYPE_PARAM_NAME)
tool_type = request.args.get(Constants.TOOL_PARAM_NAME)
description = request.json[Constants.DESCRIPTION_FIELD_NAME]
class_method_name = request.json[Constants.METHOD_FIELD_NAME]
method_parameters = request.json[Constants.METHOD_PARAMETERS_FIELD_NAME]
storage = None
if service_type == Constants.EXPLORE_TYPE:
storage = explore_storage
elif service_type == Constants.TRANSFORM_TYPE:
storage = transform_storage
data = Data(database, storage)
request_errors = analyse_patch_request_errors(request_validator, data,
filename, class_method_name,
method_parameters)
if request_errors is not None:
return request_errors
module_path, class_name = data.get_module_and_class(filename)
class_parameters = data.get_class_parameters(filename)
parameters = Parameters(database, data)
metadata_creator = Metadata(database)
execution = Execution(database, filename, service_type, storage,
metadata_creator, module_path, class_name,
class_parameters, parameters)
execution.update(class_method_name, method_parameters, description)
response_params = None
if service_type == Constants.TRANSFORM_TYPE:
response_params = Constants.MICROSERVICE_URI_GET_PARAMS
return (
jsonify({
Constants.MESSAGE_RESULT:
f'{Constants.MICROSERVICE_URI_SWITCHER[service_type]}'
f'{tool_type}/{filename}{response_params}'
}),
Constants.HTTP_STATUS_CODE_SUCCESS_CREATED,
)
def parse_tfr(exp_proto):
features = {
'image_raw': tf.FixedLenFeature(shape=[], dtype=tf.string),
'label_raw': tf.FixedLenFeature([], tf.string),
'nu': tf.FixedLenFeature([], tf.int64),
'singles_raw': tf.FixedLenFeature([], tf.string)
}
psed = tf.parse_single_example(exp_proto, features)
image = tf.decode_raw(psed['image_raw'], tf.float32)
image = tf.reshape(image, out_img_size)
label = tf.decode_raw(psed['label_raw'], tf.float32)
label = tf.reshape(label, label_size)
nu = tf.cast(psed['nu'], tf.float32)
single = tf.decode_raw(psed['singles_raw'], tf.float32)
single = tf.reshape(single, singles_size)
return Data(image, label, nu, single)
def parse_tfr(self, exp_proto):
features = {
'image_raw': tf.FixedLenFeature(shape=[], dtype=tf.string),
'label_raw': tf.FixedLenFeature([], tf.string),
'nu': tf.FixedLenFeature([], tf.int64),
'singles_raw': tf.FixedLenFeature([], tf.string)
}
psed = tf.parse_single_example(exp_proto, features)
image = tf.decode_raw(psed['image_raw'], tf.float32)
image = tf.reshape(image, self.out_img_size)
label = tf.decode_raw(psed['label_raw'], tf.float32)
label = tf.reshape(label, self.label_size)
nu = tf.cast(psed['nu'], tf.float32)
single = tf.decode_raw(psed['singles_raw'], tf.float32)
single = tf.reshape(single, self.singles_size)
return Data(image, label, nu, single) #tf Dataset knows each tuple item is a component that needs to be batched together. Each Data tuple is an element.
def augment(self, Data_tup, offset_value=15, label_cutt_off=6, class_of_interest=1):
def flipper(m_img, m_label):
m_img = tf.image.flip_left_right(m_img)
m_label = tf.image.flip_left_right(m_label)
return m_img, m_label
def no_change(x, y):
return x, y
def label_mod(f_label_arr, f_lab): #f_lab is 0 or 1 from initial label
class_0_labels = f_label_arr[..., 0] #unchanged from initial
far_right_side = f_label_arr[:, :(label_cutt_off - 1), class_of_interest] # unchanged
new_right_paracentral = tf.ones([f_label_arr.shape[0], 1]) * f_lab #either 0 or 1 based on f_lab
new_left_paracentral = tf.ones([f_label_arr.shape[0], 1]) * f_lab #either 0 or 1 based on f_lab
far_left_side = f_label_arr[:, (label_cutt_off + 1):, class_of_interest] #unchanged
class_1_labels = tf.concat([far_right_side, new_right_paracentral, new_left_paracentral, far_left_side], axis=1)
compiled_label = tf.stack([class_0_labels, class_1_labels], axis=-1)
return compiled_label
img = Data_tup.images
label = Data_tup.labels
nu = Data_tup.nus
single = Data_tup.singles
img = tf.image.random_brightness(img, 0.3)
img = tf.image.random_contrast(img, 0.7, 1.3)
c_rand = tf.random_uniform([1])[0]
img, label = tf.cond(c_rand < 0.5, lambda: no_change(img, label), lambda: flipper(img, label))
right_label = label[0, label_cutt_off - 1, class_of_interest] #copy since int is immutable
left_label = label[0, label_cutt_off, class_of_interest]
H_off = tf.cast((tf.random_uniform([1]) * offset_value), tf.int32)[0]
W_off = tf.cast((tf.random_uniform([1]) * offset_value), tf.int32)[0]
label = tf.cond(W_off < 3, lambda: label_mod(label, right_label), lambda: label)
label = tf.cond(W_off > 12, lambda: label_mod(label, left_label), lambda: label)
img = tf.image.crop_to_bounding_box(img, H_off, W_off, self.out_img_size[0], self.out_img_size[1])
return Data(img, label, nu, single)
