def search():
# return "the f**k"
likeCat = request.form.get('interest')
learnCat = request.form.get('learn')
likeUrl = baseUrl + likeCat
learnUrl = baseUrl + learnCat
resultingKeywords = wc.keyWords(wc.getParagraph(wc.getBestSection(wc.getBestSpan(\
wc.findBestSpan(wc.subCatDict(learnUrl), wc.allConnections(likeUrl))), learnUrl), learnUrl)).split(' ')
queries = resultingKeywords[:10]
paragraphs = []
common_words = []
images_links = []
paragraphs = sample(queries)
full_dict = defaultdict(dict)
for x in range(0, len(queries)):
paragraph = paragraphs[x]
removed_paragraph = wc.keyWords(paragraph)
query = queries[x]
counter = Counter(removed_paragraph.split(' '))
most_common = counter.most_common(10)
most_common = {x[0]: x[1] for x in most_common}
images = getImages(query)
full_dict[x]['query'] = query
full_dict[x]['paragraph'] = paragraph
full_dict[x]['common_words'] = most_common
full_dict[x]['images'] = images
return jsonify(dict(full_dict))
def newSample(self,
samID,
dataFile,
samplName,
samplID="",
sampleType="unknown"):
self.sampleList.append(
sample(samID, self, dataFile, samplName, samplID, sampleType))
def random_select():
"""
随机选择一组号码
"""
red_balls = [x for x in range(1, 34)]
selected_balls = []
selected_balls = sample(red_balls, 6)
selected_balls.sort()
selected_balls.append(randint(1, 16))
return selected_balls
def sample_model(args, logger=None):
if args.text == '':
strings = [
'call me ishmael some years ago', 'A project by Sam Greydanus',
'mmm mmm mmm mmm mmm mmm mmm',
'What I cannot create I do not understand',
'You know nothing Jon Snow'
] # test strings
else:
strings = [args.text]
logger = Logger(
args) if logger is None else logger # instantiate logger, if None
logger.write("\nSAMPLING MODE...")
logger.write("loading data...")
logger.write("building model...")
model = Model(args, logger)
logger.write("attempt to load saved model...")
load_was_success, global_step = model.try_load_model(args.save_path)
if load_was_success:
for s in strings:
strokes, phis, windows, kappas = sample(s, model, args)
w_save_path = '{}figures/iter-{}-w-{}'.format(
args.log_dir, global_step, s[:10].replace(' ', '_'))
g_save_path = '{}figures/iter-{}-g-{}'.format(
args.log_dir, global_step, s[:10].replace(' ', '_'))
l_save_path = '{}figures/iter-{}-l-{}'.format(
args.log_dir, global_step, s[:10].replace(' ', '_'))
window_plots(phis, windows, save_path=w_save_path)
gauss_plot(strokes,
'Heatmap for "{}"'.format(s),
figsize=(2 * len(s), 4),
save_path=g_save_path)
line_plot(strokes,
'Line plot for "{}"'.format(s),
figsize=(len(s), 2),
save_path=l_save_path)
# make sure that kappas are reasonable
logger.write("kappas: \n{}".format(
str(kappas[min(kappas.shape[0] -
1, args.tsteps_per_ascii), :])))
else:
logger.write("load failed, sampling canceled")
if True:
tf.reset_default_graph()
time.sleep(args.sleep_time)
sample_model(args, logger=logger)
def dev_or_test_data_feature_extract(data_file,dependency_file):
"""
load dev or test data from dev or test data file and extract features
return a sample list
"""
dev_f = codecs.open(data_file, 'r', 'utf-8')
dev_dependency_f = codecs.open(dependency_file, 'r', 'utf-8')
lines = dev_f.readlines()
dependency_lines = dev_dependency_f.readlines()
dev_f.close()
dev_dependency_f.close()
samples = []
for i,(line, dependency_line) in enumerate(zip(lines,dependency_lines)):
print i
new_sample = sample()
line = line.split('\t')
dependency_line = dependency_line.split('\t')
new_sample.sent1_id = int(line[0])
new_sample.sent2_id = int(line[1])
assert line[0] == dependency_line[0]
assert line[1] == dependency_line[1]
#preprocessing of text:tokenization and stemming
sent1 = [porter.stem(t) for t in nltk.word_tokenize(line[2])]
new_sample.sent1 = sent1
sent2 = [porter.stem(t) for t in nltk.word_tokenize(line[3])]
new_sample.sent2 = sent2
#filter stop words
sent1_nonstop_word = [word for word in sent1 if not word in english_stopwords and not word in english_punctuations]
sent2_nonstop_word = [word for word in sent2 if not word in english_stopwords and not word in english_punctuations]
#get n-gram features
new_sample.word_overlap = unigram_overlap(sent1_nonstop_word,sent2_nonstop_word)
new_sample.bigram_overlap = bigram_overlap(sent1, sent2)
new_sample.trigram_overlap = trigram_overlap(sent1, sent2)
new_sample.fourgram_overlap = fourgram_overlap(sent1, sent2)
new_sample.lcs = float(max_LCS(sent1, sent2)) / min(len(sent1),len(sent2))
new_sample.length_diff = float(abs(len(sent1)-len(sent2))) / min(len(sent1),len(sent2))
pos1 = nltk.pos_tag(sent1)
pos2 = nltk.pos_tag(sent2)
pos1_nonstop_word = [word for word in pos1 if not word[0] in english_stopwords and not word[0] in english_punctuations]
pos2_nonstop_word = [word for word in pos2 if not word[0] in english_stopwords and not word[0] in english_punctuations]
new_sample.nnp_overlap = nnp_overlap(pos1, pos2)
new_sample.ne_overlap = ne_overlap(pos1, pos2)
new_sample.num_overlap = num_overlap(pos1,pos2)
#get knowledge based features
new_sample.path_best_sim = path_best_sim(pos1_nonstop_word, pos2_nonstop_word, pos_map, pos_list)
new_sample.wup_best_sim = wup_best_sim(pos1_nonstop_word, pos2_nonstop_word, pos_map, pos_list)
new_sample.lch_best_sim = lch_best_sim(pos1_nonstop_word, pos2_nonstop_word, pos_map, pos_list)
new_sample.lin_best_sim = lin_best_sim(pos1_nonstop_word, pos2_nonstop_word, brown_ic, pos_map, pos_list)
new_sample.res_best_sim = res_best_sim(pos1_nonstop_word, pos2_nonstop_word, brown_ic, pos_map, pos_list)
new_sample.jcn_best_sim = jcn_best_sim(pos1_nonstop_word, pos2_nonstop_word, brown_ic, pos_map, pos_list)
new_sample.dependency_sim = dependency_sim(dependency_line[2], dependency_line[3])
samples.append(new_sample)
return samples
def sample_model(args, logger=None):
if args.text == '':
strings = ['very good', 'a project', 'hello', \
'this', 'how is'] # test strings
else:
strings = [args.text]
logger = Logger(
args) if logger is None else logger # instantiate logger, if None
logger.write("\nSAMPLING MODE...")
logger.write("loading data...")
logger.write("building model...")
model = Model(args, logger)
logger.write("attempt to load saved model...")
load_was_success, global_step = model.try_load_model(args.save_path)
if load_was_success:
for s in strings:
strokes, phis, windows, kappas = sample(s, model, args)
w_save_path = '{}figures/iter-{}-w-{}'.format(
args.log_dir, global_step, s[:10].replace(' ', '_'))
g_save_path = '{}figures/iter-{}-g-{}'.format(
args.log_dir, global_step, s[:10].replace(' ', '_'))
l_save_path = '{}figures/iter-{}-l-{}'.format(
args.log_dir, global_step, s[:10].replace(' ', '_'))
window_plots(phis, windows, save_path=w_save_path)
# gauss_plot(strokes, 'Heatmap for "{}"'.format(s), figsize = (2*len(s),2*len(s)), save_path=g_save_path)
line_plot(strokes,
'3D plot for "{}"'.format(s),
figsize=(len(s), len(s)),
save_path=l_save_path)
# make sure that kappas are reasonable
logger.write("kappas: \n{}".format(
str(kappas[min(kappas.shape[0] -
1, args.tsteps_per_ascii), :])))
else:
logger.write("load failed, sampling canceled")
if True:
tf.reset_default_graph()
time.sleep(args.sleep_time)
sample_model(args, logger=logger)
def post(self):
"""Says hi to given name."""
request, err = flask_request_response.json_request(
SAMPLE_API, POST_REQUEST
)
if err is not None:
return flask_request_response.json_response(
{"error_message": str(err)}, SAMPLE_API, POST_REQUEST, 400
)
if "name" not in request:
return flask_request_response.json_response(
{"error_message": "Unable to find the 'name' in the request"},
SAMPLE_API, POST_REQUEST, 400
)
return flask_request_response.json_response(
{"text": sample(request["name"])},
SAMPLE_API, POST_REQUEST, 200
)
def sample_model(args, logger=None):
if args.text == '':
strings = [
'call me ishmael some years ago', 'A project by Sam Greydanus',
'mmm mmm mmm mmm mmm mmm mmm',
'What I cannot create I do not understand',
'You know nothing Jon Snow'
] # test strings
else:
strings = [args.text]
model = Model(args)
load_was_success, global_step = load_pretrained_model(
model, args['save_path'])
if load_was_success:
for s in strings:
strokes = sample(s, model, args)
w_save_path = '{}figures/iter-{}-w-{}'.format(
args['logs_dir'], global_step, s[:10].replace(' ', '_'))
g_save_path = '{}figures/iter-{}-g-{}'.format(
args['logs_dir'], global_step, s[:10].replace(' ', '_'))
l_save_path = '{}figures/iter-{}-l-{}'.format(
args['logs_dir'], global_step, s[:10].replace(' ', '_'))
gauss_plot(strokes,
'Heatmap for "{}"'.format(s),
figsize=(2 * len(s), 4),
save_path=g_save_path)
line_plot(strokes,
'Line plot for "{}"'.format(s),
figsize=(len(s), 2),
save_path=l_save_path)
else:
print("load failed, sampling canceled")
if True:
tf.reset_default_graph()
time.sleep(args.sleep_time)
sample_model(args, logger=logger)
def get(self):
"""Says hi to cookiecutter."""
return flask_request_response.json_response(
{"text": sample()},
SAMPLE_API, GET_REQUEST, 200
)
board = [0] * length
for i in range(1, length):
board[i] = random.randrange(11)
return board
def sample(board):
# get the correct result with the test case generated
import user41_LrDbe6YZgW_0 as sample
sample_game = sample.SolitaireMancala()
sample_game.set_board(board)
return sample_game.plan_moves()
def error(board):
# get the output from error sample with the test case generated
import user41_cwKeKZrsxa_1 as error
error_game = error.SolitaireMancala()
error_game.set_board(board)
return error_game.plan_moves()
for i in range(50):
# run 50 times and find the mismatched cases and
#output their corresponding test cases
config = generate()
sample_move = sample(config)
error_move = error(config)
if sample_move != error_move:
print config
#"Model_LSTM_Scene_common_subgrids_nonlinear" : Model_LSTM_Scene_common_subgrids_nonlinear
}
return model_dict[args.model_name]
if __name__ == '__main__':
args = get_args() # Get input argurments
args.max_datasets = 5 # Maximum number of sequences could be used for storing scene data
args.log_dir = os.path.join(args.save_root, args.dataset_size,
args.model_dir, str(args.model_dataset), 'log')
args.save_model_dir = os.path.join(args.save_root,
args.dataset_size, args.model_dir,
str(args.model_dataset), 'model')
logger = Logger(args, train=False) # make logging utility
logger.write("{}\n".format(args))
data_loader = DataLoader(args, logger, train=False)
model = get_model(args)
logger.write('evaluating on test data ......')
save_model_file = '{}/best_epoch_model.pt'.format(args.save_model_dir)
mse_eval, nde_eval, fde_eval = sample(model,
data_loader,
save_model_file,
args,
test=True)
# Print out results
logger.write('mse_eval: {:.3f}, nde_eval: {:.3f}, fde_eval: {:.3f}'.format(
mse_eval, nde_eval, fde_eval))
def test_sample():
assert sample(True)
assert not sample(False)
import sample sample()
if hasattr(obj, '__name__') and hasattr(obj, '__module__'):
return obj.__module__ + '.' + obj.__name__
raise AttributeError('no attributes found the make a usable string ' \
'out oj the object')
def change_line_identitation(lines, spaces = 4):
"""change the minimum identation of the given lines to the given amount of whitespaces"""
min_ident = min([len(line) - len(line.lstrip()) for line in lines if \
line.lstrip() and line.lstrip()[0] != "#"])
if min_ident > spaces:
return [line[min_ident - spaces:] for line in lines]
return [(spaces - min_ident) * ' ' + line for line in lines]
def sample(obj, count = 5):
"""print samples where the object is used"""
for i, sample in enumerate(find_samples(obj)):
if i >= count: break
if i != 0:
print()
print(sample.project)
for i, lines in enumerate(sample.sample_lines(5)):
print('\n'.join(change_line_identitation(lines)))
__all__ = ['sample', 'Samples', 'find_samples', 'nullege_url', 'nullege_json']
if __name__ == '__main__':
sample('bytearray')
def register():
reg1 = sample()
reg1.register()
def train_data_feature_extract(train_file, train_dependency_file):
"""
load train data from train data file and extract features
return a sample list
"""
train_f = codecs.open(train_file, 'r', 'utf-8')
train_dependency_f = codecs.open(train_dependency_file, 'r', 'utf-8')
lines = train_f.readlines()
dependency_lines = train_dependency_f.readlines()
train_f.close()
train_dependency_f.close()
train_samples = []
for i, (line, dependency_line) in enumerate(zip(lines, dependency_lines)):
print i
new_sample = sample()
line = line.split('\t')
dependency_line = dependency_line.split('\t')
if line[0] == '0':
new_sample.is_repeat = 0
else:
new_sample.is_repeat = 1
new_sample.sent1_id = int(line[1])
new_sample.sent2_id = int(line[2])
assert line[1] == dependency_line[0]
assert line[2] == dependency_line[1]
#preprocessing of text:tokenization and stemming
sent1 = [porter.stem(t) for t in nltk.word_tokenize(line[3])]
new_sample.sent1 = sent1
sent2 = [porter.stem(t) for t in nltk.word_tokenize(line[4])]
new_sample.sent2 = sent2
#filter stop words
sent1_nonstop_word = [
word for word in sent1 if not word in english_stopwords
and not word in english_punctuations
]
sent2_nonstop_word = [
word for word in sent2 if not word in english_stopwords
and not word in english_punctuations
]
#get n-gram features
new_sample.word_overlap = unigram_overlap(sent1_nonstop_word,
sent2_nonstop_word)
new_sample.bigram_overlap = bigram_overlap(sent1, sent2)
new_sample.trigram_overlap = trigram_overlap(sent1, sent2)
new_sample.fourgram_overlap = fourgram_overlap(sent1, sent2)
#get lcs feature
new_sample.lcs = float(max_LCS(sent1, sent2)) / min(
len(sent1), len(sent2))
#get length different feature
new_sample.length_diff = float(abs(len(sent1) - len(sent2))) / min(
len(sent1), len(sent2))
#pos texts
pos1 = nltk.pos_tag(sent1)
pos2 = nltk.pos_tag(sent2)
#filter stop words
pos1_nonstop_word = [
word for word in pos1 if not word[0] in english_stopwords
and not word[0] in english_punctuations
]
pos2_nonstop_word = [
word for word in pos2 if not word[0] in english_stopwords
and not word[0] in english_punctuations
]
#get proper noun, name entity and number overlap feature
new_sample.nnp_overlap = nnp_overlap(pos1, pos2)
new_sample.ne_overlap = ne_overlap(pos1, pos2)
new_sample.num_overlap = num_overlap(pos1, pos2)
#get knowledge based features
new_sample.path_best_sim = path_best_sim(pos1_nonstop_word,
pos2_nonstop_word, pos_map,
pos_list)
new_sample.wup_best_sim = wup_best_sim(pos1_nonstop_word,
pos2_nonstop_word, pos_map,
pos_list)
new_sample.lch_best_sim = lch_best_sim(pos1_nonstop_word,
pos2_nonstop_word, pos_map,
pos_list)
new_sample.lin_best_sim = lin_best_sim(pos1_nonstop_word,
pos2_nonstop_word, brown_ic,
pos_map, pos_list)
new_sample.res_best_sim = res_best_sim(pos1_nonstop_word,
pos2_nonstop_word, brown_ic,
pos_map, pos_list)
new_sample.jcn_best_sim = jcn_best_sim(pos1_nonstop_word,
pos2_nonstop_word, brown_ic,
pos_map, pos_list)
new_sample.dependency_sim = dependency_sim(dependency_line[2],
dependency_line[3])
#add sample to train sample set
train_samples.append(new_sample)
return train_samples
from sample import *
SVJ_mZ1000_mDM20_rinv03_alpha02 = sample()
SVJ_mZ1000_mDM20_rinv03_alpha02.mZ = 1000
SVJ_mZ1000_mDM20_rinv03_alpha02.mDark = 20
SVJ_mZ1000_mDM20_rinv03_alpha02.rinv = 0.3
SVJ_mZ1000_mDM20_rinv03_alpha02.alpha = 0.2
SVJ_mZ1000_mDM20_rinv03_alpha02.sigma = 1
SVJ_mZ1000_mDM20_rinv03_alpha02.color = ROOT.kBlue - 4
SVJ_mZ1000_mDM20_rinv03_alpha02.style = 1
SVJ_mZ1000_mDM20_rinv03_alpha02.fill = 1001
SVJ_mZ1000_mDM20_rinv03_alpha02.leglabel = "SVJ m_{Z}=%d,m_{DM}=%d,r_{inv}=%.1f,#alpha=%.1f" % (
SVJ_mZ1000_mDM20_rinv03_alpha02.mZ, SVJ_mZ1000_mDM20_rinv03_alpha02.mDark,
SVJ_mZ1000_mDM20_rinv03_alpha02.rinv,
SVJ_mZ1000_mDM20_rinv03_alpha02.alpha)
SVJ_mZ1000_mDM20_rinv03_alpha02.label = "SVJ2_mZprime-1000_mDark-20_rinv-0.3_alpha-0.2"
SVJ_mZ2000_mDM20_rinv03_alpha02 = sample()
SVJ_mZ2000_mDM20_rinv03_alpha02.mZ = 2000
SVJ_mZ2000_mDM20_rinv03_alpha02.mDark = 20
SVJ_mZ2000_mDM20_rinv03_alpha02.rinv = 0.3
SVJ_mZ2000_mDM20_rinv03_alpha02.alpha = 0.2
SVJ_mZ2000_mDM20_rinv03_alpha02.sigma = 1
SVJ_mZ2000_mDM20_rinv03_alpha02.color = ROOT.kBlue - 3
SVJ_mZ2000_mDM20_rinv03_alpha02.style = 1
SVJ_mZ2000_mDM20_rinv03_alpha02.fill = 1001
SVJ_mZ2000_mDM20_rinv03_alpha02.leglabel = "SVJ m_{Z}=%d,m_{DM}=%d,r_{inv}=%.1f,#alpha=%.1f" % (
SVJ_mZ2000_mDM20_rinv03_alpha02.mZ, SVJ_mZ2000_mDM20_rinv03_alpha02.mDark,
SVJ_mZ2000_mDM20_rinv03_alpha02.rinv,
SVJ_mZ2000_mDM20_rinv03_alpha02.alpha)
SVJ_mZ2000_mDM20_rinv03_alpha02.label = "SVJ2_mZprime-2000_mDark-20_rinv-0.3_alpha-0.2"