def main(argv):
usage = 'Usage: generate_typescript_interfaces.py --inspector-scripts-path <path to JavaScriptCore/inspector/scripts> --combined-domains-path <path to CombinedDomains.json> --output-dir <result directory path>'
if len(argv) != 6:
print usage
sys.exit(2)
inspector_scripts_path = ''
combined_domains_path = ''
output_dir = ''
try:
opts, args = getopt.getopt(argv,"h",["help", "inspector-scripts-path=","combined-domains-path=", "output-dir="])
except getopt.GetoptError:
print usage
sys.exit(2)
for opt, arg in opts:
if opt in ("-h", "--help"):
print usage
sys.exit()
elif opt == "--inspector-scripts-path":
inspector_scripts_path = arg
elif opt == "--combined-domains-path":
combined_domains_path = arg
elif opt == "--output-dir":
output_dir = arg
sys.path.append(inspector_scripts_path)
from generate import generate
generate(combined_domains_path, output_dir)
def process(clean_word):
w, new = word.get_word(clean_word)
if w.completed():
logging.info('Processing COMPLETED for %s' % clean_word)
return
state = status.get_latest_state(clean_word)
logging.info('PROCESSING %s [%i]' % (w.word(), state))
# Dispatch word to appropriate processing stage.
if state < 10:
analyse.analyse(w, state)
elif state < 20:
generate.generate(w, state)
else:
# Completion state!
state = 808
# Persist latest word payload if needed.
w.persist_payload()
# Queue up next processing step if required.
if not w.completed():
deferred.defer(process, clean_word)
def cleanup(fn):
'''
clean up what rename() might have left, generate mp3s afterwards
'''
for f in glob("./*.mp3"):
os.remove(f)
generate(fn)
def submit():
if request.method == 'POST':
date = request.json["startDate"]
painted = request.json["painted"]
user_full_name = request.json.get("user_full_name", None)
user_email = request.json.get("user_email", None)
print date
print pretty_canvas(painted)
generate(test_repo(), date, painted,
author=user_full_name, email=user_email)
return "OK"
def check_hashes():
modified = False
for f in glob.glob(os.path.join(config.get("sourcedir"), "*.md")):
md5 = hashlib.md5()
md5.update(open(f, 'rb').read())
h = md5.hexdigest()
if file_hashes[f] != h:
file_hashes[f] = h
modified = True
if modified:
print "regenerating"
generate.generate()
def upload_article():
"""上传文件
1. 保存至本地
2. md转换为html
3. 重新加载索引信息
"""
f = request.files["md_file"]
f_name = secure_filename(f.filename)
f.save(os.path.join(INPUT_CONTENT, f_name))
generate()
IndexData.reload_index_data()
return redirect(url_for("page_articles"))
def generate_index():
"""生成索引信息
"""
try:
generate()
IndexData.reload_index_data()
return jsonify({
"msg": "ok"
})
except Exception as e:
current_app.logger.exception(e)
return jsonify({
"msg": "failed"
})
def testAll(self):
for buildout in self.buildouts:
buildout_string = to_string(buildout)
generated_buildout = to_string(render(generate(buildout)))
if buildout_string != generated_buildout:
print "Test Case Failure!"
print "Input"
print etree.tostring(generate(buildout),pretty_print=True)
print buildout_string
print ""
print "!="
print "Generated"
print
print generated_buildout
self.fail()
def main():
global batch_run
for i in range(1):
batch_run = open("batch_run.txt","w")
width = 100
numstr = str(width)
numstr="0"*(3-len(numstr))+numstr
run_parameter(str(numstr))
batch_run.close()
generate.width2 = width
generate.generate(numstr)
# time.sleep(.3)numstr
os.system("cat batch_run.txt | ./1D\ Poisson" )
print numstr
# time.sleep(.1)
os.system("mv Qwell"+str(numstr)+"* energy_data7")
def compile(self, rdql):
from lex import Lexer
from yacc import Parser
self.ast = Parser().parse(rdql, lexer=Lexer())
self.ast = resolve(self.ast)
select, count, self.ast = generate(self.ast)
return select, count
def create_generation(num):
if num <= 0: raise IndexError(str(num) + " is too small. Must be > 0.")
pool = proc.Pool(mp.max_num_workers)
#We now have fitness function output for every program in this directory.
#Lets arrange them by the fittest discarding the stupid ones.
ordered_candidates = filtered_filenames(str(num-1))
partners = choose_partners(ordered_candidates)
#Breeding time.
result = pool.map_async(breed_fittest, partners)
result.wait()
children = result.get()
num_randgen = random.randint(mp.min_new_functions(), mp.max_new_functions())
for i in xrange(num_randgen):
children.append(generate.generate(gp.num_rnd_instrs))
os.mkdir(gen_dir(num))
for child, i in zip(children, xrange(len(children))):
fh = open(gen_member(num, i), 'w')
child = [str(word) for word in child]
child.append("\n")
fh.write(''.join(child))
fh.close()
def main():
return generate(
("Nullable Instance", generate_nullable_instance),
("Type Cache", generate_type_cache),
("Type Casts", generate_type_casts),
("Enum Casts", generate_enum_casts),
)
def main():
return generate(
("LightLambda Run Methods", gen_run_methods),
("Dynamic Instructions", gen_instructions),
("Dynamic Instruction Types", gen_types),
("Untyped Dynamic Instructions", gen_untyped),
)
def setUpClass(cls):
if config.debug:
stream_handler.setLevel(logging.DEBUG)
else:
stream_handler.setLevel(logging.INFO)
cls._randomize_bd()
# Check `config.script_folder` permissions
if (
subprocess.check_output(
config.cmd_env_stat_permissions_s % (config.script_folder),
shell=True).strip()
!= config.script_folder_expected_perms
):
raise DevException(
"Error: give to the http user full permissions to the folder \'%s\'"
% config.script_folder
)
obfuscated = generate(cls.password)
tmp_handler, tmp_path = tempfile.mkstemp()
save_generated(obfuscated, tmp_path)
subprocess.check_call(
config.cmd_env_move_s_s % (tmp_path, cls.path),
shell=True)
subprocess.check_call(
config.cmd_env_chmod_s_s % ('777', cls.path),
shell=True)
def test_build(self):
""" Try to invoke the build target of each demo """
# TODO: please use bouwer modules directly instead? e.g. bouwer.execute()
for lang in os.listdir(self.demodir):
for demo in os.listdir(self.demodir + os.sep + lang):
os.chdir(self.demodir + os.sep + lang + os.sep + demo)
if os.path.exists('generate.py'):
import generate
generate.generate()
result = os.system(self.bouwer + ' -qf')
self.assertEqual(result, 0, 'building ' + os.getcwd() + ' failed')
result = os.system(self.bouwer + ' -c')
self.assertEqual(result, 0, 'cleaning ' + os.getcwd() + ' failed')
def test_generators(self):
for i in range(0, 500):
self._randomize_bd()
obfuscated = generate(self.password)
save_generated(obfuscated, self.path)
self.channel = Channel(self.url, self.password, 'StegaRef')
self._clean_bd()
def initialize():
add_gitignore()
status = generate.generate()
if status["code"] == 0:
print "ERROR: could not generate config.oscm, " + str(status.error_desc)
return 0
# oscm_prompt.prompt()
return 1
def meta_generated(alg): for i in range(1, 10): for _ in range(1, 10): g = generate(i) fvs = alg(g, i) assert fvs != None assert is_fvs(g, fvs) assert len(fvs) == i
def main():
generators = [
('Set Operations (Keys)', gen_ops('DictionaryKeyView')),
('Set Comparison Operations (Keys)', gen_comps('DictionaryKeyView')),
('Set Operations (Items)', gen_ops('DictionaryItemView')),
('Set Comparison Operations (Items)', gen_comps('DictionaryItemView')),
]
return generate(*generators)
def affixate(text):
word_list=token_text(text);
import generate
if len(word_list)==0:
return u'';
else:
for word in word_list:
list_gen_words=generate.generate(word);
return list_gen_words;
def main():
return generate(
("Expression Debugger Proxies", gen_debug_proxies),
("Expression Tree Node Types", gen_tree_nodes),
("Checked Operations", gen_checked_ops),
("Binary Operation Binder Validator", gen_binop_validator),
("Unary Operation Binder Validator", gen_unop_validator),
("StackSpiller Switch", gen_stackspiller_switch),
("Expression Compiler", gen_compiler),
)
def main():
if not os.path.exists(assembly_file):
raise Exception('No file found at {0}'.format(assembly_file))
tokens = []
line_num = 1
with open(assembly_file) as f:
for line in f:
try:
sanitized = sanitize_line(line)
if sanitized:
tokens.append(parse.parse_line(sanitized, line_num))
except parse.ParseException as e:
raise parse.ParseException('Error at line number {0}: {1}, {2}'.format(line_num, line.strip(), str(e)))
line_num += 1
generate.generate(tokens)
def affixate(text):
"""
generate all affixed froms from a word
"""
word_list = token_text(text)
import generate
if len(word_list) == 0:
return u''
else:
for word in word_list:
list_gen_words = generate.generate(word)
return list_gen_words
def main():
gens = [
("ToPython Exception Helper", gen_topython_helper),
("Exception Factories", factory_gen),
("Python New-Style Exceptions", newstyle_gen),
("builtin exceptions", builtin_gen),
]
for e in pythonExcs:
gens.append((get_clr_name(e), gen_one_exception_maker(e)))
return generate(*gens)
def main():
return generate(
("Func Types", gen_func_types),
("Action Types", gen_action_types),
("UpdateAndExecute Methods", gen_update_targets),
("Delegate Action Types", gen_delegate_action),
("Delegate Func Types", gen_delegate_func),
("Maximum Delegate Arity", gen_max_delegate_arity),
# outer ring generators
("Delegate Microsoft Scripting Action Types", gen_delegate_action),
("Delegate Microsoft Scripting Scripting Func Types", gen_delegate_func),
)
def main():
temp_list = [ ("Expression Tree Node Types", gen_tree_nodes),
("Checked Operations", gen_checked_ops),
("Binary Operation Binder Validator", gen_binop_validator),
("Unary Operation Binder Validator", gen_unop_validator),
("StackSpiller Switch", gen_stackspiller_switch),
("Expression Compiler", gen_compiler)
]
import System
if System.Environment.Version.Major<4:
temp_list.append(("Expression Debugger Proxies", gen_debug_proxies))
return generate(*temp_list)
def setUpClass(cls):
cls._randomize_bd()
obfuscated = generate(cls.password)
tmp_handler, tmp_path = tempfile.mkstemp()
save_generated(obfuscated, tmp_path)
subprocess.check_call(
config.cmd_env_move_s_s % (tmp_path, cls.path),
shell=True)
subprocess.check_call(
config.cmd_env_chmod_s_s % ('777', cls.path),
shell=True)
def main():
# add other generation functions to this list
generate.register_generation({
"basic" : easy1.generate,
"who" : who.generate,
"whose" : who.generate_plural,
"which" : which.generate#,
})
fname = sys.argv[1]
nquestions = int(sys.argv[2])
paras = []
f = open(fname)
line = f.readline()
# take only real paragraphs, not titles or tables
while line != "":
if len(line) > 70:
paras += [line]
line = f.readline()
f.close()
all_qs = []
qs = []
while (len(qs) < nquestions) and (len(paras) > 0):
i = random.randint(0, len(paras) - 1)
p = paras.pop(i)
parsed = parse.parse(p)
all_qs += filter(lambda x: (type(x) == tuple) and
(x[0] != None) and
(lib.wc(x[0]) > 5),
generate.generate(parsed))
flips = [random.randint(0, 2) for i in xrange(len(all_qs))]
new_qs = []
for i in xrange(len(flips)):
if flips[i] == 0:
qs += [all_qs[i][0]]
else:
new_qs += [all_qs[i]]
all_qs = new_qs
lib.pretty_print(qs)
lib.pretty_print(all_qs)
questions = map (lib.format, qs)
for q in questions:
print q
def generate_model(p=30, p_zero=0.7, type=None):
DEBUG = True
'''
this function creates a model in order to generate
data according to it's distribution.
and in order to compare the approximated model to the original
distribution
'''
if not type:
adjmat = random_graph(p, p_zero)
if type == 'grid':
adjmat = grid_graph(p)
adjmat.nodes()
adjmat.nodes_iter()
gen = generate(adj_mat=adjmat)
return gen
def main():
return generate(
("Python Constant Folding", gen_constant_folding),
("Python Fast Ops RetBool Chooser", fast_op_ret_bool_chooser),
("Python Fast Ops Ret Bool", fast_op_ret_bool),
("Tokenize Ops", tokenize_generator),
("Token Kinds", tokenkinds_generator),
("Tokens", tokens_generator),
("Table of Operators", gen_OperatorTable),
("PythonOperator Mapping", gen_operatorMapping),
#("OperatorToSymbol", gen_OperatorToSymbol),
("StringOperatorToSymbol", gen_StringOperatorToSymbol),
("WeakRef Operators Initialization", weakref_operators),
("OldInstance Operators", oldinstance_operators),
#("Operator Reversal", operator_reversal),
("WeakRef Callable Proxy Operators Initialization", weakrefCallabelProxy_operators),
)
def main():
return generate(
("IntOps", gen_all),
)
# File: randomWalk.py import sys from generate import generate n = int(sys.argv[2]) a = int(sys.argv[1]) output = generate(a, n) print output
def main():
args = get_config()
#---display model type---#
print('-' * 80)
print('# Training Conditional SpecGAN!') if args.conditional else print(
'# Training SpecGAN!')
print('-' * 80)
#---make train dir---#
if args.conditional: args.train_dir = args.train_dir + '_cond'
if not os.path.isdir(args.train_dir):
os.makedirs(args.train_dir)
#---save args---#
with open(os.path.join(args.train_dir, 'args.txt'), 'w') as f:
f.write('\n'.join([
str(k) + ',' + str(v)
for k, v in sorted(vars(args).items(), key=lambda x: x[0])
]))
#---make model kwarg dicts---#
setattr(
args, 'SpecGAN_g_kwargs', {
'kernel_len': args.SpecGAN_kernel_len,
'dim': args.SpecGAN_dim,
'use_batchnorm': args.SpecGAN_batchnorm,
'upsample': args.SpecGAN_genr_upsample,
'initializer': args.SpecGAN_model_initializer,
})
setattr(
args, 'SpecGAN_d_kwargs', {
'kernel_len': args.SpecGAN_kernel_len,
'dim': args.SpecGAN_dim,
'use_batchnorm': args.SpecGAN_batchnorm,
'initializer': args.SpecGAN_model_initializer,
})
#---collect path to data---#
if args.mode == 'train' or args.mode == 'moments':
fps = glob.glob(
os.path.join(args.data_dir, args.data_tfrecord_prefix) +
'*.tfrecord')
#---load moments---#
if args.mode != 'moments' and args.data_moments_file is not None:
while True:
try:
print('# Moments: Loading existing moments file...')
with open(os.path.join(args.train_dir, args.data_moments_file),
'rb') as f:
_mean, _std = pickle.load(f)
break
except:
print(
'# Moments: Failed to load, computing new moments file...')
moments(fps, args)
setattr(args, 'data_moments_mean', _mean)
setattr(args, 'data_moments_std', _std)
#---run selected mode---#
#---run generate mode--#
if args.mode == 'train':
infer(args, cond=args.conditional)
train(fps, args, cond=args.conditional)
elif args.mode == 'generate':
infer(args, cond=args.conditional)
generate(args, cond=args.conditional)
elif args.mode == 'moments':
moments(fps, args)
elif args.mode == 'preview':
preview(args)
elif args.mode == 'incept':
incept(args)
elif args.mode == 'infer':
infer(args)
else:
raise NotImplementedError()
def main():
return generate(
("IntOps", gen_int),
("FloatOps", gen_float),
)
def main():
return generate(
("Tuples", gen_tuples),
("Tuple Get From Size", gen_get_size),
)
else:
device = 'cpu'
print(' Found ' + device + '.')
print('>> Loading perturbation...')
# generate perturbation v of 224*224*3 of [-10,10] directly on original image.
file_perturbation = 'data/universal.npy'
if os.path.isfile(file_perturbation) == 0:
print(' No perturbation found, computing...')
print('>> Checking dataset...')
if not os.path.exists(args.PATH):
print("Data set not found. please check!")
sys.exit()
print(' Done.')
v = generate(args.PATH, 'dataset4u-trn.txt', 'dataset4u-val.txt', net, max_iter_uni=10, delta=0.1, p=np.inf, num_classes=25, overshoot=0.1, max_iter_df=500, xi=args.xi, batch_size=args.batch_size)
# Saving the universal perturbation
np.save('./data/universal.npy', v)
else:
print(' Found a pre-computed universal perturbation at', file_perturbation)
v = np.load(file_perturbation)
testimg = "./data/test_im4.jpg"
print('>> Testing the universal perturbation on', testimg)
labels = open('./data/labels.txt', 'r').read().split('\n')
testimgToInput = Image.open(testimg).convert('RGB')
pertimgToInput = np.clip(cut(testimgToInput)+v, 0, 255)
pertimg = Image.fromarray(pertimgToInput.astype(np.uint8))
img_orig = transform(testimgToInput)
[p[1] for p in dataset if p[2] < 0], 'ro')
pylab.contour(xrange,
yrange,
grid, (-1.0, 0.0, 1.0),
colors=('red', 'black', 'blue'),
linewidths=(1, 1, 1))
pylab.show()
if __name__ == '__main__':
# datasets = generate.generate3(points=25)
# kernels = (
# linear_kernel,
# lambda x, y: poly_kernel(x, y, 4),
# lambda x, y: radial_kernel(x, y, 0.54)
# #lambda x, y: sigmoid_kernel(x, y, 0.1, 0.3)
# )
# for d in datasets:
# for k in kernels:
# continue
# plot_classification(d, k)
# dataset = generate.generate(points=25)
# for g in range(2, 10, 2):
# plot_classification(dataset, lambda x, y: radial_kernel(x, y, g))
data = generate.generate(points=25)
for p in range(5, 10):
plot_classification(data, lambda x, y: poly_kernel(x, y, p))
if query_result == 1:
print("Target found at [%s,%s], %s." %
(current_i, current_j, map[current_i][current_j]))
print("Totoal Search Steps:%d" % search_count)
return search_count, move_count
else:
# print("Search [%d,%d]" % (query_i,query_j))
makeMove(map, beliefmap, current_i, current_j)
for i in range(dim):
for j in range(dim):
if beliefmap[i][j] < 0:
raise Exception("ERROR")
if __name__ == '__main__':
game = generate(50)
map = game.get('board')
target_x = game.get('target').row
target_y = game.get('target').col
searchstep1 = 0
searchstep2 = 0
movestep1 = 0
movestep2 = 0
policy = 1
for i in range(10):
searchsteps, movesteps = probablisticSearch(map, target_x, target_y, 1)
searchstep1 += searchsteps
movestep1 += movesteps
print(searchsteps, movesteps)
import argparse
import generate
parser = argparse.ArgumentParser()
parser.add_argument('tag', help='The desired clan tag')
if __name__ == '__main__':
args = parser.parse_args()
generate.generate(args.tag)
def create_tweet():
"""Create the text of the tweet you want to send."""
text = generate()
return text
def main():
return generate(
("Python AST Walker", gen_python_walker),
("Python AST Walker Nonrecursive", gen_python_walker_nr),
)
parser.add_argument('-d',
'--dataset-path',
type=str,
help='The dataset to use')
parser.add_argument('-dt',
'--dataset-type',
type=str,
default='cars',
help='The dataset to use')
FLAGS, unparsed = parser.parse_known_args()
# Check if cuda is available
FLAGS.cuda = FLAGS.cuda and torch.cuda.is_available()
# Get the desired pretrained models for the dataset
'''
if not FLAGS.dataset is None:
FLAGS.dpath = '/'.join(FLAGS.dpath.split('/')[:2] + [FLAGS.dataset] + [FLAGS.dpath.split('/')[3]])
FLAGS.gpath = '/'.join(FLAGS.gpath.split('/')[:2] + [FLAGS.dataset] + [FLAGS.gpath.split('/')[3]])
'''
if FLAGS.mode == 'train':
train(FLAGS)
elif FLAGS.mode == 'predict':
generate(FLAGS)
else:
raise RuntimeError('Invalid value passed for mode. \
Valid arguments are: "train" and "test"')
def main():
return generate(*GetGeneratorList())
def main():
return generate(("Symbols - Other Symbols", generate_symbols), )
from keras.models import load_model
from keras.datasets import fashion_mnist
import numpy as np
from process.ssim import get_ssim
from generate import generate
# load model
model = load_model("model.hdf5")
# load test data and labels
(train_data, train_labels), (t_data, test_labels) = fashion_mnist.load_data()
test_data = np.load("test_data/test_data.npy")
generate_data = test_data
# generate attack data
attack_data = generate(generate_data, (len(generate_data), 28, 28, 1))
# judge where attack success
def attack_success(prev, attack):
prev = prev.reshape((1, prev.shape[0], prev.shape[1], prev.shape[2]))
attack = attack.reshape(
(1, attack.shape[0], attack.shape[1], attack.shape[2]))
prev_pred = np.argmax(model.predict(prev))
current_pred = np.argmax(model.predict(attack))
return prev_pred != current_pred
# attack success
success_count = 0
success_index = []
def main():
return generate(("Reflected Caller", gen_all), )
#on chip memory test
genModules = generate(); #store all names of generated modules
genModules.onChipMem("d","wren","clk","write_addr","read_addr","q",newFile)
newFile.writeTopModule(genModules) #connect all generated modules inside a top_module
"""
if __name__ == "__main__":
"""
ring osc prompt
"""
#creates new file with name "ro.sv"
newFile = writeToFile("ro.sv")
#store all names of generated modules
genModules = generate()
#ask user for number of ring oscillators
while True:
try:
c = int(input("Please enter the number of ring oscillators:"))
except ValueError or c <= 0:
continue
else:
break
#for each ring oscillator, ask user for specifications
for i in range(int(c)):
print("\nRO #{num}:".format(num=str(i + 1)))
in_port = input(
from rule import Rule
from generate import generate
from validate import validate
# 已知:有斑点、长脖子、长腿、有奶、有蹄。 询问:这是什么动物呢?
if __name__ == '__main__':
file = open('input.txt', 'r')
list_lines = file.readlines()
target = list_lines[0].split(' ')
rules = []
num = int(list_lines[1])
for i in range(2, 2 + num):
rule = list_lines[i]
condition, result = rule.split('#')
condition = condition.split(' ')
result = result[:-1]
rules.append(Rule(condition, result))
num2 = int(list_lines[2 + num])
for i in range(2 + num + 1, 2 + num + 1 + num2):
conds = list_lines[i].split(' ')
conds[-1] = conds[-1][:-1]
print('generate:')
generate(target, rules, conds)
print('')
print('validate:')
validate(target, rules, conds)
def main():
return generate(
("Python AST Walker", gen_python_walker),
("Python AST Walker Nonrecursive", gen_python_walker_nr),
("Python Name Binder Propagate Current Scope", gen_python_name_binder),
)
def solve(opts, msg):
if opts['generate']:
logging.info('Operation started: generation.')
import sys
sys.path.append('./lib')
from generate import generate
count = 0
for each in generate()[0]:
letters[count] = each[0]
count += 1
ALF = generate()[1]
print letters
print ALF
logging.info('SUCCEEDED: puzzle generation.')
if opts['publish']:
# 140 character limit
# 22 characters
import time
import os
date = time.strftime("%x")
header = 'Beehive for ' + date + ':' + '\n'
hive = [0] * 5
hive[0] = ' \ ' + str(letters[1]).upper() + ' /' + '\n'
hive[1] = str(letters[2]).upper() + ' \ / ' + str(
letters[3]).upper() + '\n'
hive[2] = '---( ' + str(letters[0]).upper() + ' )---' + '\n'
hive[3] = str(letters[4]).upper() + ' / \ ' + str(
letters[5]).upper() + '\n'
hive[4] = ' / ' + str(letters[6]).upper() + ' \\'
footer = '\nALF: ' + str("%.2f" % ALF)
if len(msg) > 1:
top_message = msg
payload = top_message + '\n' + header + hive[0] + hive[1] + hive[
2] + hive[3] + hive[4] + footer
else:
payload = header + hive[0] + hive[1] + hive[2] + hive[3] + hive[
4] + footer
print payload
logging.info('SUCCEEDED: Twitter publication.')
if opts['solution']:
logging.info('Operation started: solution generation.')
f = open('./lib/wordlist.txt', 'r')
uncheck_words = []
letter_anag = ''
for each in letters:
letter_anag += each
# Can you think of any word with the same letter appearing more than 5 times?
letter_anag += letter_anag + letter_anag + letter_anag + letter_anag + letter_anag
def anagramchk(word, chkword):
for letter in word:
if letter in chkword:
chkword = chkword.replace(letter, '', 1)
else:
return 0
return 1
for line in f:
for line in f:
word = line.strip()
if anagramchk(word, letter_anag):
uncheck_words.append(word)
f.close()
logging.info('SUCCEEDED: Unchecked word solutions.')
logging.info('Operation started: Checking word solutions for reqs.')
words = []
target_letter = set(letters[0])
for each in uncheck_words:
if target_letter & set(each):
words.append(each)
solutions_file = './solutions/beehive' + str(date.replace('/',
'')) + '.txt'
f = open(solutions_file, 'w')
f.write('Daily Beehive solution for ' + date + ':\n')
f.close()
f = open(solutions_file, 'a')
for each in words:
f.write(each + '\n')
f.close()
logging.info('SUCCEEDED: Solution generation accounting for reqs.')
logging.info('Operation started: Uploading solution via git')
os.system('git add ' + solutions_file)
os.system('git commit -m "added solutions file"')
os.system('git push origin master')
logging.info('SUCCEEDED: Uploaded solutions to GitHub.')
if opts['twitter']:
logging.info('Operation started: Twitter publication.')
payload += '\nSolved: https://raw.githubusercontent.com/aaronsdevera/dailybeehive/master/prod/solutions/beehive' + str(
date.replace('/', '')) + '.txt'
import twitter
from twitter_api_keys import KEYS
api = twitter.Api(consumer_key=KEYS[0],
consumer_secret=KEYS[1],
access_token_key=KEYS[2],
access_token_secret=KEYS[3])
status = api.PostUpdate(payload)
logging.info('SUCCEEDED: Posted to Twitter.')
def generate_midi(config_folder, score_source, number_of_version, duration_gen,
rhythmic_reconstruction, logger_generate):
"""This function generate the orchestration of a midi piano score
Parameters
----------
config_folder : str
Absolute path to the configuration folder, i.e. the folder containing the saved model and the results
score_source : str
Either a path to a folder containing two midi files (piano and orchestration) or the path toa piano midi files
number_of_version : int
Number of version generated in a batch manner. Since the generation process involves sampling it might be interesting to generate several versions
duration_gen : int
Length of the generated score (in number of events). Useful for generating only the beginning of the piece.
rhythmic_reconstruction: bool
Whether rythmic reconstrcution from event-level representation to frame-level reconstrcution is performed or not. If true is selected, the rhtyhmic structure of the original piano score is used.
logger_generate : logger
Instanciation of logging. Can be None
"""
logger_generate.info("#############################################")
logger_generate.info("Orchestrating piano score : " + score_source)
############################################################
# Load model, config and data
############################################################
########################
# Load config and model
parameters = pkl.load(open(config_folder + '/script_parameters.pkl', 'rb'))
model_parameters = pkl.load(open(config_folder + '/model_params.pkl',
'rb'))
# Set a minimum seed size, because for very short models you don't event see the beginning
seed_size = max(model_parameters['temporal_order'], 10) - 1
quantization = parameters['quantization']
temporal_granularity = parameters['temporal_granularity']
instru_mapping = parameters['instru_mapping']
########################
########################
# Load data
if re.search(r'mid$', score_source):
pr_piano, event_piano, duration_piano, name_piano, pr_orch, instru_orch, duration = load_solo(
score_source, quantization, parameters["binarize_piano"],
temporal_granularity)
else:
pr_piano, event_piano, duration_piano, name_piano, pr_orch, instru_orch, duration = load_from_pair(
score_source, quantization, parameters["binarize_piano"],
parameters["binarize_orch"], temporal_granularity)
########################
########################
# Shorten
# Keep only the beginning of the pieces (let's say a 100 events)
pr_piano = extract_pianoroll_part(pr_piano, 0, duration_gen)
if parameters["duration_piano"]:
duration_piano = np.asarray(duration_piano[:duration_gen])
else:
duration_piano = None
event_piano = event_piano[:duration_gen]
pr_orch = extract_pianoroll_part(pr_orch, 0, duration_gen)
########################
########################
# Instanciate piano pianoroll
N_piano = instru_mapping['Piano']['index_max']
pr_piano_gen = np.zeros((duration_gen, N_piano), dtype=np.float32)
pr_piano_gen = build_data_aux.cast_small_pr_into_big_pr(
pr_piano, {}, 0, duration_gen, instru_mapping, pr_piano_gen)
pr_piano_gen_flat = pr_piano_gen.sum(axis=1)
silence_piano = [
e for e in range(duration_gen) if pr_piano_gen_flat[e] == 0
]
########################
########################
# Instanciate orchestra pianoroll with orchestra seed
N_orchestra = parameters['N_orchestra']
if pr_orch:
pr_orchestra_gen = np.zeros((seed_size, N_orchestra), dtype=np.float32)
orch_seed_beginning = {k: v[:seed_size] for k, v in pr_orch.items()}
pr_orchestra_gen = build_data_aux.cast_small_pr_into_big_pr(
orch_seed_beginning, instru_orch, 0, seed_size, instru_mapping,
pr_orchestra_gen)
pr_orchestra_truth = np.zeros((duration_gen, N_orchestra),
dtype=np.float32)
pr_orchestra_truth = build_data_aux.cast_small_pr_into_big_pr(
pr_orch, instru_orch, 0, duration_gen, instru_mapping,
pr_orchestra_truth)
else:
pr_orchestra_gen = None
pr_orchestra_truth = None
########################
#######################################
# Embed piano
time_embedding = time.time()
if parameters['embedded_piano']:
# Load model
embedding_path = parameters["embedding_path"]
embedding_model = embedDenseNet(380, 12, (1500, 500), 100, 1500, 2, 3,
12, 0.5, 0, False, True)
embedding_model.load_state_dict(torch.load(embedding_path))
# Build embedding (no need to batch here, len(pr_piano_gen) is sufficiently small)
piano_resize_emb = np.zeros(
(len(pr_piano_gen), 1, 128)) # Embeddings accetp size 128 samples
piano_resize_emb[:, 0, instru_mapping['Piano']['pitch_min']:
instru_mapping['Piano']['pitch_max']] = pr_piano_gen
piano_resize_emb_TT = torch.tensor(piano_resize_emb)
piano_embedded_TT = embedding_model(piano_resize_emb_TT.float(), 0)
pr_piano_gen_embedded = piano_embedded_TT.numpy()
else:
pr_piano_gen_embedded = pr_piano_gen
time_embedding = time.time() - time_embedding
#######################################
########################
# Inputs' normalization
normalizer = pkl.load(
open(os.path.join(config_folder, 'normalizer.pkl'), 'rb'))
if parameters["embedded_piano"]: # When using embedding, no normalization
pr_piano_gen_norm = pr_piano_gen_embedded
else:
pr_piano_gen_norm = normalizer.transform(pr_piano_gen_embedded)
########################
########################
# Store folder
string = re.split(r'/', name_piano)[-1]
name_track = re.sub('piano_solo.mid', '', string)
generated_folder = config_folder + '/generation_reference_example/' + name_track
if not os.path.isdir(generated_folder):
os.makedirs(generated_folder)
########################
########################
# Get trainer
with open(os.path.join(config_folder, 'which_trainer'), 'r') as ff:
which_trainer = ff.read()
# Trainer
if which_trainer == 'standard_trainer':
from LOP.Scripts.standard_learning.standard_trainer import Standard_trainer as Trainer
kwargs_trainer = {'temporal_order': model_parameters["temporal_order"]}
elif which_trainer == 'NADE_trainer':
from LOP.Scripts.NADE_learning.NADE_trainer import NADE_trainer as Trainer
kwargs_trainer = {
'temporal_order': model_parameters["temporal_order"],
'num_ordering': model_parameters["num_ordering"]
}
else:
raise Exception("Undefined trainer")
trainer = Trainer(**kwargs_trainer)
########################
############################################################
# Generate
############################################################
time_generate_0 = time.time()
generated_sequences = {}
for measure_name in parameters['save_measures']:
model_path = 'model_' + measure_name
generated_sequences[measure_name] = generate(
trainer,
pr_piano_gen_norm,
silence_piano,
duration_piano,
config_folder,
model_path,
pr_orchestra_gen,
batch_size=number_of_version)
time_generate_1 = time.time()
logger_generate.info(
'TTT : Generating data took {} seconds'.format(time_generate_1 -
time_generate_0))
############################################################
# Reconstruct and write
############################################################
def reconstruct_write_aux(generated_sequences, prefix):
for write_counter in range(generated_sequences.shape[0]):
# To distinguish when seed stop, insert a sustained note
this_seq = generated_sequences[write_counter] * 127
this_seq[:seed_size, 0] = 20
# Reconstruct
if rhythmic_reconstruction:
pr_orchestra_clean = from_event_to_frame(this_seq, event_piano)
else:
pr_orchestra_clean = this_seq
pr_orchestra = instrument_reconstruction(pr_orchestra_clean,
instru_mapping)
# Write
write_path = generated_folder + '/' + prefix + '_' + str(
write_counter) + '_generated.mid'
if rhythmic_reconstruction:
write_midi(pr_orchestra, quantization, write_path, tempo=80)
else:
write_midi(pr_orchestra, 1, write_path, tempo=80)
return
for measure_name in parameters["save_measures"]:
reconstruct_write_aux(generated_sequences[measure_name], measure_name)
############################################################
############################################################
# Write original orchestration and piano scores, but reconstructed version, just to check
if rhythmic_reconstruction:
A = from_event_to_frame(pr_piano_gen, event_piano)
else:
A = pr_piano_gen
B = A * 127
piano_reconstructed = instrument_reconstruction_piano(B, instru_mapping)
write_path = generated_folder + '/piano_reconstructed.mid'
if rhythmic_reconstruction:
write_midi(piano_reconstructed, quantization, write_path, tempo=80)
else:
write_midi(piano_reconstructed, 1, write_path, tempo=80)
#
if rhythmic_reconstruction:
A = from_event_to_frame(pr_orchestra_truth, event_piano)
else:
A = pr_orchestra_truth
B = A * 127
orchestra_reconstructed = instrument_reconstruction(B, instru_mapping)
write_path = generated_folder + '/orchestra_reconstructed.mid'
if rhythmic_reconstruction:
write_midi(orchestra_reconstructed, quantization, write_path, tempo=80)
else:
write_midi(orchestra_reconstructed, 1, write_path, tempo=80)
def main():
return generate.generate(
("Exception Factory", gen_expr_factory_core),
("Com Exception Factory", gen_expr_factory_com),
("Microsoft.Scripting Exception Factory", gen_expr_factory_scripting),
)
def main():
return generate(
("math functions", gen_funcs)
)
alg_pars = {}
alg_pars['densest_eps'] = 0.1
alg_pars['dp_eps'] = 0.1
res = np.empty((len(innerdegree_range), len(noise_range), 9))
for j in xrange(len(innerdegree_range)):
generator_pars['innerdegree'] = innerdegree_range[j]
print 'inner noise (avg. degree of the planted graph):', innerdegree_range[
j]
for i in xrange(len(noise_range)):
print 'outer noise (avg. degree of the background network):', noise_range[
i]
noise = noise_range[i]
generator_pars['noise'] = noise_range[i]
TS, backNoise, innerNoise, generated_C = generate.generate(
generator_pars)
st = time.time()
ADP = ApproxDP(alg_pars['dp_eps'], generator_pars['k'], TS)
ADP.run_DP(alg_pars['densest_eps'])
el_time = time.time() - st
graphs, densities, intervals = ADP.get_sol_graphs()
nq = get_node_quality(graphs, generated_C)
iq = get_interval_quality(intervals, generated_C)
res[j, i, 0:3] = nq
res[j, i, 3:6] = iq
res[j, i, 6] = sum(densities)
print 'total density', res[j, i, 6]
print
res[j, i, 7] = get_true_density(generated_C)
res[j, i, 8] = el_time
from generate import generate file_name = "outofsample1.csv" # file_name = "oos_w_fund_mom.csv" generate(file_name, 2008) generate(file_name, 2009) generate(file_name, 2010) generate(file_name, 2011) generate(file_name, 2012)
def main():
return generate(
("TypeCache Storage", gen_typecache_storage),
("TypeCache Entries", gen_typecache),
)
def generate_midi(config_folder_fd, config_folder_bd, score_source,
save_folder, initialization_type, number_of_version,
duration_gen, number_fdbd_pass, logger_generate):
"""This function generate the orchestration of a midi piano score
Parameters
----------
config_folder : str
Absolute path to the configuration folder, i.e. the folder containing the saved model and the results
score_source : str
Either a path to a folder containing two midi files (piano and orchestration) or the path toa piano midi files
number_of_version : int
Number of version generated in a batch manner. Since the generation process involves sampling it might be interesting to generate several versions
duration_gen : int
Length of the generated score (in number of events). Useful for generating only the beginning of the piece.
logger_generate : logger
Instanciation of logging. Can be None
"""
logger_generate.info("#############################################")
logger_generate.info("Orchestrating : " + score_source)
# Load parameters
parameters = pkl.load(
open(config_folder_fd + '/script_parameters.pkl', 'rb'))
model_parameters_fd = pkl.load(
open(config_folder_fd + '/model_params.pkl', 'rb'))
parameters_bd = pkl.load(
open(config_folder_bd + '/script_parameters.pkl', 'rb'))
model_parameters_bd = pkl.load(
open(config_folder_bd + '/model_params.pkl', 'rb'))
assert (model_parameters_fd["temporal_order"] ==
model_parameters_bd["temporal_order"]
), "The two model have different seed_size"
assert (parameters["quantization"] == parameters_bd["quantization"]
), "The two model have different quantization"
assert (parameters["temporal_granularity"] ==
parameters_bd["temporal_granularity"]
), "The two model have different temporal_granularity"
assert (parameters["instru_mapping"] == parameters_bd["instru_mapping"]
), "The two model have different instru_mapping"
assert (parameters["normalizer"] == parameters_bd["normalizer"]
), "The two model have different normalizer"
seed_size = max(model_parameters_fd['temporal_order'], 10) - 1
#######################
# Load data
if re.search(r'mid$', score_source):
pr_piano, event_piano, duration_piano, name_piano, pr_orch, instru_orch, duration = generation_utils.load_solo(
score_source, parameters["quantization"],
parameters["binarize_piano"], parameters["temporal_granularity"])
else:
if initialization_type == "seed":
pr_piano, event_piano, duration_piano, name_piano, pr_orch, instru_orch, duration = generation_utils.load_from_pair(
score_source,
parameters["quantization"],
parameters["binarize_piano"],
parameters["binarize_orch"],
parameters["temporal_granularity"],
align_bool=True)
else:
pr_piano, event_piano, duration_piano, name_piano, pr_orch, instru_orch, duration = generation_utils.load_from_pair(
score_source,
parameters["quantization"],
parameters["binarize_piano"],
parameters["binarize_orch"],
parameters["temporal_granularity"],
align_bool=False)
if (duration is None) or (duration < duration_gen):
logger_generate.info("Track too short to be used")
return
########################
########################
# Shorten
# Keep only the beginning of the pieces (let's say a 100 events)
pr_piano = pianoroll_processing.extract_pianoroll_part(
pr_piano, 0, duration_gen)
if parameters["duration_piano"]:
duration_piano = np.asarray(duration_piano[:duration_gen])
else:
duration_piano = None
if parameters["temporal_granularity"] == "event_level":
event_piano = event_piano[:duration_gen]
pr_orch = pianoroll_processing.extract_pianoroll_part(
pr_orch, 0, duration_gen)
########################
########################
# Instanciate piano pianoroll
N_piano = parameters["instru_mapping"]['Piano']['index_max']
pr_piano_gen = np.zeros((duration_gen, N_piano), dtype=np.float32)
pr_piano_gen = build_data_aux.cast_small_pr_into_big_pr(
pr_piano, {}, 0, duration_gen, parameters["instru_mapping"],
pr_piano_gen)
pr_piano_gen_flat = pr_piano_gen.sum(axis=1)
silence_piano = [
e for e in range(duration_gen) if pr_piano_gen_flat[e] == 0
]
########################
########################
# Initialize orchestra pianoroll with orchestra seed (choose one)
N_orchestra = parameters['N_orchestra']
pr_orchestra_truth = np.zeros((duration_gen, N_orchestra),
dtype=np.float32)
pr_orchestra_truth = build_data_aux.cast_small_pr_into_big_pr(
pr_orch, instru_orch, 0, duration_gen, parameters["instru_mapping"],
pr_orchestra_truth)
if initialization_type == "seed":
pr_orchestra_seed = generation_utils.init_with_seed(
pr_orch, number_of_version, seed_size, N_orchestra, instru_orch,
parameters["instru_mapping"])
elif initialization_type == "zeros":
pr_orchestra_seed = generation_utils.init_with_zeros(
number_of_version, seed_size, N_orchestra)
elif initialization_type == "constant":
const_value = 0.1
pr_orchestra_seed = generation_utils.init_with_constant(
number_of_version, seed_size, N_orchestra, const_value)
elif initialization_type == "random":
proba_activation = 0.5
pr_orchestra_seed = generation_utils.init_with_random(
number_of_version, seed_size, N_orchestra, proba_activation)
########################
#######################################
# Embed piano
time_embedding = time.time()
if parameters['embedded_piano']:
# Load model
embedding_path = parameters["embedding_path"]
embedding_model = torch.load(embedding_path, map_location="cpu")
# Build embedding (no need to batch here, len(pr_piano_gen) is sufficiently small)
# Plus no CUDA here because : afradi of mix with TF + possibly very long piano chunks
piano_resize_emb = np.zeros(
(len(pr_piano_gen), 1, 128)) # Embeddings accetp size 128 samples
piano_resize_emb[:, 0, parameters["instru_mapping"]['Piano']
['pitch_min']:parameters["instru_mapping"]['Piano']
['pitch_max']] = pr_piano_gen
piano_resize_emb_TT = torch.tensor(piano_resize_emb)
piano_embedded_TT = embedding_model(piano_resize_emb_TT.float(), 0)
pr_piano_gen_embedded = piano_embedded_TT.numpy()
else:
pr_piano_gen_embedded = pr_piano_gen
time_embedding = time.time() - time_embedding
#######################################
########################
# Inputs' normalization
normalizer = pkl.load(
open(os.path.join(config_folder_fd, 'normalizer.pkl'), 'rb'))
if parameters["embedded_piano"]: # When using embedding, no normalization
pr_piano_gen_norm = pr_piano_gen_embedded
else:
pr_piano_gen_norm = normalizer.transform(pr_piano_gen_embedded)
########################
########################
# Store folder
string = re.split(r'/', name_piano)[-1]
name_track = re.sub('piano_solo.mid', '', string)
generated_folder = save_folder + '/fd_bd_' + initialization_type + '_init/' + name_track
if not os.path.isdir(generated_folder):
os.makedirs(generated_folder)
########################
########################
# Get trainer
with open(os.path.join(config_folder_fd, 'which_trainer'), 'r') as ff:
which_trainer_fd = ff.read()
# Trainer
trainer_fd = import_trainer(which_trainer_fd, model_parameters_fd,
parameters)
#
with open(os.path.join(config_folder_bd, 'which_trainer'), 'r') as ff:
which_trainer_bd = ff.read()
# Trainer
trainer_bd = import_trainer(which_trainer_bd, model_parameters_bd,
parameters)
########################
############################################################
# Generate
############################################################
time_generate_0 = time.time()
for pass_index in range(number_fdbd_pass):
model_path = 'model_accuracy'
pr_orchestra_gen = generate(trainer_fd,
pr_piano_gen_norm,
silence_piano,
duration_piano,
config_folder_fd,
model_path,
pr_orchestra_seed,
batch_size=number_of_version)
pr_orchestra_seed = pr_orchestra_gen[:, -seed_size:]
prefix_name = 'fd_' + str(pass_index) + '_'
generation_utils.reconstruct_generation(pr_orchestra_gen, event_piano,
generated_folder, prefix_name,
parameters, seed_size)
pr_orchestra_gen = generate_backward(trainer_bd,
pr_piano_gen_norm,
silence_piano,
duration_piano,
config_folder_bd,
model_path,
pr_orchestra_seed,
batch_size=number_of_version)
pr_orchestra_seed = pr_orchestra_gen[:, :seed_size]
prefix_name = 'bd_' + str(pass_index) + '_'
generation_utils.reconstruct_generation(pr_orchestra_gen, event_piano,
generated_folder, prefix_name,
parameters, seed_size)
time_generate_1 = time.time()
logger_generate.info(
'TTT : Generating data took {} seconds'.format(time_generate_1 -
time_generate_0))
############################################################
# Reconstruct and write
############################################################
prefix_name = 'final_'
generation_utils.reconstruct_generation(pr_orchestra_gen, event_piano,
generated_folder, prefix_name,
parameters, seed_size)
generation_utils.reconstruct_original(pr_piano_gen, pr_orchestra_truth,
event_piano, generated_folder,
parameters)
return
input_size = vocab.size
output_size = vocab.size
model = LanguageModel('RNN', input_size, hidden_size, output_size)
# create criterion and optimiser
criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(model.parameters(), lr=0.001)
# training loop
every_n = int(total_n / 100)
running_loss = 0
losses = []
for i, (batch, labels) in enumerate(
generate.generate('train',
token=token,
max_len=max_len,
small=small)):
# one hot encode
if token == 'character':
batch = generate.one_hot_encode(batch, vocab)
# or embed
elif token == 'word':
batch = generate.w2v_encode(batch, emb, vocab)
# turn into torch tensors
batch = torch.Tensor(batch)
labels = torch.Tensor(labels).long()
# zero the gradients
optimizer.zero_grad()
def update_index():
from generate import generate
generate()
from generate import generate
import os
import subprocess
import sys
if (not os.path.exists("build")):
os.mkdir("build")
else:
for r, _, fl in os.walk("build"):
for f in fl:
os.remove(os.path.join(r, f))
if ("--generate" in sys.argv):
print("Generating SHA1 Checking Code...")
generate()
tmp = os.getcwd()
os.chdir("build")
if ("--gpu" in sys.argv):
if ("--release" in sys.argv):
if (subprocess.run([
"nvcc", "-O3", "-Xptxas", "-v,-O3", "-use_fast_math", "-D",
"_NVCC", "-D", "NDEBUG", "-D", "_WINDOWS", "-D", "_UNICODE",
"-D", "UNICODE", "-o", "gpu_cracker.exe",
"../src/gpu_cracker/gpu_cracker.cu", "-I",
"../src/gpu_cracker/include"
]).returncode != 0):
os.chdir(tmp)
sys.exit(1)
else:
if (subprocess.run([
"nvcc", "-G", "-use_fast_math", "-D", "_NVCC", "-D", "_DEBUG",
"-D", "_WINDOWS", "-D", "_UNICODE", "-D", "UNICODE", "-o",
