def main():
"""main"""
desc = "This module reads the information from given folder."
parser = argparse.ArgumentParser(description=desc)
print("\n[ TCC - TRACK EXTRACTOR ]\n%s\n" % desc)
parser.add_argument('-i',
'--input_folder',
default=utils.get_cur_location())
parser.add_argument('-f',
'--output_folder',
default=utils.get_valid_folder_path("results"))
parser.add_argument('-o', '--output_file', default="output")
parsed = parser.parse_args()
csv_file = os.path.join(parsed.output_folder, parsed.output_file + ".txt")
json_file = os.path.join(parsed.output_folder,
parsed.output_file + ".json")
content = walk_in_folders(parsed.input_folder)
utils.write_file_with(csv_file, content)
json_parser.parse(parsed.input_folder, csv_file, json_file)
utils.show_message("All done, take a look: %s" % csv_file)
def from_json(json_string):
try:
assert type(json_string) == str, str(json_string) + ' is not string'
logger.info('from_json: Received string: ' + json_string)
tokens = json_lexer.lex(json_string)
json_object = json_parser.parse(tokens, is_root=True)[0]
assert type(json_object) == dict, 'Dictionary object was not returned'
logger.info('from_json: Returned object: ' + str(json_object))
return json_parser.parse(tokens, is_root=True)[0]
except AssertionError as e:
logger.error('from_json: ' + str(e))
def find_example():
spec = create_model(2, 3, 4)
with open("temp/spec.json", "w") as jsonFile:
jsonFile.write(json.dumps(spec, indent=4))
#with open("temp/spec.json", 'r') as f:
# spec = json.load(f)
index = 0
add_assertion(spec, EPS)
add_solver(spec)
model, assertion, solver, display = parse(spec)
network_tf = "temp/network.tf"
save_tf_model(network_tf, model, len(model.layers)-1)
find = False
for no in range(5):
x0s = np.round(generate_rand((100, 2), INPUT_BOUND[0], INPUT_BOUND[1]), 4)
example = []
res = refinepoly_run(x0s, None, network_tf, EPS, 2)
print("Refine result", res)
for j in res:
success_lst = newApproach(model, x0s[j], '')
if success_lst:
print("Find example")
print(x0s[j])
with open("temp/x{}.txt".format(index), "w") as exp:
exp.write(str(x0s[j].tolist()))
find = True
index += 1
return find
def run_verify(zipped_args):
start, end, args = zipped_args[0], zipped_args[1], zipped_args[2]
with open(args.spec, 'r') as f:
spec = json.load(f)
args.pathX, args.pathY = get_dataset(args.dataset)
bd_target, fa_target, sb_target = [], [], []
for target in range(start, end):
args.target = target
print(
'Backdoor target = {} with size = {}, total imgs = {}, num imgs = {}, rate = {}, and threshold = {}'
.format(args.target, args.size, args.total_imgs, args.num_imgs,
args.rate, args.threshold))
add_assertion(args, spec)
add_solver(args, spec)
model, assertion, solver, display = parse(spec)
res, sbi = solver.solve(model, assertion)
if res is not None:
bd_target.append(res)
if sbi is None:
fa_target.append(res)
else:
sb_target.append(res)
return bd_target, fa_target, sb_target
def main():
init(strip=not sys.stdout.isatty())
cprint(figlet_format('salty mike\'s salt parser', font='starwars'),
'white',
'on_blue',
attrs=['dark', 'bold'])
data_file = input(
"Provide input file path (or press enter if file is \'in.json\'): ")
if not data_file:
data_file = 'in.json'
conf_file = input(
'Provide configuration file path (or press enter if file is \'conf.properties\'): '
)
if not conf_file:
conf_file = 'conf.properties'
print()
with open(conf_file) as c:
conf = c.read().splitlines()
with open(data_file, 'r') as json_file:
hosts = json.loads(''.join(json_file))
for host in hosts:
for k, v in host.items():
addResults(conf, k, json_parser.parse(host))
json_parser.output_dict = {}
print_results()
def from_file(filename):
tokens = []
with open(filename) as f:
for _, line in enumerate(f):
line = line.strip('\n')
tokens += json_lexer.lex(line)
f.close()
return json_parser.parse(tokens, is_root=True)[0]
def generate_pandoc(directory, f, output_dir):
file_settings = json_parser.parse(os.path.join(directory,
f[:-3] + ".json"))
file_real_path = os.path.realpath(os.path.join(directory, f))
out_real_path = os.path.realpath(os.path.join(output_dir,
f[:-3] + ".html"))
command = "pandoc -s -f markdown -t html %s --metadata pagetitle=\"%s\" -o %s -c '/style/markdown.css'" % \
(file_real_path, file_settings["title"], out_real_path)
# print(command)
os.system(command)
def main():
np.set_printoptions(threshold=20)
parser = argparse.ArgumentParser(description='nSolver')
parser.add_argument('--spec',
type=str,
default='spec.json',
help='the specification file')
parser.add_argument('--algorithm', type=str, help='the chosen algorithm')
parser.add_argument('--threshold',
type=float,
help='the threshold in sprt')
args = parser.parse_args()
with open(args.spec, 'r') as f:
spec = json.load(f)
add_assertion(args, spec)
add_solver(args, spec)
model, assertion, solver, display = parse(spec)
lower = model.lower
upper = model.upper
for i in range(50):
pathX = 'benchmark/mnist_challenge/x_y/x' + str(i) + '.txt'
pathY = 'benchmark/mnist_challenge/x_y/y' + str(i) + '.txt'
x0s = np.array(ast.literal_eval(read(pathX)))
y0s = np.array(ast.literal_eval(read(pathY)))
for j in range(200):
x0 = x0s[j]
assertion['x0'] = str(x0.tolist())
output_x0 = model.apply(x0)
lbl_x0 = np.argmax(output_x0, axis=1)[0]
print('Data {}\n'.format(i * 200 + j))
print('x0 = {}'.format(x0))
print('output_x0 = {}'.format(output_x0))
print('lbl_x0 = {}'.format(lbl_x0))
print('y0 = {}\n'.format(y0s[j]))
if lbl_x0 == y0s[j]:
update_bounds(args, model, x0, lower, upper)
print('Run at data {}\n'.format(i * 200 + j))
solver.solve(model, assertion)
else:
print('Skip at data {}'.format(i * 200 + j))
print('\n============================\n')
def get_tone_for_user(username, subreddit=None, comments_limit=1):
""" Compute watson mood of user "username" on subreddit "subreddit" by
looking at last comments. """
comments = [x.body for x in get_comments(username, comments_limit)]
comment_to_tone = []
for comment in comments:
json_data = query_watson(comment)
tone = parse(json_data)
comment_to_tone.append((comment, tone))
return comment_to_tone
def print_detail():
network_tf = "temp/network.tf"
with open("temp/examples/spec.json", 'r') as f:
spec = json.load(f)
add_assertion(spec, 0.01)
add_solver(spec)
model, assertion, solver, display = parse(spec)
x0 = np.array(ast.literal_eval(open("temp/examples/x2.txt", 'r').readline()))
res = refinepoly_run(np.reshape(x0, (-1,2)), None, network_tf, 2)
success_lst = newApproach(model, x0, '')
if success_lst:
print("Find example")
assertion['x0'] = str(x0.tolist())
res = solver.solve(model, assertion)
if res == 0:
print("DeepPoly can't prove this")
def main():
np.set_printoptions(threshold=20)
parser = argparse.ArgumentParser(description='nSolver')
parser.add_argument('--spec',
type=str,
default='spec.json',
help='the specification file')
args = parser.parse_args()
with open(args.spec, 'r') as f:
spec = json.load(f)
model, assertion, solver, display = parse(spec)
solver.solve(model, assertion, display)
def run_attack(args):
print(
'Backdoor target = {} with size = {}, total imgs = {}, num imgs = {}, and attack only = {} at position = {}'
.format(args.target, args.size, args.total_imgs, args.num_imgs,
args.atk_only, args.atk_pos))
with open(args.spec, 'r') as f:
spec = json.load(f)
args.pathX, args.pathY = get_dataset(args.dataset)
add_assertion(args, spec)
add_solver(args, spec)
model, assertion, solver, display = parse(spec)
res, stamp = solver.solve(model, assertion)
def process_json(jsonpath, outputpath, topclassname, public_fields=False, getset=True):
to_load = file_handler.read(jsonpath)
file_handler.create_folder_if_not_exist(outputpath)
try:
loaded = json_parser.parse(to_load)
except ValueError:
print "not a JSON file (maybe has errors?)"
exit(-1)
if len(loaded) == 0:
print "empty json, exiting"
exit(-1)
structure = engine.prepare_structure(loaded, topclassname)
texts = engine.generate_classes(structure, public_fields, getset)
for classname, text in texts:
path = "{}{}{}.java".format(outputpath, file_handler.sep, classname)
file_handler.save(path, text)
label = 'wasted'
elif label == '0':
label = 'clean'
elif label != 'unknown':
label = 'repaired'
if label not in groups:
groups[label] = [advert]
else:
groups[label].append(advert)
return groups
if __name__ == '__main__':
# parse program arguments
parser = argparse.ArgumentParser()
parser.add_argument('--path', type=str, help="Input json file path")
parser.add_argument('--target',
type=str,
default='paint',
help="Target variable to group the adverts")
args = parser.parse_args()
# parse advert labels group the adverts by the specified target variable
labels = jp.parse(args.path, args.target, 'field')
groups = group(labels)
for label, adverts in groups.items():
print(label)
for advert in adverts:
print(advert)
print()
def main():
np.set_printoptions(threshold=20)
parser = argparse.ArgumentParser(description='nSolver')
parser.add_argument('--spec',
type=str,
default='spec.json',
help='the specification file')
parser.add_argument('--algorithm', type=str, help='the chosen algorithm')
parser.add_argument('--eps', type=float, help='the distance value')
parser.add_argument('--has_ref',
action='store_true',
help='turn on/off refinement')
parser.add_argument('--max_ref',
type=int,
default=20,
help='maximum times of refinement')
parser.add_argument('--ref_typ',
type=int,
default=0,
help='type of refinement')
parser.add_argument('--max_sus',
type=int,
default=1,
help='maximum times of finding adversarial sample')
parser.add_argument('--dataset',
type=str,
help='the data set for CEGAR experiments')
parser.add_argument('--num_tests',
type=int,
default=100,
help='maximum number of tests')
args = parser.parse_args()
with open(args.spec, 'r') as f:
spec = json.load(f)
add_assertion(args, spec)
add_solver(args, spec)
model, assertion, solver, display = parse(spec)
lower = model.lower
upper = model.upper
pathX = 'benchmark/cegar/data/mnist_fc/'
pathY = 'benchmark/cegar/data/labels/y_mnist.txt'
y0s = np.array(ast.literal_eval(read(pathY)))
for i in range(args.num_tests):
assertion['x0'] = pathX + 'data' + str(i) + '.txt'
x0 = np.array(ast.literal_eval(read(assertion['x0'])))
output_x0 = model.apply(x0)
lbl_x0 = np.argmax(output_x0, axis=1)[0]
print('Data {}\n'.format(i))
print('x0 = {}'.format(x0))
print('output_x0 = {}'.format(output_x0))
print('lbl_x0 = {}'.format(lbl_x0))
print('y0 = {}\n'.format(y0s[i]))
if lbl_x0 == y0s[i]:
best_verified, best_failed = 0, 1e9
eps, step_eps = 0.01, 0.01
while True:
t0 = time.time()
args.eps = eps
update_bounds(args, model, x0, lower, upper)
print('Run at data {}\n'.format(i))
res = solver.solve(model, assertion)
if res == 1:
print('Verified at {:.3f}'.format(eps))
best_verified = max(best_verified, eps)
elif res == 0:
print('Failed at {:.3f}'.format(eps))
best_failed = min(best_failed, eps)
else:
break
t1 = time.time()
print('time = {}'.format(t1 - t0))
print('\n============================\n')
if best_verified == round(best_failed - 0.001, 3): break
if res == 1:
if step_eps == 0.01:
eps = round(eps + step_eps, 3)
elif step_eps == -0.005:
step_eps = 0.001
eps = round(eps + step_eps, 3)
elif step_eps == 0.001:
eps = round(eps + step_eps, 3)
elif res == 0:
if step_eps == 0.01:
step_eps = -0.005
eps = round(eps + step_eps, 3)
elif step_eps == -0.005:
step_eps = -0.001
eps = round(eps + step_eps, 3)
elif step_eps == -0.001:
eps = round(eps + step_eps, 3)
print("Image {} Verified at {:.3f} and Failed at {:.3f}".format(
i, best_verified, best_failed))
else:
print('Skip at data {}'.format(i))
print("Image {} Verified at {:.3f} and Failed at {:.3f}".format(
i, -1, -1))
res = -1
print('\n============================\n')
import os
import json_parser, file_supervisor
global_settings = json_parser.parse('global.json')
if (global_settings == None):
print("Unable to open `global.json`, halting.")
exit(-1)
files = file_supervisor.supervise(directory=global_settings['BLOG_DIR'])
file_supervisor.clean(global_settings['OUTPUT_DIR'], files)
for directory in files.keys():
for f in files[directory]:
file_supervisor.generate_pandoc(
os.path.join(global_settings['BLOG_DIR'], directory), f,
os.path.join(global_settings['OUTPUT_DIR'], directory))
if (global_settings['STATIC_SERVER']):
import static_server
static_server.serve(directory=global_settings['OUTPUT_DIR'])
def main():
np.set_printoptions(threshold=20)
parser = argparse.ArgumentParser(description='nSolver')
parser.add_argument('--spec',
type=str,
default='spec.json',
help='the specification file')
parser.add_argument('--algorithm', type=str, help='the chosen algorithm')
parser.add_argument('--threshold',
type=float,
help='the threshold in sprt')
parser.add_argument('--eps', type=float, help='the distance value')
parser.add_argument('--dataset',
type=str,
help='the data set for rnn experiments')
args = parser.parse_args()
with open(args.spec, 'r') as f:
spec = json.load(f)
add_assertion(args, spec)
add_solver(args, spec)
model, assertion, solver, display = parse(spec)
lower = model.lower[0]
upper = model.upper[0]
if args.dataset == 'jigsaw':
pathX = 'benchmark/rnn/data/jigsaw/'
pathY = 'benchmark/rnn/data/jigsaw/labels.txt'
elif args.dataset == 'wiki':
pathX = 'benchmark/rnn/data/wiki/'
pathY = 'benchmark/rnn/data/wiki/labels.txt'
y0s = np.array(ast.literal_eval(read(pathY)))
for i in range(100):
assertion['x0'] = pathX + 'data' + str(i) + '.txt'
x0 = np.array(ast.literal_eval(read(assertion['x0'])))
shape_x0 = (int(x0.size / 50), 50)
model.shape = shape_x0
model.lower = np.full(x0.size, lower)
model.upper = np.full(x0.size, upper)
output_x0 = model.apply(x0)
lbl_x0 = np.argmax(output_x0, axis=1)[0]
print('Data {}\n'.format(i))
print('x0 = {}'.format(x0))
print('output_x0 = {}'.format(output_x0))
print('lbl_x0 = {}'.format(lbl_x0))
print('y0 = {}\n'.format(y0s[i]))
if lbl_x0 == y0s[i]:
print('Run at data {}\n'.format(i))
solver.solve(model, assertion)
else:
print('Skip at data {}'.format(i))
print('\n============================\n')
def main():
np.set_printoptions(threshold=20)
parser = argparse.ArgumentParser(description='nSolver')
parser.add_argument('--spec',
type=str,
default='spec.json',
help='the specification file')
parser.add_argument('--algorithm', type=str, help='the chosen algorithm')
parser.add_argument('--threshold',
type=float,
help='the threshold in sprt')
parser.add_argument('--eps', type=float, help='the distance value')
parser.add_argument('--has_ref',
action='store_true',
help='turn on/off refinement')
parser.add_argument('--max_ref',
type=int,
default=20,
help='maximum times of refinement')
parser.add_argument('--ref_typ',
type=int,
default=0,
help='type of refinement')
parser.add_argument('--max_sus',
type=int,
default=1,
help='maximum times of finding adversarial sample')
parser.add_argument('--dataset',
type=str,
help='the data set for fairness experiments')
parser.add_argument('--num_tests',
type=int,
default=100,
help='maximum number of tests')
args = parser.parse_args()
with open(args.spec, 'r') as f:
spec = json.load(f)
add_assertion(args, spec)
add_solver(args, spec)
model, assertion, solver, display = parse(spec)
'''
if args.dataset == 'bank':
pathX = 'benchmark/fairness/bank/data/'
pathY = 'benchmark/fairness/bank/data/labels.txt'
elif args.dataset == 'census':
pathX = 'benchmark/fairness/census/data/'
pathY = 'benchmark/fairness/census/data/labels.txt'
elif args.dataset == 'credit':
pathX = 'benchmark/fairness/credit/data/'
pathY = 'benchmark/fairness/credit/data/labels.txt'
'''
if args.dataset == 'bank':
pathX = '../benchmark/fairness/bank/data/' #debug
pathY = '../benchmark/fairness/bank/data/labels.txt' #debug
elif args.dataset == 'census':
pathX = '../benchmark/fairness/census/data/' #debug
pathY = '../benchmark/fairness/census/data/labels.txt' #debug
elif args.dataset == 'credit':
pathX = '../benchmark/fairness/credit/data/' #debug
pathY = '../benchmark/fairness/credit/data/labels.txt' #debug
y0s = np.array(ast.literal_eval(read(pathY)))
for i in range(args.num_tests):
assertion['x0'] = pathX + 'data' + str(i) + '.txt'
x0 = np.array(ast.literal_eval(read(assertion['x0'])))
output_x0 = model.apply(x0)
lbl_x0 = np.argmax(output_x0, axis=1)[0]
print('Data {}\n'.format(i))
print('x0 = {}'.format(x0))
print('output_x0 = {}'.format(output_x0))
print('lbl_x0 = {}'.format(lbl_x0))
print('y0 = {}\n'.format(y0s[i]))
if lbl_x0 == y0s[i]:
print('Run at data {}\n'.format(i))
solver.solve(model, assertion)
else:
print('Skip at data {}'.format(i))
print('\n============================\n')
import time
import cl_parser
import json_parser
from kafka_avro_producer import KafkaAvroProducer
from kafka_json_producer import KafkaJsonProducer
args = cl_parser.parse()
list_of_stations = json_parser.parse()
if args.format == "avro":
kafka_producer = KafkaAvroProducer(args.brokers, args.schema_registry_url)
else:
kafka_producer = KafkaJsonProducer(args.brokers)
while True:
kafka_producer.send_message_to_kafka(list_of_stations, args.topic)
time.sleep(args.interval)
entropyMap = {}
for word, wordsPerLabel in histogram.items():
histogram[word] = {label:count / advertsPerLabel[label] for (label, count) in wordsPerLabel.items()}
entropyMap[word] = {label:abs(freq - (sum(histogram[word].values()) - freq)) for (label, freq) in histogram[word].items()}
return entropyMap
if __name__ == '__main__':
# parse program arguments
parser = argparse.ArgumentParser()
parser.add_argument('--histpath', type=str, default=DEFAULT_HISTOGRAM, help="histogram file path")
parser.add_argument('--labelpath', type=str, default=DEFAULT_LABELS, help="label file path")
parser.add_argument('--target', type=str, default='paint', help="Target variable to group the adverts")
args = parser.parse_args()
# parse and preprocess advert descriptions
advertLabelMap = jp.parse(args.labelpath, args.target, 'field')
labelAdvertMap = ag.group(advertLabelMap)
# compute the number of adverts per label
advertsPerLabel = {}
for label, adverts in labelAdvertMap.items():
advertsPerLabel[label] = len(adverts)
# read the histogram from file
histogram = json.load(open(args.histpath))
hg.printHist(histogram)
# print words with high entropy
entropyMap = entropy(histogram, advertsPerLabel)
for word, entropy in entropyMap.items():
if sum(entropy.values()) > 3:
def main():
np.set_printoptions(threshold=20)
parser = argparse.ArgumentParser(description='nSolver')
parser.add_argument('--spec', type=str, default='spec.json',
help='the specification file')
parser.add_argument('--algorithm', type=str,
help='the chosen algorithm')
parser.add_argument('--threshold', type=float,
help='the threshold in sprt')
parser.add_argument('--eps', type=float,
help='the distance value')
parser.add_argument('--dataset', type=str,
help='the data set for ERAN experiments')
args = parser.parse_args()
with open(args.spec, 'r') as f:
spec = json.load(f)
add_assertion(args, spec)
add_solver(args, spec)
model, assertion, solver, display = parse(spec)
lower = model.lower
upper = model.upper
if args.dataset == 'cifar_conv':
pathX = 'benchmark/eran/data/cifar_conv/'
pathY = 'benchmark/eran/data/labels/y_cifar.txt'
elif args.dataset == 'cifar_fc':
pathX = 'benchmark/eran/data/cifar_fc/'
pathY = 'benchmark/eran/data/labels/y_cifar.txt'
elif args.dataset == 'mnist_conv':
pathX = 'benchmark/eran/data/mnist_conv/'
pathY = 'benchmark/eran/data/labels/y_mnist.txt'
elif args.dataset == 'mnist_fc':
pathX = 'benchmark/eran/data/mnist_fc/'
pathY = 'benchmark/eran/data/labels/y_mnist.txt'
y0s = np.array(ast.literal_eval(read(pathY)))
for i in range(10):
assertion['x0'] = pathX + 'data' + str(i) + '.txt'
x0 = np.array(ast.literal_eval(read(assertion['x0'])))
output_x0 = model.apply(x0)
lbl_x0 = np.argmax(output_x0, axis=1)[0]
print('Data {}\n'.format(i))
print('x0 = {}'.format(x0))
print('output_x0 = {}'.format(output_x0))
print('lbl_x0 = {}'.format(lbl_x0))
print('y0 = {}\n'.format(y0s[i]))
t0 = time.time()
if lbl_x0 == y0s[i]:
update_bounds(args, model, x0, lower, upper)
print('Run at data {}\n'.format(i))
solver.solve(model, assertion)
else:
print('Skip at data {}'.format(i))
t1 = time.time()
print('time = {}'.format(t1 - t0))
print('\n============================\n')
print('{0: <25}'.format("\n" + word), end="")
for label in labelCountMap.keys():
print('{0: >4}'.format(labelCountMap[label]), end="")
print('{0: <10}'.format(" " + label), end="")
if __name__ == '__main__':
# parse program arguments
parser = argparse.ArgumentParser()
parser.add_argument('--datapath', type=str, help="Input json file path")
parser.add_argument('--labelpath', type=str, help="Input json file path")
parser.add_argument('--target', type=str, default='paint', help="Target variable to group the adverts")
args = parser.parse_args()
# parse and preprocess advert descriptions
advertDescriptionMap = jp.parse(args.datapath, 'description', 'data')
advertLabelMap = jp.parse(args.labelpath, args.target, 'field')
labelAdvertMap = ag.group(advertLabelMap)
advertsPerLabel = {}
for label, adverts in labelAdvertMap.items():
advertsPerLabel[label] = len(adverts)
print(advertsPerLabel)
input()
# create histograms for each label
preprocessor.preprocess(advertDescriptionMap)
histogram = histLabels(advertDescriptionMap, labelAdvertMap)
printHist(histogram)
# write histogram to a json file
def mnist_challenge(model_path, _range, x_path, y_path=None):
model_name = str(model_path).split('/')[-2]
#list_file = []
#with open("result_newapproach/{}_refinepoly_failed.csv".format(model_name), 'r') as csvfile:
# list_file_raw = csv.reader(csvfile, delimiter=',')
# for test in list_file_raw:
# list_file.append((test[0], ast.literal_eval(test[1])))
with open(model_path, 'r') as f:
spec = json.load(f)
add_assertion(spec, EPS)
add_solver(spec)
model, assertion, solver, display = parse(spec)
list_potential2, unknow2, find_exp2 = [], [], []
index = 0
for i in _range:
#for i in list_file:
testfileName = i
pathX = x_path + str(testfileName) + ".txt" #i -> i[0]
pathY = None if y_path is None else y_path + str(
testfileName) + ".txt" #i -> i[0]
x0s = np.array(ast.literal_eval(read(pathX)))
y0s = None if y_path is None else np.array(
ast.literal_eval(read(pathY)))
x0s = x0s if len(x0s.shape) == 2 else np.array([x0s])
list_potential, unknow, find_exp, csv_result = [], [], [], []
for j in range(x0s.shape[0]):
#for j in i[1]:
#print('Index:', index)
#index += 1
x0 = x0s[j]
assertion['x0'] = str(x0.tolist())
output_x0 = model.apply(x0)
lbl_x0 = np.argmax(output_x0, axis=1)[0]
y0 = y0s[j] if y0s is not None else lbl_x0
if lbl_x0 != y0:
print("Skip")
continue
start = time.time()
#res = solver.solve(model, assertion)
res = refinepoly_run(np.array([x0]), np.array([y0]))
print("DeepPoly time:", time.time() - start)
if res == 1 or res == 2:
print("DeepPoly can prove this model")
continue
else:
print("DeepPoly can't prove this model:", j)
list_potential.append(j)
#res = testCegar(model, str(x0.tolist()))
#if res == 1:
# list_potential.append(j)
#elif res == 2:
# continue
#else:
# print("Cegar also can't prove this")
# unknow.append(j)
start = time.time()
success_lst = newApproach(model, x0, '')
if success_lst:
print("Find example:", j)
find_exp.append((j, success_lst))
csv_result.append([
str(testfileName) + '_' + str(j),
time.time() - start, success_lst
])
if list_potential:
list_potential2.append([testfileName, list_potential])
with open(
'result_newapproach/{}_refinepoly_failed.csv'.format(
model_name), 'a') as f:
writer = csv.writer(f)
writer.writerow([testfileName, list_potential])
#unknow2.append((i,unknow))
find_exp2.append((testfileName, find_exp))
with open(
'result_newapproach/{}_approach_cp_refine_result.csv'.format(
model_name), 'a') as f:
writer = csv.writer(f)
writer.writerows(csv_result)
print("potential:", list_potential2)
#print("Unknow:", unknow2)
print("Example Find:", find_exp2)
def get_raw_tone(string):
json_data = query_watson(string)
tone = parse(json_data)
return tone
# replace any number, http(s) address and email address with the words 'number', 'httpaddr' and 'emailaddr'
text = re.sub(r'[0-9]+', ' number ', text)
text = re.sub('(http|https)://[^\s]*', 'httpaddr', text)
text = re.sub('[^\s]+@[^\s]+', 'emailaddr', text)
# split the text into words and get rid of any punctuation
text = re.findall(r"[\w']+", text)
# remove any non-alphanumeric characters and remove words with less than 2 letters
for index, word in enumerate(text):
word = re.sub('[^a-zA-Z0-9]', '', word)
if (len(word) > 1):
text[index] = word
else:
del text[index]
# replace the raw description text with a preprocessed list of words
descriptions[key] = text
if __name__ == '__main__':
# parse program arguments
parser = argparse.ArgumentParser()
parser.add_argument('--path', type=str, help="Input json file path")
args = parser.parse_args()
# parse and preprocess advert descriptions
descriptions = jp.parse(args.path, 'description', 'data')
preprocess(descriptions)
print(descriptions)
def get_services():
with open("prices.json", "r") as file:
services = parse(file)
return services["prices"]
def main():
screen_width = 90
screen_height = 60
tcod.console_set_custom_font(
'terminal8x12_gs_tc.png',
tcod.FONT_TYPE_GRAYSCALE | tcod.FONT_LAYOUT_TCOD)
tcod.console_init_root(screen_width,
screen_height,
'Spaceship',
False,
renderer=tcod.RENDERER_SDL2,
vsync=True)
tcod.console_map_ascii_code_to_font(7, 12, 2)
tcod.console_map_ascii_code_to_font(9, 13, 2)
object_view_width = 40
object_view_height = 40
main_console = Console(screen_width, screen_height, order='F')
object_console = Console(screen_width, screen_height, order='F')
parse(all_objects, object_list)
frame = all_objects['base'][1]
spaceship_x, spaceship_y = 0, 0
file_path = getcwd() + '/spaceship.dat'
if not path.exists(file_path) or path.getsize(file_path) == 0:
pickle.dump(Object('Spaceship', 50, 50), open(file_path, 'wb+'))
try:
spaceship = pickle.load(open(getcwd() + '/spaceship.dat', 'rb'))
except pickle.UnpicklingError:
pickle.dump(Object('Spaceship', 50, 50), open(file_path, 'wb+'))
spaceship = pickle.load(open(file_path, 'rb'))
examine_x = screen_width - 50
examine_y = 1
examine_width = 50
examine_height = screen_height - 2
examine_menu = None
build_menu_object_key_list = ['default']
for x in range(spaceship.width):
for y in range(spaceship.height):
for part in spaceship.tiles[x][y]:
for object in object_list:
if not 'json_id' in part.__dict__ and object.name == part.name:
part.json_id = object.json_id
print('Assigned {} to {}'.format(
part.name, object.json_id))
if 'json_id' in part.__dict__ and object.json_id == part.json_id:
part.fg = object.fg
part.bg = object.bg
part.char = object.char
part.sorting_index = object.sorting_index
part.mass = object.mass
break
else:
print(
'Could not find corresponding JSON part for existing part {}'
.format(part.name))
spaceship.add_part(0, 0, frame, override=True)
spaceship.calculate_rooms()
viewing_mode = 0
num_modes = len(view_dict) + 2
selection_index = 0
max_select_index = 0
build_menu_open = False
selection_box = (0, 0), (0, 0)
screen_offset_x, screen_offset_y = 0, 0
selection_console = Console(screen_width, screen_height, order='F')
selection_console.clear(bg=tcod.lightest_blue)
while True:
response = None
prev_select_index = selection_index
prev_selection_coord = selection_box[0]
for event in tcod.event.get():
if event.type == 'KEYDOWN':
response = {}
input_resp = handle_input(event)
if input_resp:
response.update(input_resp)
if build_menu_open and ord('z') >= event.sym >= ord('a'):
response['select_item'] = event.sym - ord('a')
elif event.type == 'MOUSEBUTTONDOWN' or event.type == 'MOUSEBUTTONUP' or event.type == 'MOUSEMOTION':
response = handle_mouse(event)
elif event.type == 'QUIT':
pickle.dump(spaceship, open(file_path, 'wb+'))
return
if response:
if response.get('move'):
move = response.get('move')
if examine_menu:
examine_menu.selected_index += move[1]
if move[0] != 0:
if examine_menu.try_change_option(move[0]):
spaceship.update_all()
else:
screen_offset_x = max(0, screen_offset_x + move[0])
screen_offset_y = max(0, screen_offset_y + move[1])
elif response.get('viewing_mode'):
viewing_mode = (viewing_mode +
response.get('viewing_mode')) % num_modes
elif response.get('navigate'):
selection_index += response.get('navigate')
if selection_index < 0:
selection_index += max_select_index
elif selection_index >= max_select_index:
selection_index -= max_select_index
if response.get('build'):
if not build_menu_open:
build_menu_open = True
build_menu_object_key_list = []
selection_index = 0
if response.get('delete') and not build_menu_open:
if spaceship.is_valid_coord(
selection_box[0][0] - spaceship_x,
selection_box[0][1] -
spaceship_y) and selection_index < len(
spaceship.tiles[selection_box[0][0] - spaceship_x][
selection_box[0][1] - spaceship_y]):
part_name = spaceship.tiles[
selection_box[0][0] -
spaceship_x][selection_box[0][1] -
spaceship_y][selection_index].name
for x in range(selection_box[0][0], selection_box[1][0]):
for y in range(selection_box[0][1],
selection_box[1][1]):
for part in spaceship.tiles[x][y]:
if part.name == part_name:
spaceship.remove_part(
x - spaceship_x, y - spaceship_y, part)
elif (response.get('select') or response.get('mouse_move_lheld')
or response.get('lclick_down')) and build_menu_open:
if type(get_object(build_menu_object_key_list)) is list:
if response.get('mouse_move_lheld'):
selection_box = response.get('mouse_move_lheld'), (
response.get('mouse_move_lheld')[0] + 1,
response.get('mouse_move_lheld')[1] + 1)
for x in range(selection_box[0][0], selection_box[1][0]):
for y in range(selection_box[0][1],
selection_box[1][1]):
spaceship.add_part(
x - spaceship_x, y - spaceship_y,
get_object(build_menu_object_key_list)
[selection_index])
else:
build_menu_object_key_list.append(
list(get_object(build_menu_object_key_list).keys())
[selection_index])
selection_index = 0
elif response.get('run'):
for x in range(spaceship.width):
for y in range(spaceship.height):
for part in [
part for part in spaceship.tiles[x][y]
if part.runnable
]:
part.runnable.run(spaceship, part, x, y)
spaceship.update_all()
if response.get('rclick_down'):
selection_box = response.get('rclick_down'), response.get(
'rclick_down')
elif response.get('lclick_down'):
selection_box = response.get('lclick_down'), response.get(
'lclick_down')
elif response.get('mouse_move_rheld'):
selection_box = selection_box[0], (
response.get('mouse_move_rheld')[0] + 1,
response.get('mouse_move_rheld')[1] + 1)
elif response.get(
'mouse_move'
) and selection_box[1][0] - selection_box[0][
0] <= 1 and selection_box[1][1] - selection_box[0][1] <= 1:
selection_box = response.get('mouse_move'), (
response.get('mouse_move')[0] + 1,
response.get('mouse_move')[1] + 1)
elif response.get('select_item') is not None:
index = response.get('select_item')
if index < max_select_index:
selection_index = index
if type(get_object(build_menu_object_key_list)) is dict:
build_menu_object_key_list.append(
list(
get_object(build_menu_object_key_list).keys())
[selection_index])
selection_index = 0
if response.get('escape'):
if build_menu_open:
if len(build_menu_object_key_list) == 0:
build_menu_open = False
selection_index = 0
else:
build_menu_object_key_list.pop(-1)
elif examine_menu:
examine_menu = None
if prev_select_index != selection_index or prev_selection_coord != selection_box[
0]:
if spaceship.is_valid_coord(
selection_box[0][0] - spaceship_x, selection_box[0][1] -
spaceship_y) and selection_index < len(
spaceship.tiles[selection_box[0][0] - spaceship_x][
selection_box[0][1] - spaceship_y]):
examine_menu = PartMenu(
spaceship,
spaceship.tiles[selection_box[0][0] -
spaceship_x][selection_box[0][1] -
spaceship_y][selection_index],
examine_width, examine_height, selection_box[0][0],
selection_box[0][1])
else:
examine_menu = None
tcod.console_flush()
main_console.clear(bg=tcod.black)
selection_pos = selection_box[0]
object_console.clear()
render_object(object_console, spaceship, spaceship_x, spaceship_y,
viewing_mode)
object_console.blit(main_console, 0, 0, screen_offset_x,
screen_offset_y, object_view_width,
object_view_height)
# main_console.put_char(selection_box[0][0], selection_box[0][1], ord('X'))
# main_console.fg[selection_box[0][0], selection_box[0][1]] = tcod.cyan
if build_menu_open:
iterable = get_object(build_menu_object_key_list)
if type(iterable) is list:
iterable = [item.name for item in iterable]
else:
iterable = [item.capitalize() for item in iterable]
for i in range(len(iterable)):
main_console.print(screen_width - 50,
i,
chr(ord('a') + i) + ' - ' + iterable[i],
bg=(tcod.desaturated_blue if selection_index
== i else tcod.black))
main_console.fg[screen_width - 50, i] = tcod.light_green
max_select_index = len(iterable)
else:
main_console.print(0, screen_height - 1, str(viewing_mode))
if spaceship_x <= selection_pos[
0] < spaceship_x + spaceship.width and spaceship_y <= selection_pos[
1] < spaceship_y + spaceship.height:
room = spaceship.find_room(selection_pos[0] - spaceship_x,
selection_pos[1] - spaceship_y)
parts = [
part
for part in spaceship.tiles[selection_pos[0] -
spaceship_x][selection_pos[1] -
spaceship_y]
]
if room:
gas_y = len(parts) + 2
for gas in room.gas_content:
main_console.print(
20, gas_y, '{1}: {0:.2f} kPa'.format(
components.calc_pressure(
room.gas_content[gas], len(room.tiles)),
gas.capitalize()))
gas_y += 1
for i in range(len(parts)):
part = parts[i]
string = part.name
if part.gas_grid:
string += ' {}'.format(part.gas_grid.index)
main_console.print(
20,
i + 1,
string,
bg=(tcod.desaturated_blue
if selection_index == i else tcod.black))
max_select_index = len(parts)
if examine_menu:
examine_menu.draw()
examine_menu.console.blit(main_console, examine_x, examine_y,
0, 0, examine_width, examine_height)
dsel_x = selection_box[1][0] - selection_box[0][0]
dsel_y = selection_box[1][1] - selection_box[0][1]
if dsel_x > 0 and dsel_y > 0:
selection_console.blit(main_console,
selection_box[0][0],
selection_box[0][1],
0,
0,
dsel_x,
dsel_y,
bg_alpha=0.25)
main_console.blit(tcod.console._root_console,
width=screen_width,
height=screen_height,
bg_alpha=1)
merge(a[key], b[key], path + [str(key)])
elif a[key] == b[key]:
pass
else:
raise Exception('Conflict at %s' % '.'.join(path + [str(key)]))
else:
a[key] = b[key]
return a
if __name__ == '__main__':
import json
from json_parser import parse
parsed = {}
with open('translates/ru.json', 'r') as fp:
parsed = json.load(fp)
keys_by_translate = {}
parse(parsed, [], 0, keys_by_translate)
arr = fill_arr(keys_by_translate)
arr = transform_list_to_dict(arr)
cor_obj = arr[0]
for obj in arr[1:]:
cor_obj = merge(cor_obj, obj)
with open('result.json', 'w') as fp:
json.dump(cor_obj, fp, ensure_ascii=False, indent=4)
broccoli.broccoli
~~~~~~~~~~~~~
This project aims to ease the computation of certain problems by introducing
tools parallelization and multiprocessing.
:copyright: 2015 Manuel Martins, see AUTHORS for more details
:license: Apache 2.0, see LICENSE for more details
"""
import argparse
from logger import initialize as logger_init
import json_parser
from job import Job
import logging
if __name__ == '__main__':
args_parser = argparse.ArgumentParser(
description='Main entry point for Broccoli Module. Usage: python -m broccoli -i <input.json>'
)
args_parser.add_argument('-v', '--verbose', help='increase output verbosity', action='store_true',
required=False)
args_parser.add_argument('-i', '--input', help='input json file / string', action='store',
dest='input', required=True)
logging.info('Broccoli - Initializing...')
args = args_parser.parse_args()
json = json_parser.parse(args.input)
logger_init(json, args.verbose)
job = Job(json)
job.start()
