def main(): """main""" desc = "This module reads the information from given folder." parser = argparse.ArgumentParser(description=desc) print("\n[ TCC - TRACK EXTRACTOR ]\n%s\n" % desc) parser.add_argument('-i', '--input_folder', default=utils.get_cur_location()) parser.add_argument('-f', '--output_folder', default=utils.get_valid_folder_path("results")) parser.add_argument('-o', '--output_file', default="output") parsed = parser.parse_args() csv_file = os.path.join(parsed.output_folder, parsed.output_file + ".txt") json_file = os.path.join(parsed.output_folder, parsed.output_file + ".json") content = walk_in_folders(parsed.input_folder) utils.write_file_with(csv_file, content) json_parser.parse(parsed.input_folder, csv_file, json_file) utils.show_message("All done, take a look: %s" % csv_file)
def from_json(json_string): try: assert type(json_string) == str, str(json_string) + ' is not string' logger.info('from_json: Received string: ' + json_string) tokens = json_lexer.lex(json_string) json_object = json_parser.parse(tokens, is_root=True)[0] assert type(json_object) == dict, 'Dictionary object was not returned' logger.info('from_json: Returned object: ' + str(json_object)) return json_parser.parse(tokens, is_root=True)[0] except AssertionError as e: logger.error('from_json: ' + str(e))
def find_example(): spec = create_model(2, 3, 4) with open("temp/spec.json", "w") as jsonFile: jsonFile.write(json.dumps(spec, indent=4)) #with open("temp/spec.json", 'r') as f: # spec = json.load(f) index = 0 add_assertion(spec, EPS) add_solver(spec) model, assertion, solver, display = parse(spec) network_tf = "temp/network.tf" save_tf_model(network_tf, model, len(model.layers)-1) find = False for no in range(5): x0s = np.round(generate_rand((100, 2), INPUT_BOUND[0], INPUT_BOUND[1]), 4) example = [] res = refinepoly_run(x0s, None, network_tf, EPS, 2) print("Refine result", res) for j in res: success_lst = newApproach(model, x0s[j], '') if success_lst: print("Find example") print(x0s[j]) with open("temp/x{}.txt".format(index), "w") as exp: exp.write(str(x0s[j].tolist())) find = True index += 1 return find
def run_verify(zipped_args): start, end, args = zipped_args[0], zipped_args[1], zipped_args[2] with open(args.spec, 'r') as f: spec = json.load(f) args.pathX, args.pathY = get_dataset(args.dataset) bd_target, fa_target, sb_target = [], [], [] for target in range(start, end): args.target = target print( 'Backdoor target = {} with size = {}, total imgs = {}, num imgs = {}, rate = {}, and threshold = {}' .format(args.target, args.size, args.total_imgs, args.num_imgs, args.rate, args.threshold)) add_assertion(args, spec) add_solver(args, spec) model, assertion, solver, display = parse(spec) res, sbi = solver.solve(model, assertion) if res is not None: bd_target.append(res) if sbi is None: fa_target.append(res) else: sb_target.append(res) return bd_target, fa_target, sb_target
def main(): init(strip=not sys.stdout.isatty()) cprint(figlet_format('salty mike\'s salt parser', font='starwars'), 'white', 'on_blue', attrs=['dark', 'bold']) data_file = input( "Provide input file path (or press enter if file is \'in.json\'): ") if not data_file: data_file = 'in.json' conf_file = input( 'Provide configuration file path (or press enter if file is \'conf.properties\'): ' ) if not conf_file: conf_file = 'conf.properties' print() with open(conf_file) as c: conf = c.read().splitlines() with open(data_file, 'r') as json_file: hosts = json.loads(''.join(json_file)) for host in hosts: for k, v in host.items(): addResults(conf, k, json_parser.parse(host)) json_parser.output_dict = {} print_results()
def from_file(filename): tokens = [] with open(filename) as f: for _, line in enumerate(f): line = line.strip('\n') tokens += json_lexer.lex(line) f.close() return json_parser.parse(tokens, is_root=True)[0]
def generate_pandoc(directory, f, output_dir): file_settings = json_parser.parse(os.path.join(directory, f[:-3] + ".json")) file_real_path = os.path.realpath(os.path.join(directory, f)) out_real_path = os.path.realpath(os.path.join(output_dir, f[:-3] + ".html")) command = "pandoc -s -f markdown -t html %s --metadata pagetitle=\"%s\" -o %s -c '/style/markdown.css'" % \ (file_real_path, file_settings["title"], out_real_path) # print(command) os.system(command)
def main(): np.set_printoptions(threshold=20) parser = argparse.ArgumentParser(description='nSolver') parser.add_argument('--spec', type=str, default='spec.json', help='the specification file') parser.add_argument('--algorithm', type=str, help='the chosen algorithm') parser.add_argument('--threshold', type=float, help='the threshold in sprt') args = parser.parse_args() with open(args.spec, 'r') as f: spec = json.load(f) add_assertion(args, spec) add_solver(args, spec) model, assertion, solver, display = parse(spec) lower = model.lower upper = model.upper for i in range(50): pathX = 'benchmark/mnist_challenge/x_y/x' + str(i) + '.txt' pathY = 'benchmark/mnist_challenge/x_y/y' + str(i) + '.txt' x0s = np.array(ast.literal_eval(read(pathX))) y0s = np.array(ast.literal_eval(read(pathY))) for j in range(200): x0 = x0s[j] assertion['x0'] = str(x0.tolist()) output_x0 = model.apply(x0) lbl_x0 = np.argmax(output_x0, axis=1)[0] print('Data {}\n'.format(i * 200 + j)) print('x0 = {}'.format(x0)) print('output_x0 = {}'.format(output_x0)) print('lbl_x0 = {}'.format(lbl_x0)) print('y0 = {}\n'.format(y0s[j])) if lbl_x0 == y0s[j]: update_bounds(args, model, x0, lower, upper) print('Run at data {}\n'.format(i * 200 + j)) solver.solve(model, assertion) else: print('Skip at data {}'.format(i * 200 + j)) print('\n============================\n')
def get_tone_for_user(username, subreddit=None, comments_limit=1): """ Compute watson mood of user "username" on subreddit "subreddit" by looking at last comments. """ comments = [x.body for x in get_comments(username, comments_limit)] comment_to_tone = [] for comment in comments: json_data = query_watson(comment) tone = parse(json_data) comment_to_tone.append((comment, tone)) return comment_to_tone
def print_detail(): network_tf = "temp/network.tf" with open("temp/examples/spec.json", 'r') as f: spec = json.load(f) add_assertion(spec, 0.01) add_solver(spec) model, assertion, solver, display = parse(spec) x0 = np.array(ast.literal_eval(open("temp/examples/x2.txt", 'r').readline())) res = refinepoly_run(np.reshape(x0, (-1,2)), None, network_tf, 2) success_lst = newApproach(model, x0, '') if success_lst: print("Find example") assertion['x0'] = str(x0.tolist()) res = solver.solve(model, assertion) if res == 0: print("DeepPoly can't prove this")
def main(): np.set_printoptions(threshold=20) parser = argparse.ArgumentParser(description='nSolver') parser.add_argument('--spec', type=str, default='spec.json', help='the specification file') args = parser.parse_args() with open(args.spec, 'r') as f: spec = json.load(f) model, assertion, solver, display = parse(spec) solver.solve(model, assertion, display)
def run_attack(args): print( 'Backdoor target = {} with size = {}, total imgs = {}, num imgs = {}, and attack only = {} at position = {}' .format(args.target, args.size, args.total_imgs, args.num_imgs, args.atk_only, args.atk_pos)) with open(args.spec, 'r') as f: spec = json.load(f) args.pathX, args.pathY = get_dataset(args.dataset) add_assertion(args, spec) add_solver(args, spec) model, assertion, solver, display = parse(spec) res, stamp = solver.solve(model, assertion)
def process_json(jsonpath, outputpath, topclassname, public_fields=False, getset=True): to_load = file_handler.read(jsonpath) file_handler.create_folder_if_not_exist(outputpath) try: loaded = json_parser.parse(to_load) except ValueError: print "not a JSON file (maybe has errors?)" exit(-1) if len(loaded) == 0: print "empty json, exiting" exit(-1) structure = engine.prepare_structure(loaded, topclassname) texts = engine.generate_classes(structure, public_fields, getset) for classname, text in texts: path = "{}{}{}.java".format(outputpath, file_handler.sep, classname) file_handler.save(path, text)
label = 'wasted' elif label == '0': label = 'clean' elif label != 'unknown': label = 'repaired' if label not in groups: groups[label] = [advert] else: groups[label].append(advert) return groups if __name__ == '__main__': # parse program arguments parser = argparse.ArgumentParser() parser.add_argument('--path', type=str, help="Input json file path") parser.add_argument('--target', type=str, default='paint', help="Target variable to group the adverts") args = parser.parse_args() # parse advert labels group the adverts by the specified target variable labels = jp.parse(args.path, args.target, 'field') groups = group(labels) for label, adverts in groups.items(): print(label) for advert in adverts: print(advert) print()
def main(): np.set_printoptions(threshold=20) parser = argparse.ArgumentParser(description='nSolver') parser.add_argument('--spec', type=str, default='spec.json', help='the specification file') parser.add_argument('--algorithm', type=str, help='the chosen algorithm') parser.add_argument('--eps', type=float, help='the distance value') parser.add_argument('--has_ref', action='store_true', help='turn on/off refinement') parser.add_argument('--max_ref', type=int, default=20, help='maximum times of refinement') parser.add_argument('--ref_typ', type=int, default=0, help='type of refinement') parser.add_argument('--max_sus', type=int, default=1, help='maximum times of finding adversarial sample') parser.add_argument('--dataset', type=str, help='the data set for CEGAR experiments') parser.add_argument('--num_tests', type=int, default=100, help='maximum number of tests') args = parser.parse_args() with open(args.spec, 'r') as f: spec = json.load(f) add_assertion(args, spec) add_solver(args, spec) model, assertion, solver, display = parse(spec) lower = model.lower upper = model.upper pathX = 'benchmark/cegar/data/mnist_fc/' pathY = 'benchmark/cegar/data/labels/y_mnist.txt' y0s = np.array(ast.literal_eval(read(pathY))) for i in range(args.num_tests): assertion['x0'] = pathX + 'data' + str(i) + '.txt' x0 = np.array(ast.literal_eval(read(assertion['x0']))) output_x0 = model.apply(x0) lbl_x0 = np.argmax(output_x0, axis=1)[0] print('Data {}\n'.format(i)) print('x0 = {}'.format(x0)) print('output_x0 = {}'.format(output_x0)) print('lbl_x0 = {}'.format(lbl_x0)) print('y0 = {}\n'.format(y0s[i])) if lbl_x0 == y0s[i]: best_verified, best_failed = 0, 1e9 eps, step_eps = 0.01, 0.01 while True: t0 = time.time() args.eps = eps update_bounds(args, model, x0, lower, upper) print('Run at data {}\n'.format(i)) res = solver.solve(model, assertion) if res == 1: print('Verified at {:.3f}'.format(eps)) best_verified = max(best_verified, eps) elif res == 0: print('Failed at {:.3f}'.format(eps)) best_failed = min(best_failed, eps) else: break t1 = time.time() print('time = {}'.format(t1 - t0)) print('\n============================\n') if best_verified == round(best_failed - 0.001, 3): break if res == 1: if step_eps == 0.01: eps = round(eps + step_eps, 3) elif step_eps == -0.005: step_eps = 0.001 eps = round(eps + step_eps, 3) elif step_eps == 0.001: eps = round(eps + step_eps, 3) elif res == 0: if step_eps == 0.01: step_eps = -0.005 eps = round(eps + step_eps, 3) elif step_eps == -0.005: step_eps = -0.001 eps = round(eps + step_eps, 3) elif step_eps == -0.001: eps = round(eps + step_eps, 3) print("Image {} Verified at {:.3f} and Failed at {:.3f}".format( i, best_verified, best_failed)) else: print('Skip at data {}'.format(i)) print("Image {} Verified at {:.3f} and Failed at {:.3f}".format( i, -1, -1)) res = -1 print('\n============================\n')
import os import json_parser, file_supervisor global_settings = json_parser.parse('global.json') if (global_settings == None): print("Unable to open `global.json`, halting.") exit(-1) files = file_supervisor.supervise(directory=global_settings['BLOG_DIR']) file_supervisor.clean(global_settings['OUTPUT_DIR'], files) for directory in files.keys(): for f in files[directory]: file_supervisor.generate_pandoc( os.path.join(global_settings['BLOG_DIR'], directory), f, os.path.join(global_settings['OUTPUT_DIR'], directory)) if (global_settings['STATIC_SERVER']): import static_server static_server.serve(directory=global_settings['OUTPUT_DIR'])
def main(): np.set_printoptions(threshold=20) parser = argparse.ArgumentParser(description='nSolver') parser.add_argument('--spec', type=str, default='spec.json', help='the specification file') parser.add_argument('--algorithm', type=str, help='the chosen algorithm') parser.add_argument('--threshold', type=float, help='the threshold in sprt') parser.add_argument('--eps', type=float, help='the distance value') parser.add_argument('--dataset', type=str, help='the data set for rnn experiments') args = parser.parse_args() with open(args.spec, 'r') as f: spec = json.load(f) add_assertion(args, spec) add_solver(args, spec) model, assertion, solver, display = parse(spec) lower = model.lower[0] upper = model.upper[0] if args.dataset == 'jigsaw': pathX = 'benchmark/rnn/data/jigsaw/' pathY = 'benchmark/rnn/data/jigsaw/labels.txt' elif args.dataset == 'wiki': pathX = 'benchmark/rnn/data/wiki/' pathY = 'benchmark/rnn/data/wiki/labels.txt' y0s = np.array(ast.literal_eval(read(pathY))) for i in range(100): assertion['x0'] = pathX + 'data' + str(i) + '.txt' x0 = np.array(ast.literal_eval(read(assertion['x0']))) shape_x0 = (int(x0.size / 50), 50) model.shape = shape_x0 model.lower = np.full(x0.size, lower) model.upper = np.full(x0.size, upper) output_x0 = model.apply(x0) lbl_x0 = np.argmax(output_x0, axis=1)[0] print('Data {}\n'.format(i)) print('x0 = {}'.format(x0)) print('output_x0 = {}'.format(output_x0)) print('lbl_x0 = {}'.format(lbl_x0)) print('y0 = {}\n'.format(y0s[i])) if lbl_x0 == y0s[i]: print('Run at data {}\n'.format(i)) solver.solve(model, assertion) else: print('Skip at data {}'.format(i)) print('\n============================\n')
def main(): np.set_printoptions(threshold=20) parser = argparse.ArgumentParser(description='nSolver') parser.add_argument('--spec', type=str, default='spec.json', help='the specification file') parser.add_argument('--algorithm', type=str, help='the chosen algorithm') parser.add_argument('--threshold', type=float, help='the threshold in sprt') parser.add_argument('--eps', type=float, help='the distance value') parser.add_argument('--has_ref', action='store_true', help='turn on/off refinement') parser.add_argument('--max_ref', type=int, default=20, help='maximum times of refinement') parser.add_argument('--ref_typ', type=int, default=0, help='type of refinement') parser.add_argument('--max_sus', type=int, default=1, help='maximum times of finding adversarial sample') parser.add_argument('--dataset', type=str, help='the data set for fairness experiments') parser.add_argument('--num_tests', type=int, default=100, help='maximum number of tests') args = parser.parse_args() with open(args.spec, 'r') as f: spec = json.load(f) add_assertion(args, spec) add_solver(args, spec) model, assertion, solver, display = parse(spec) ''' if args.dataset == 'bank': pathX = 'benchmark/fairness/bank/data/' pathY = 'benchmark/fairness/bank/data/labels.txt' elif args.dataset == 'census': pathX = 'benchmark/fairness/census/data/' pathY = 'benchmark/fairness/census/data/labels.txt' elif args.dataset == 'credit': pathX = 'benchmark/fairness/credit/data/' pathY = 'benchmark/fairness/credit/data/labels.txt' ''' if args.dataset == 'bank': pathX = '../benchmark/fairness/bank/data/' #debug pathY = '../benchmark/fairness/bank/data/labels.txt' #debug elif args.dataset == 'census': pathX = '../benchmark/fairness/census/data/' #debug pathY = '../benchmark/fairness/census/data/labels.txt' #debug elif args.dataset == 'credit': pathX = '../benchmark/fairness/credit/data/' #debug pathY = '../benchmark/fairness/credit/data/labels.txt' #debug y0s = np.array(ast.literal_eval(read(pathY))) for i in range(args.num_tests): assertion['x0'] = pathX + 'data' + str(i) + '.txt' x0 = np.array(ast.literal_eval(read(assertion['x0']))) output_x0 = model.apply(x0) lbl_x0 = np.argmax(output_x0, axis=1)[0] print('Data {}\n'.format(i)) print('x0 = {}'.format(x0)) print('output_x0 = {}'.format(output_x0)) print('lbl_x0 = {}'.format(lbl_x0)) print('y0 = {}\n'.format(y0s[i])) if lbl_x0 == y0s[i]: print('Run at data {}\n'.format(i)) solver.solve(model, assertion) else: print('Skip at data {}'.format(i)) print('\n============================\n')
import time import cl_parser import json_parser from kafka_avro_producer import KafkaAvroProducer from kafka_json_producer import KafkaJsonProducer args = cl_parser.parse() list_of_stations = json_parser.parse() if args.format == "avro": kafka_producer = KafkaAvroProducer(args.brokers, args.schema_registry_url) else: kafka_producer = KafkaJsonProducer(args.brokers) while True: kafka_producer.send_message_to_kafka(list_of_stations, args.topic) time.sleep(args.interval)
entropyMap = {} for word, wordsPerLabel in histogram.items(): histogram[word] = {label:count / advertsPerLabel[label] for (label, count) in wordsPerLabel.items()} entropyMap[word] = {label:abs(freq - (sum(histogram[word].values()) - freq)) for (label, freq) in histogram[word].items()} return entropyMap if __name__ == '__main__': # parse program arguments parser = argparse.ArgumentParser() parser.add_argument('--histpath', type=str, default=DEFAULT_HISTOGRAM, help="histogram file path") parser.add_argument('--labelpath', type=str, default=DEFAULT_LABELS, help="label file path") parser.add_argument('--target', type=str, default='paint', help="Target variable to group the adverts") args = parser.parse_args() # parse and preprocess advert descriptions advertLabelMap = jp.parse(args.labelpath, args.target, 'field') labelAdvertMap = ag.group(advertLabelMap) # compute the number of adverts per label advertsPerLabel = {} for label, adverts in labelAdvertMap.items(): advertsPerLabel[label] = len(adverts) # read the histogram from file histogram = json.load(open(args.histpath)) hg.printHist(histogram) # print words with high entropy entropyMap = entropy(histogram, advertsPerLabel) for word, entropy in entropyMap.items(): if sum(entropy.values()) > 3:
def main(): np.set_printoptions(threshold=20) parser = argparse.ArgumentParser(description='nSolver') parser.add_argument('--spec', type=str, default='spec.json', help='the specification file') parser.add_argument('--algorithm', type=str, help='the chosen algorithm') parser.add_argument('--threshold', type=float, help='the threshold in sprt') parser.add_argument('--eps', type=float, help='the distance value') parser.add_argument('--dataset', type=str, help='the data set for ERAN experiments') args = parser.parse_args() with open(args.spec, 'r') as f: spec = json.load(f) add_assertion(args, spec) add_solver(args, spec) model, assertion, solver, display = parse(spec) lower = model.lower upper = model.upper if args.dataset == 'cifar_conv': pathX = 'benchmark/eran/data/cifar_conv/' pathY = 'benchmark/eran/data/labels/y_cifar.txt' elif args.dataset == 'cifar_fc': pathX = 'benchmark/eran/data/cifar_fc/' pathY = 'benchmark/eran/data/labels/y_cifar.txt' elif args.dataset == 'mnist_conv': pathX = 'benchmark/eran/data/mnist_conv/' pathY = 'benchmark/eran/data/labels/y_mnist.txt' elif args.dataset == 'mnist_fc': pathX = 'benchmark/eran/data/mnist_fc/' pathY = 'benchmark/eran/data/labels/y_mnist.txt' y0s = np.array(ast.literal_eval(read(pathY))) for i in range(10): assertion['x0'] = pathX + 'data' + str(i) + '.txt' x0 = np.array(ast.literal_eval(read(assertion['x0']))) output_x0 = model.apply(x0) lbl_x0 = np.argmax(output_x0, axis=1)[0] print('Data {}\n'.format(i)) print('x0 = {}'.format(x0)) print('output_x0 = {}'.format(output_x0)) print('lbl_x0 = {}'.format(lbl_x0)) print('y0 = {}\n'.format(y0s[i])) t0 = time.time() if lbl_x0 == y0s[i]: update_bounds(args, model, x0, lower, upper) print('Run at data {}\n'.format(i)) solver.solve(model, assertion) else: print('Skip at data {}'.format(i)) t1 = time.time() print('time = {}'.format(t1 - t0)) print('\n============================\n')
print('{0: <25}'.format("\n" + word), end="") for label in labelCountMap.keys(): print('{0: >4}'.format(labelCountMap[label]), end="") print('{0: <10}'.format(" " + label), end="") if __name__ == '__main__': # parse program arguments parser = argparse.ArgumentParser() parser.add_argument('--datapath', type=str, help="Input json file path") parser.add_argument('--labelpath', type=str, help="Input json file path") parser.add_argument('--target', type=str, default='paint', help="Target variable to group the adverts") args = parser.parse_args() # parse and preprocess advert descriptions advertDescriptionMap = jp.parse(args.datapath, 'description', 'data') advertLabelMap = jp.parse(args.labelpath, args.target, 'field') labelAdvertMap = ag.group(advertLabelMap) advertsPerLabel = {} for label, adverts in labelAdvertMap.items(): advertsPerLabel[label] = len(adverts) print(advertsPerLabel) input() # create histograms for each label preprocessor.preprocess(advertDescriptionMap) histogram = histLabels(advertDescriptionMap, labelAdvertMap) printHist(histogram) # write histogram to a json file
def mnist_challenge(model_path, _range, x_path, y_path=None): model_name = str(model_path).split('/')[-2] #list_file = [] #with open("result_newapproach/{}_refinepoly_failed.csv".format(model_name), 'r') as csvfile: # list_file_raw = csv.reader(csvfile, delimiter=',') # for test in list_file_raw: # list_file.append((test[0], ast.literal_eval(test[1]))) with open(model_path, 'r') as f: spec = json.load(f) add_assertion(spec, EPS) add_solver(spec) model, assertion, solver, display = parse(spec) list_potential2, unknow2, find_exp2 = [], [], [] index = 0 for i in _range: #for i in list_file: testfileName = i pathX = x_path + str(testfileName) + ".txt" #i -> i[0] pathY = None if y_path is None else y_path + str( testfileName) + ".txt" #i -> i[0] x0s = np.array(ast.literal_eval(read(pathX))) y0s = None if y_path is None else np.array( ast.literal_eval(read(pathY))) x0s = x0s if len(x0s.shape) == 2 else np.array([x0s]) list_potential, unknow, find_exp, csv_result = [], [], [], [] for j in range(x0s.shape[0]): #for j in i[1]: #print('Index:', index) #index += 1 x0 = x0s[j] assertion['x0'] = str(x0.tolist()) output_x0 = model.apply(x0) lbl_x0 = np.argmax(output_x0, axis=1)[0] y0 = y0s[j] if y0s is not None else lbl_x0 if lbl_x0 != y0: print("Skip") continue start = time.time() #res = solver.solve(model, assertion) res = refinepoly_run(np.array([x0]), np.array([y0])) print("DeepPoly time:", time.time() - start) if res == 1 or res == 2: print("DeepPoly can prove this model") continue else: print("DeepPoly can't prove this model:", j) list_potential.append(j) #res = testCegar(model, str(x0.tolist())) #if res == 1: # list_potential.append(j) #elif res == 2: # continue #else: # print("Cegar also can't prove this") # unknow.append(j) start = time.time() success_lst = newApproach(model, x0, '') if success_lst: print("Find example:", j) find_exp.append((j, success_lst)) csv_result.append([ str(testfileName) + '_' + str(j), time.time() - start, success_lst ]) if list_potential: list_potential2.append([testfileName, list_potential]) with open( 'result_newapproach/{}_refinepoly_failed.csv'.format( model_name), 'a') as f: writer = csv.writer(f) writer.writerow([testfileName, list_potential]) #unknow2.append((i,unknow)) find_exp2.append((testfileName, find_exp)) with open( 'result_newapproach/{}_approach_cp_refine_result.csv'.format( model_name), 'a') as f: writer = csv.writer(f) writer.writerows(csv_result) print("potential:", list_potential2) #print("Unknow:", unknow2) print("Example Find:", find_exp2)
def get_raw_tone(string): json_data = query_watson(string) tone = parse(json_data) return tone
# replace any number, http(s) address and email address with the words 'number', 'httpaddr' and 'emailaddr' text = re.sub(r'[0-9]+', ' number ', text) text = re.sub('(http|https)://[^\s]*', 'httpaddr', text) text = re.sub('[^\s]+@[^\s]+', 'emailaddr', text) # split the text into words and get rid of any punctuation text = re.findall(r"[\w']+", text) # remove any non-alphanumeric characters and remove words with less than 2 letters for index, word in enumerate(text): word = re.sub('[^a-zA-Z0-9]', '', word) if (len(word) > 1): text[index] = word else: del text[index] # replace the raw description text with a preprocessed list of words descriptions[key] = text if __name__ == '__main__': # parse program arguments parser = argparse.ArgumentParser() parser.add_argument('--path', type=str, help="Input json file path") args = parser.parse_args() # parse and preprocess advert descriptions descriptions = jp.parse(args.path, 'description', 'data') preprocess(descriptions) print(descriptions)
def get_services(): with open("prices.json", "r") as file: services = parse(file) return services["prices"]
def main(): screen_width = 90 screen_height = 60 tcod.console_set_custom_font( 'terminal8x12_gs_tc.png', tcod.FONT_TYPE_GRAYSCALE | tcod.FONT_LAYOUT_TCOD) tcod.console_init_root(screen_width, screen_height, 'Spaceship', False, renderer=tcod.RENDERER_SDL2, vsync=True) tcod.console_map_ascii_code_to_font(7, 12, 2) tcod.console_map_ascii_code_to_font(9, 13, 2) object_view_width = 40 object_view_height = 40 main_console = Console(screen_width, screen_height, order='F') object_console = Console(screen_width, screen_height, order='F') parse(all_objects, object_list) frame = all_objects['base'][1] spaceship_x, spaceship_y = 0, 0 file_path = getcwd() + '/spaceship.dat' if not path.exists(file_path) or path.getsize(file_path) == 0: pickle.dump(Object('Spaceship', 50, 50), open(file_path, 'wb+')) try: spaceship = pickle.load(open(getcwd() + '/spaceship.dat', 'rb')) except pickle.UnpicklingError: pickle.dump(Object('Spaceship', 50, 50), open(file_path, 'wb+')) spaceship = pickle.load(open(file_path, 'rb')) examine_x = screen_width - 50 examine_y = 1 examine_width = 50 examine_height = screen_height - 2 examine_menu = None build_menu_object_key_list = ['default'] for x in range(spaceship.width): for y in range(spaceship.height): for part in spaceship.tiles[x][y]: for object in object_list: if not 'json_id' in part.__dict__ and object.name == part.name: part.json_id = object.json_id print('Assigned {} to {}'.format( part.name, object.json_id)) if 'json_id' in part.__dict__ and object.json_id == part.json_id: part.fg = object.fg part.bg = object.bg part.char = object.char part.sorting_index = object.sorting_index part.mass = object.mass break else: print( 'Could not find corresponding JSON part for existing part {}' .format(part.name)) spaceship.add_part(0, 0, frame, override=True) spaceship.calculate_rooms() viewing_mode = 0 num_modes = len(view_dict) + 2 selection_index = 0 max_select_index = 0 build_menu_open = False selection_box = (0, 0), (0, 0) screen_offset_x, screen_offset_y = 0, 0 selection_console = Console(screen_width, screen_height, order='F') selection_console.clear(bg=tcod.lightest_blue) while True: response = None prev_select_index = selection_index prev_selection_coord = selection_box[0] for event in tcod.event.get(): if event.type == 'KEYDOWN': response = {} input_resp = handle_input(event) if input_resp: response.update(input_resp) if build_menu_open and ord('z') >= event.sym >= ord('a'): response['select_item'] = event.sym - ord('a') elif event.type == 'MOUSEBUTTONDOWN' or event.type == 'MOUSEBUTTONUP' or event.type == 'MOUSEMOTION': response = handle_mouse(event) elif event.type == 'QUIT': pickle.dump(spaceship, open(file_path, 'wb+')) return if response: if response.get('move'): move = response.get('move') if examine_menu: examine_menu.selected_index += move[1] if move[0] != 0: if examine_menu.try_change_option(move[0]): spaceship.update_all() else: screen_offset_x = max(0, screen_offset_x + move[0]) screen_offset_y = max(0, screen_offset_y + move[1]) elif response.get('viewing_mode'): viewing_mode = (viewing_mode + response.get('viewing_mode')) % num_modes elif response.get('navigate'): selection_index += response.get('navigate') if selection_index < 0: selection_index += max_select_index elif selection_index >= max_select_index: selection_index -= max_select_index if response.get('build'): if not build_menu_open: build_menu_open = True build_menu_object_key_list = [] selection_index = 0 if response.get('delete') and not build_menu_open: if spaceship.is_valid_coord( selection_box[0][0] - spaceship_x, selection_box[0][1] - spaceship_y) and selection_index < len( spaceship.tiles[selection_box[0][0] - spaceship_x][ selection_box[0][1] - spaceship_y]): part_name = spaceship.tiles[ selection_box[0][0] - spaceship_x][selection_box[0][1] - spaceship_y][selection_index].name for x in range(selection_box[0][0], selection_box[1][0]): for y in range(selection_box[0][1], selection_box[1][1]): for part in spaceship.tiles[x][y]: if part.name == part_name: spaceship.remove_part( x - spaceship_x, y - spaceship_y, part) elif (response.get('select') or response.get('mouse_move_lheld') or response.get('lclick_down')) and build_menu_open: if type(get_object(build_menu_object_key_list)) is list: if response.get('mouse_move_lheld'): selection_box = response.get('mouse_move_lheld'), ( response.get('mouse_move_lheld')[0] + 1, response.get('mouse_move_lheld')[1] + 1) for x in range(selection_box[0][0], selection_box[1][0]): for y in range(selection_box[0][1], selection_box[1][1]): spaceship.add_part( x - spaceship_x, y - spaceship_y, get_object(build_menu_object_key_list) [selection_index]) else: build_menu_object_key_list.append( list(get_object(build_menu_object_key_list).keys()) [selection_index]) selection_index = 0 elif response.get('run'): for x in range(spaceship.width): for y in range(spaceship.height): for part in [ part for part in spaceship.tiles[x][y] if part.runnable ]: part.runnable.run(spaceship, part, x, y) spaceship.update_all() if response.get('rclick_down'): selection_box = response.get('rclick_down'), response.get( 'rclick_down') elif response.get('lclick_down'): selection_box = response.get('lclick_down'), response.get( 'lclick_down') elif response.get('mouse_move_rheld'): selection_box = selection_box[0], ( response.get('mouse_move_rheld')[0] + 1, response.get('mouse_move_rheld')[1] + 1) elif response.get( 'mouse_move' ) and selection_box[1][0] - selection_box[0][ 0] <= 1 and selection_box[1][1] - selection_box[0][1] <= 1: selection_box = response.get('mouse_move'), ( response.get('mouse_move')[0] + 1, response.get('mouse_move')[1] + 1) elif response.get('select_item') is not None: index = response.get('select_item') if index < max_select_index: selection_index = index if type(get_object(build_menu_object_key_list)) is dict: build_menu_object_key_list.append( list( get_object(build_menu_object_key_list).keys()) [selection_index]) selection_index = 0 if response.get('escape'): if build_menu_open: if len(build_menu_object_key_list) == 0: build_menu_open = False selection_index = 0 else: build_menu_object_key_list.pop(-1) elif examine_menu: examine_menu = None if prev_select_index != selection_index or prev_selection_coord != selection_box[ 0]: if spaceship.is_valid_coord( selection_box[0][0] - spaceship_x, selection_box[0][1] - spaceship_y) and selection_index < len( spaceship.tiles[selection_box[0][0] - spaceship_x][ selection_box[0][1] - spaceship_y]): examine_menu = PartMenu( spaceship, spaceship.tiles[selection_box[0][0] - spaceship_x][selection_box[0][1] - spaceship_y][selection_index], examine_width, examine_height, selection_box[0][0], selection_box[0][1]) else: examine_menu = None tcod.console_flush() main_console.clear(bg=tcod.black) selection_pos = selection_box[0] object_console.clear() render_object(object_console, spaceship, spaceship_x, spaceship_y, viewing_mode) object_console.blit(main_console, 0, 0, screen_offset_x, screen_offset_y, object_view_width, object_view_height) # main_console.put_char(selection_box[0][0], selection_box[0][1], ord('X')) # main_console.fg[selection_box[0][0], selection_box[0][1]] = tcod.cyan if build_menu_open: iterable = get_object(build_menu_object_key_list) if type(iterable) is list: iterable = [item.name for item in iterable] else: iterable = [item.capitalize() for item in iterable] for i in range(len(iterable)): main_console.print(screen_width - 50, i, chr(ord('a') + i) + ' - ' + iterable[i], bg=(tcod.desaturated_blue if selection_index == i else tcod.black)) main_console.fg[screen_width - 50, i] = tcod.light_green max_select_index = len(iterable) else: main_console.print(0, screen_height - 1, str(viewing_mode)) if spaceship_x <= selection_pos[ 0] < spaceship_x + spaceship.width and spaceship_y <= selection_pos[ 1] < spaceship_y + spaceship.height: room = spaceship.find_room(selection_pos[0] - spaceship_x, selection_pos[1] - spaceship_y) parts = [ part for part in spaceship.tiles[selection_pos[0] - spaceship_x][selection_pos[1] - spaceship_y] ] if room: gas_y = len(parts) + 2 for gas in room.gas_content: main_console.print( 20, gas_y, '{1}: {0:.2f} kPa'.format( components.calc_pressure( room.gas_content[gas], len(room.tiles)), gas.capitalize())) gas_y += 1 for i in range(len(parts)): part = parts[i] string = part.name if part.gas_grid: string += ' {}'.format(part.gas_grid.index) main_console.print( 20, i + 1, string, bg=(tcod.desaturated_blue if selection_index == i else tcod.black)) max_select_index = len(parts) if examine_menu: examine_menu.draw() examine_menu.console.blit(main_console, examine_x, examine_y, 0, 0, examine_width, examine_height) dsel_x = selection_box[1][0] - selection_box[0][0] dsel_y = selection_box[1][1] - selection_box[0][1] if dsel_x > 0 and dsel_y > 0: selection_console.blit(main_console, selection_box[0][0], selection_box[0][1], 0, 0, dsel_x, dsel_y, bg_alpha=0.25) main_console.blit(tcod.console._root_console, width=screen_width, height=screen_height, bg_alpha=1)
merge(a[key], b[key], path + [str(key)]) elif a[key] == b[key]: pass else: raise Exception('Conflict at %s' % '.'.join(path + [str(key)])) else: a[key] = b[key] return a if __name__ == '__main__': import json from json_parser import parse parsed = {} with open('translates/ru.json', 'r') as fp: parsed = json.load(fp) keys_by_translate = {} parse(parsed, [], 0, keys_by_translate) arr = fill_arr(keys_by_translate) arr = transform_list_to_dict(arr) cor_obj = arr[0] for obj in arr[1:]: cor_obj = merge(cor_obj, obj) with open('result.json', 'w') as fp: json.dump(cor_obj, fp, ensure_ascii=False, indent=4)
broccoli.broccoli ~~~~~~~~~~~~~ This project aims to ease the computation of certain problems by introducing tools parallelization and multiprocessing. :copyright: 2015 Manuel Martins, see AUTHORS for more details :license: Apache 2.0, see LICENSE for more details """ import argparse from logger import initialize as logger_init import json_parser from job import Job import logging if __name__ == '__main__': args_parser = argparse.ArgumentParser( description='Main entry point for Broccoli Module. Usage: python -m broccoli -i <input.json>' ) args_parser.add_argument('-v', '--verbose', help='increase output verbosity', action='store_true', required=False) args_parser.add_argument('-i', '--input', help='input json file / string', action='store', dest='input', required=True) logging.info('Broccoli - Initializing...') args = args_parser.parse_args() json = json_parser.parse(args.input) logger_init(json, args.verbose) job = Job(json) job.start()