def test_loadFileArgsError(self): try: ujson.load("[]") except TypeError: pass else: assert False, "expected TypeError"
def test_loadFileLikeObject(self): class filelike: def read(self): try: self.end except AttributeError: self.end = True return "[1,2,3,4]" f = filelike() self.assertEquals([1, 2, 3, 4], ujson.load(f)) f = filelike() assert_array_equal(np.array([1, 2, 3, 4]), ujson.load(f, numpy=True))
def test_loadFileLikeObject(self): class filelike: def read(self): try: self.end except AttributeError: self.end = True return "[1,2,3,4]" f = filelike() self.assertEqual([1, 2, 3, 4], ujson.load(f)) f = filelike() assert_array_equal(np.array([1, 2, 3, 4]), ujson.load(f, numpy=True))
def main(): categories = {"essential constraints": "Essential"} cat_counters = {n: [] for n in categories} measures = [("accuracy", calc_accuracy), ("expected accuracy", calc_expected_accuracy), ("redundancy", calc_redundancy)] speed = [] for name in files: print(name) csv_file = local("data/csv/{}.csv".format(name)) if not os.path.isfile(csv_file): raise Exception("File not found: {}".format(csv_file)) groups_file = local("data/groups/{}.txt".format(name)) silent = False # TODO True if not os.path.isfile(groups_file): with open(groups_file, "w+") as f: print("{\n\t\"Tables\":\n\t\t[\n\n\t\t]\n}", file=f, flush=True) groups_file = None silent = False t_before = time.time() solutions = workflow.main(csv_file, groups_file, False, silent=True, parse_silent=silent) t_elapsed = time.time() - t_before speed.append(t_elapsed) print("\tTook: {:.2f} seconds".format(t_elapsed)) truth_file = local("data/truth/{}.txt".format(name)) # constraint_map = {c.name: c for c in workflow.constraint_list} if os.path.isfile(truth_file): with open(truth_file) as f: json_data = json.load(f) counters = {n: CategoryCounter(json_data[k]) for n, k in categories.items()} for constraint in solutions.solutions: for solution in solutions.get_solutions(constraint): for counter in counters.values(): counter.count(constraint, solution) for c_name, c in counters.items(): cat_counters[c_name].append(c) for m_name, m_func in measures: res, actual, expected = m_func([c], per_file=False) print("\t{} for {}: {:.2%} ({} of {})" .format(m_name.capitalize(), c_name, res, actual, expected)) print_constraints("MISSING", c.not_found) redundant = CategoryCounter.filter_constraints(c.not_present, is_excel_constraint) print_constraints("REDUNDANT", redundant) print() elif groups_file is not None: with open(truth_file, "w+") as f: print("{\n\t\"Essential\":\n\t\t{\n\n\t\t}\n}", file=f, flush=True) print_truth.main(name) for n in categories: print("Time elapsed: {:.2f}s (total), {:.2f}s (average)".format(sum(speed), numpy.average(speed))) for m_name, m_func in measures: res, not_zero, file_count = m_func(cat_counters[n], per_file=True) print("{} {} per file ({} of {} files): {:.2%}".format(n.capitalize(), m_name, not_zero, file_count, res)) res, actual, expected = m_func(cat_counters[n], per_file=False) print("Total {} {}: {:.2%} ({} of {})".format(n, m_name, res, actual, expected)) print() return cat_counters
def show_result(): print("before") os.system("python ../dirbot/spiders/webcrawler.py 1") print("after") with open('result.json') as data_file: data = json.load(data_file) plec = [] wiek = [] wiekrozpoczecia = [] wiekdiagnozy = [] czastrwania = [] edss = [] kursms = [] nawrot12 = [] nawrot24 = [] for x in data: v1 = x["typ"] v2 = x["opis"] v3 = x["ilosc"] v1 = " ".join(str(v) for v in v1) v2 = " ".join(str(v) for v in v2) v3 = " ".join(str(v) for v in v3) if v1 == "Distribution of gender": plec.extend([[v2, int(v3)]]) if v1 == "Age": wiek.extend([[v2, int(v3)]]) if v1 == "Age at onset": wiekrozpoczecia.extend([[v2, int(v3)]]) if v1 == "Age at diagnosis": wiekdiagnozy.extend([[v2, int(v3)]]) if v1 == "Duration of MS": czastrwania.extend([[v2, int(v3)]]) if v1 == "EDSS at last visit": edss.extend([[v2, int(v3)]]) if v1 == "Last MSCourse": kursms.extend([[v2, int(v3)]]) if v1 == "Relapse count over last 12 months": nawrot12.extend([[v2, int(v3)]]) if v1 == "Relapse count over last 24 months": nawrot24.extend([[v2, int(v3)]]) my_dict = { 'plec': plec, 'wiek': wiek, 'wiekrozpoczecia': wiekrozpoczecia, 'wiekdiagnozy': wiekdiagnozy, 'czastrwania': czastrwania, 'edss': edss, 'kursms': kursms, 'nawrot12': nawrot12, 'nawrot24': nawrot24 } return render_template('result.html', my_dict=my_dict)
def count_constraints(self, solutions): truth_file = self._get_truth_file() with open(truth_file) as file: json_data = json.load(file) self._counter = ConstraintCounter({ **json_data["Essential"], **json_data["Non-trivial"] }) for constraint in solutions.solutions: for solution in solutions.get_solutions(constraint): self._counter.register(constraint, solution)
def get_diff_cache(filename): """ Loads a cache result by filename :param filename: :return: """ file_name = _cache_directory + filename + '.json' if os.path.isfile(file_name): with open(file_name) as data_file: data = ujson.load(data_file) return data # if the file doesn't exist return None
def show_result(): print ("before") os.system("python ../dirbot/spiders/crawlmycrownspider.py 1") print ("after") # call(['python ../dirbot/spiders/crawlmycrownspider.py shell=True']) with open('result.json') as data_file: data = json.load(data_file) gender = [] age = [] ageonset = [] agediagnosis = [] durationms = [] edss = [] lastms = [] relapse12 = [] relapse24 = [] for x in data: v1 = x["type"] v2 = x["description"] v3 = x["count"] v1 = " ".join(str(v) for v in v1) v2 = " ".join(str(v) for v in v2) v3 = " ".join(str(v) for v in v3) if v1 == "Distribution of gender": gender.extend([[v2, int(v3)]]) if v1 == "Age": age.extend([[v2,int(v3)]]) if v1 == "Age at onset": ageonset.extend([[v2,int(v3)]]) if v1 == "Age at diagnosis": agediagnosis.extend([[v2, int(v3)]]) if v1 == "Duration of MS": durationms.extend([[v2, int(v3)]]) if v1 == "EDSS at last visit": edss.extend([[v2, int(v3)]]) if v1 == "Last MSCourse": lastms.extend([[v2, int(v3)]]) if v1 == "Relapse count over last 12 months": relapse12.extend([[v2, int(v3)]]) if v1 == "Relapse count over last 24 months": relapse24.extend([[v2, int(v3)]]) my_dict={'gender': gender, 'age': age, 'ageonset':ageonset, 'agediagnosis': agediagnosis,'durationms':durationms, 'edss': edss,'lastms': lastms,'relapse12': relapse12,'relapse24':relapse24} return render_template('result.html', my_dict=my_dict)
def measure_size(self, config_file): with open(config_file) as file: tables = json.load(file)["Tables"] cell_count = 0 vector_count = 0 for table in tables: bounds = Bounds(table["Bounds"]) cell_count += bounds.columns() * bounds.rows() if "Orientation" not in table or table["Orientation"].lower( ) == "none": vector_count += bounds.columns() + bounds.rows() elif table["Orientation"].lower()[0:3] == "row": vector_count += bounds.rows() elif table["Orientation"].lower()[0:3] == "col": vector_count += bounds.columns() else: raise Exception("Unexpected orientation: {}".format( table["Orientation"])) self._cells = cell_count self._vectors = vector_count self._tables = len(tables)
def load_tid_data(): global g_train_image_paths global g_train_scores global g_val_image_paths global g_val_scores print("Loading tid training set and val set") print 'tip_base_images_path = ' + tip_base_images_path files = glob.glob(tip_base_images_path + "*.bmp") files = sorted(files) f = open(tip_dataset_path, 'r') data = json.load(f) image_size = 0 for item in data: image_id = item['image_id'] label = item['label'] file_path = tip_base_images_path + image_id + '.bmp' file_path_BMP = tip_base_images_path + image_id + '.BMP' if os.path.exists(file_path): g_train_image_paths.append(file_path) g_train_scores.append(label) image_size = image_size + 1 elif os.path.exists(file_path_BMP): g_train_image_paths.append(file_path_BMP) g_train_scores.append(label) image_size = image_size + 1 g_train_image_paths = np.array(g_train_image_paths) g_train_scores = np.array(g_train_scores, dtype='float32') g_val_image_paths = g_train_image_paths[-5000:] g_val_scores = g_train_scores[-5000:] train_image_paths = g_train_image_paths[:-5000] train_scores = g_train_scores[:-5000] print('Train set size : ', train_image_paths.shape, train_scores.shape) print('Val set size : ', g_val_image_paths.shape, g_val_scores.shape) print('Train and validation datasets ready !')
def test_loadFile(self): f = StringIO("[1,2,3,4]") self.assertEquals([1, 2, 3, 4], ujson.load(f)) f = StringIO("[1,2,3,4]") assert_array_equal(np.array([1, 2, 3, 4]), ujson.load(f, numpy=True))
def main(): info = [ ("Tables", lambda e: e.tables, "grey", lambda x: str(x)), # ("Vectors", lambda e: e.vectors, "cyan", lambda x: str(x)), ("Cells", lambda e: e.cells, "black", lambda x: str(x)), ("Intended Constraints", lambda e: e.counter.count(relevant=True, supported=True), "magenta", lambda x: str(x)), ("Recall", lambda e: recall(e.counter), "green", lambda x: "{:.2f}".format(x)), ("Recall Supported", lambda e: recall(e.counter, True), "lightgreen", lambda x: "{:.2f}".format(x)), ("Precision", lambda e: precision(e.counter), "yellow", lambda x: "{:.2f}".format(x)), ("Runtime", lambda e: numpy.average(e.running_times(1)), "red", lambda x: "{:.2f}s".format(x)) ] with open(local("data/data.txt")) as data_file: data_json = json.load(data_file) categories = ["Exercises", "Tutorials", "Data"] stats = {} for category in categories: files = data_json[category] stats[category] = [] for file in files: experiment = Experiment(file) print(file) for accident in experiment.counter.additional: print("\tADDITIONAL:\t{}".format(accident)) for missing in experiment.counter.missed: print("\tMISSING:\t{}".format(missing)) entry = {t[0]: t[1](experiment) for t in info} entry["relevant.found"] = experiment.counter.count( relevant=True, found=True) entry["extra.found"] = experiment.counter.count(relevant=False, found=True) stats[category].append(entry) maximal = { t[0]: max( max(stats[t[0]] for stats in cat_stats) for cat_stats in stats.values()) for t in info } colors = dict(zip(categories, ["red", "yellow", "black", "green"])) print("\n" * 2) max_label = list(t[3](maximal[t[0]]) for t in info) global_plot = Plotter("comparison", list(t[0] for t in info), upper=max_label) test = {"Exercises": (30, 2), "Tutorials": (46, 21), "Data": (6, 0)} for category, cat_stats in [(cat, stats[cat]) for cat in categories]: print(category) plotter = Plotter("benchmark_" + category) averages = [] errors = [] for name, f, color, pretty in info: plotter.add_category( name, list(stats[name] / maximal[name] for stats in cat_stats), color) result_string = "\tAverage {name}: {average} - total: {total}" average = numpy.average( list(stats[name] for stats in cat_stats)) averages.append(average / maximal[name]) errors.append( numpy.std(list(stats[name] for stats in cat_stats)) / maximal[name]) total = sum(stats[name] for stats in cat_stats) print( result_string.format(name=name.lower(), average=pretty(average), total=pretty(total))) relevant = sum(stats["relevant.found"] for stats in cat_stats) print("\tRelevant found: {}".format(relevant)) extra = sum(stats["extra.found"] for stats in cat_stats) print("\tExtra found: {}".format(extra)) relevant_expected, extra_expected = test[category] if relevant_expected != relevant: print( "ERROR ({}): found {} instead of {} relevant constraints". format(category, relevant, relevant_expected)) if extra_expected != extra: print("ERROR ({}): found {} instead of {} extra constraints". format(category, extra, extra_expected)) global_plot.add_category(category, averages, colors[category], errors, "green") plotter.plot() global_plot.plot()
base_images_path = r'/home/cola/work/nenet/nima/images-data/tid2013/distorted_images/' tid_dataset_path = r'/home/cola/work/nenet/nima/images-data/tid2013/TID2013/tid_labels_train.json' IMAGE_SIZE = 128 BASE_LEN = len(base_images_path) - 1 files = glob.glob(base_images_path + "*.jpg") files = sorted(files) train_image_paths = [] train_scores = [] print("Loading training set and val set") with open(tid_dataset_path, mode='r') as f: f = open(tid_dataset_path, 'r') data = json.load(f) for item in data: image_id = item['image_id'] label = item['label'] file_path = base_images_path + image_id + '.bmp' file_path_BMP = base_images_path + image_id + '.BMP' if os.path.exists(file_path): train_image_paths.append(file_path) train_scores.append(label) elif os.path.exists(file_path_BMP): train_image_paths.append(file_path_BMP) train_scores.append(label) train_image_paths = np.array(train_image_paths) train_scores = np.array(train_scores, dtype='float32')
def test_loadFile(self): f = io.StringIO("[1,2,3,4]") self.assertEqual([1, 2, 3, 4], ujson.load(f)) f = io.StringIO("[1,2,3,4]") assert_array_equal(np.array([1, 2, 3, 4]), ujson.load(f, numpy=True))