def test_loadFileArgsError(self):
try:
ujson.load("[]")
except TypeError:
pass
else:
assert False, "expected TypeError"
def test_loadFileArgsError(self):
try:
ujson.load("[]")
except TypeError:
pass
else:
assert False, "expected TypeError"
def test_loadFileLikeObject(self):
class filelike:
def read(self):
try:
self.end
except AttributeError:
self.end = True
return "[1,2,3,4]"
f = filelike()
self.assertEquals([1, 2, 3, 4], ujson.load(f))
f = filelike()
assert_array_equal(np.array([1, 2, 3, 4]), ujson.load(f, numpy=True))
def test_loadFileLikeObject(self):
class filelike:
def read(self):
try:
self.end
except AttributeError:
self.end = True
return "[1,2,3,4]"
f = filelike()
self.assertEqual([1, 2, 3, 4], ujson.load(f))
f = filelike()
assert_array_equal(np.array([1, 2, 3, 4]), ujson.load(f, numpy=True))
def main():
categories = {"essential constraints": "Essential"}
cat_counters = {n: [] for n in categories}
measures = [("accuracy", calc_accuracy), ("expected accuracy", calc_expected_accuracy),
("redundancy", calc_redundancy)]
speed = []
for name in files:
print(name)
csv_file = local("data/csv/{}.csv".format(name))
if not os.path.isfile(csv_file):
raise Exception("File not found: {}".format(csv_file))
groups_file = local("data/groups/{}.txt".format(name))
silent = False # TODO True
if not os.path.isfile(groups_file):
with open(groups_file, "w+") as f:
print("{\n\t\"Tables\":\n\t\t[\n\n\t\t]\n}", file=f, flush=True)
groups_file = None
silent = False
t_before = time.time()
solutions = workflow.main(csv_file, groups_file, False, silent=True, parse_silent=silent)
t_elapsed = time.time() - t_before
speed.append(t_elapsed)
print("\tTook: {:.2f} seconds".format(t_elapsed))
truth_file = local("data/truth/{}.txt".format(name))
# constraint_map = {c.name: c for c in workflow.constraint_list}
if os.path.isfile(truth_file):
with open(truth_file) as f:
json_data = json.load(f)
counters = {n: CategoryCounter(json_data[k]) for n, k in categories.items()}
for constraint in solutions.solutions:
for solution in solutions.get_solutions(constraint):
for counter in counters.values():
counter.count(constraint, solution)
for c_name, c in counters.items():
cat_counters[c_name].append(c)
for m_name, m_func in measures:
res, actual, expected = m_func([c], per_file=False)
print("\t{} for {}: {:.2%} ({} of {})"
.format(m_name.capitalize(), c_name, res, actual, expected))
print_constraints("MISSING", c.not_found)
redundant = CategoryCounter.filter_constraints(c.not_present, is_excel_constraint)
print_constraints("REDUNDANT", redundant)
print()
elif groups_file is not None:
with open(truth_file, "w+") as f:
print("{\n\t\"Essential\":\n\t\t{\n\n\t\t}\n}", file=f, flush=True)
print_truth.main(name)
for n in categories:
print("Time elapsed: {:.2f}s (total), {:.2f}s (average)".format(sum(speed), numpy.average(speed)))
for m_name, m_func in measures:
res, not_zero, file_count = m_func(cat_counters[n], per_file=True)
print("{} {} per file ({} of {} files): {:.2%}".format(n.capitalize(), m_name, not_zero, file_count, res))
res, actual, expected = m_func(cat_counters[n], per_file=False)
print("Total {} {}: {:.2%} ({} of {})".format(n, m_name, res, actual, expected))
print()
return cat_counters
def show_result():
print("before")
os.system("python ../dirbot/spiders/webcrawler.py 1")
print("after")
with open('result.json') as data_file:
data = json.load(data_file)
plec = []
wiek = []
wiekrozpoczecia = []
wiekdiagnozy = []
czastrwania = []
edss = []
kursms = []
nawrot12 = []
nawrot24 = []
for x in data:
v1 = x["typ"]
v2 = x["opis"]
v3 = x["ilosc"]
v1 = " ".join(str(v) for v in v1)
v2 = " ".join(str(v) for v in v2)
v3 = " ".join(str(v) for v in v3)
if v1 == "Distribution of gender":
plec.extend([[v2, int(v3)]])
if v1 == "Age":
wiek.extend([[v2, int(v3)]])
if v1 == "Age at onset":
wiekrozpoczecia.extend([[v2, int(v3)]])
if v1 == "Age at diagnosis":
wiekdiagnozy.extend([[v2, int(v3)]])
if v1 == "Duration of MS":
czastrwania.extend([[v2, int(v3)]])
if v1 == "EDSS at last visit":
edss.extend([[v2, int(v3)]])
if v1 == "Last MSCourse":
kursms.extend([[v2, int(v3)]])
if v1 == "Relapse count over last 12 months":
nawrot12.extend([[v2, int(v3)]])
if v1 == "Relapse count over last 24 months":
nawrot24.extend([[v2, int(v3)]])
my_dict = {
'plec': plec,
'wiek': wiek,
'wiekrozpoczecia': wiekrozpoczecia,
'wiekdiagnozy': wiekdiagnozy,
'czastrwania': czastrwania,
'edss': edss,
'kursms': kursms,
'nawrot12': nawrot12,
'nawrot24': nawrot24
}
return render_template('result.html', my_dict=my_dict)
def count_constraints(self, solutions):
truth_file = self._get_truth_file()
with open(truth_file) as file:
json_data = json.load(file)
self._counter = ConstraintCounter({
**json_data["Essential"],
**json_data["Non-trivial"]
})
for constraint in solutions.solutions:
for solution in solutions.get_solutions(constraint):
self._counter.register(constraint, solution)
def get_diff_cache(filename):
"""
Loads a cache result by filename
:param filename:
:return:
"""
file_name = _cache_directory + filename + '.json'
if os.path.isfile(file_name):
with open(file_name) as data_file:
data = ujson.load(data_file)
return data
# if the file doesn't exist
return None
def show_result():
print ("before")
os.system("python ../dirbot/spiders/crawlmycrownspider.py 1")
print ("after")
# call(['python ../dirbot/spiders/crawlmycrownspider.py shell=True'])
with open('result.json') as data_file:
data = json.load(data_file)
gender = []
age = []
ageonset = []
agediagnosis = []
durationms = []
edss = []
lastms = []
relapse12 = []
relapse24 = []
for x in data:
v1 = x["type"]
v2 = x["description"]
v3 = x["count"]
v1 = " ".join(str(v) for v in v1)
v2 = " ".join(str(v) for v in v2)
v3 = " ".join(str(v) for v in v3)
if v1 == "Distribution of gender":
gender.extend([[v2, int(v3)]])
if v1 == "Age":
age.extend([[v2,int(v3)]])
if v1 == "Age at onset":
ageonset.extend([[v2,int(v3)]])
if v1 == "Age at diagnosis":
agediagnosis.extend([[v2, int(v3)]])
if v1 == "Duration of MS":
durationms.extend([[v2, int(v3)]])
if v1 == "EDSS at last visit":
edss.extend([[v2, int(v3)]])
if v1 == "Last MSCourse":
lastms.extend([[v2, int(v3)]])
if v1 == "Relapse count over last 12 months":
relapse12.extend([[v2, int(v3)]])
if v1 == "Relapse count over last 24 months":
relapse24.extend([[v2, int(v3)]])
my_dict={'gender': gender, 'age': age, 'ageonset':ageonset, 'agediagnosis': agediagnosis,'durationms':durationms, 'edss': edss,'lastms': lastms,'relapse12': relapse12,'relapse24':relapse24}
return render_template('result.html', my_dict=my_dict)
def measure_size(self, config_file):
with open(config_file) as file:
tables = json.load(file)["Tables"]
cell_count = 0
vector_count = 0
for table in tables:
bounds = Bounds(table["Bounds"])
cell_count += bounds.columns() * bounds.rows()
if "Orientation" not in table or table["Orientation"].lower(
) == "none":
vector_count += bounds.columns() + bounds.rows()
elif table["Orientation"].lower()[0:3] == "row":
vector_count += bounds.rows()
elif table["Orientation"].lower()[0:3] == "col":
vector_count += bounds.columns()
else:
raise Exception("Unexpected orientation: {}".format(
table["Orientation"]))
self._cells = cell_count
self._vectors = vector_count
self._tables = len(tables)
def load_tid_data():
global g_train_image_paths
global g_train_scores
global g_val_image_paths
global g_val_scores
print("Loading tid training set and val set")
print 'tip_base_images_path = ' + tip_base_images_path
files = glob.glob(tip_base_images_path + "*.bmp")
files = sorted(files)
f = open(tip_dataset_path, 'r')
data = json.load(f)
image_size = 0
for item in data:
image_id = item['image_id']
label = item['label']
file_path = tip_base_images_path + image_id + '.bmp'
file_path_BMP = tip_base_images_path + image_id + '.BMP'
if os.path.exists(file_path):
g_train_image_paths.append(file_path)
g_train_scores.append(label)
image_size = image_size + 1
elif os.path.exists(file_path_BMP):
g_train_image_paths.append(file_path_BMP)
g_train_scores.append(label)
image_size = image_size + 1
g_train_image_paths = np.array(g_train_image_paths)
g_train_scores = np.array(g_train_scores, dtype='float32')
g_val_image_paths = g_train_image_paths[-5000:]
g_val_scores = g_train_scores[-5000:]
train_image_paths = g_train_image_paths[:-5000]
train_scores = g_train_scores[:-5000]
print('Train set size : ', train_image_paths.shape, train_scores.shape)
print('Val set size : ', g_val_image_paths.shape, g_val_scores.shape)
print('Train and validation datasets ready !')
def test_loadFile(self):
f = StringIO("[1,2,3,4]")
self.assertEquals([1, 2, 3, 4], ujson.load(f))
f = StringIO("[1,2,3,4]")
assert_array_equal(np.array([1, 2, 3, 4]), ujson.load(f, numpy=True))
def main():
info = [
("Tables", lambda e: e.tables, "grey", lambda x: str(x)),
# ("Vectors", lambda e: e.vectors, "cyan", lambda x: str(x)),
("Cells", lambda e: e.cells, "black", lambda x: str(x)),
("Intended Constraints",
lambda e: e.counter.count(relevant=True, supported=True), "magenta",
lambda x: str(x)),
("Recall", lambda e: recall(e.counter), "green",
lambda x: "{:.2f}".format(x)),
("Recall Supported", lambda e: recall(e.counter, True), "lightgreen",
lambda x: "{:.2f}".format(x)),
("Precision", lambda e: precision(e.counter), "yellow",
lambda x: "{:.2f}".format(x)),
("Runtime", lambda e: numpy.average(e.running_times(1)), "red",
lambda x: "{:.2f}s".format(x))
]
with open(local("data/data.txt")) as data_file:
data_json = json.load(data_file)
categories = ["Exercises", "Tutorials", "Data"]
stats = {}
for category in categories:
files = data_json[category]
stats[category] = []
for file in files:
experiment = Experiment(file)
print(file)
for accident in experiment.counter.additional:
print("\tADDITIONAL:\t{}".format(accident))
for missing in experiment.counter.missed:
print("\tMISSING:\t{}".format(missing))
entry = {t[0]: t[1](experiment) for t in info}
entry["relevant.found"] = experiment.counter.count(
relevant=True, found=True)
entry["extra.found"] = experiment.counter.count(relevant=False,
found=True)
stats[category].append(entry)
maximal = {
t[0]: max(
max(stats[t[0]] for stats in cat_stats)
for cat_stats in stats.values())
for t in info
}
colors = dict(zip(categories, ["red", "yellow", "black", "green"]))
print("\n" * 2)
max_label = list(t[3](maximal[t[0]]) for t in info)
global_plot = Plotter("comparison",
list(t[0] for t in info),
upper=max_label)
test = {"Exercises": (30, 2), "Tutorials": (46, 21), "Data": (6, 0)}
for category, cat_stats in [(cat, stats[cat]) for cat in categories]:
print(category)
plotter = Plotter("benchmark_" + category)
averages = []
errors = []
for name, f, color, pretty in info:
plotter.add_category(
name,
list(stats[name] / maximal[name] for stats in cat_stats),
color)
result_string = "\tAverage {name}: {average} - total: {total}"
average = numpy.average(
list(stats[name] for stats in cat_stats))
averages.append(average / maximal[name])
errors.append(
numpy.std(list(stats[name]
for stats in cat_stats)) / maximal[name])
total = sum(stats[name] for stats in cat_stats)
print(
result_string.format(name=name.lower(),
average=pretty(average),
total=pretty(total)))
relevant = sum(stats["relevant.found"] for stats in cat_stats)
print("\tRelevant found: {}".format(relevant))
extra = sum(stats["extra.found"] for stats in cat_stats)
print("\tExtra found: {}".format(extra))
relevant_expected, extra_expected = test[category]
if relevant_expected != relevant:
print(
"ERROR ({}): found {} instead of {} relevant constraints".
format(category, relevant, relevant_expected))
if extra_expected != extra:
print("ERROR ({}): found {} instead of {} extra constraints".
format(category, extra, extra_expected))
global_plot.add_category(category, averages, colors[category],
errors, "green")
plotter.plot()
global_plot.plot()
base_images_path = r'/home/cola/work/nenet/nima/images-data/tid2013/distorted_images/'
tid_dataset_path = r'/home/cola/work/nenet/nima/images-data/tid2013/TID2013/tid_labels_train.json'
IMAGE_SIZE = 128
BASE_LEN = len(base_images_path) - 1
files = glob.glob(base_images_path + "*.jpg")
files = sorted(files)
train_image_paths = []
train_scores = []
print("Loading training set and val set")
with open(tid_dataset_path, mode='r') as f:
f = open(tid_dataset_path, 'r')
data = json.load(f)
for item in data:
image_id = item['image_id']
label = item['label']
file_path = base_images_path + image_id + '.bmp'
file_path_BMP = base_images_path + image_id + '.BMP'
if os.path.exists(file_path):
train_image_paths.append(file_path)
train_scores.append(label)
elif os.path.exists(file_path_BMP):
train_image_paths.append(file_path_BMP)
train_scores.append(label)
train_image_paths = np.array(train_image_paths)
train_scores = np.array(train_scores, dtype='float32')
def test_loadFile(self):
f = io.StringIO("[1,2,3,4]")
self.assertEqual([1, 2, 3, 4], ujson.load(f))
f = io.StringIO("[1,2,3,4]")
assert_array_equal(np.array([1, 2, 3, 4]), ujson.load(f, numpy=True))
