def make_sol_dict():
"""
Run present pipeline and save the merge results
:return:
"""
file_names = ["FORMAT3_Copy of KommuneMTPLforTriangle.xls",
"C Triangulations analysis R2017 GC20161109.xls",
"EVOLUTION 2017 _ M+F - Triangles cat nat brut net.xls",
"Bsp8 _ Dreiecke aus GCNA für CU1.4.1.xls",
"Analysis MTPL MOD.xls",
"Bsp6 _ Dreiecke aus GCNA für CU1.4.1.xls",
"FORMAT6_sinistres.xls",
"FORMAT1_LOSSES-MTPL-OVER-500-GROUP-2005_modified.xls"]
solutions_dict = dict()
raw_dict = dict()
for file_name in file_names:
sr_list, file_name = ExcelLoader.load_excel(pdir.RESOURCES_DIR + "/raw_test_files/" + file_name)
dh = DataHolder()
for sr in sr_list:
dh.add_sheet(sr.sheet_name, pd.DataFrame(columns=sr.headers, data=sr.row_vals),
pd.DataFrame(columns=sr.headers, data=sr.xls_types), orig_sheet_name=sr.sheet_name)
dh = SheetPreProcessor.separate_components(dh)
raw_dict[file_name] = dh.encode()
dh = HorizontalMerger.horizontal_merge(dh)
#temp_path = pdir.RESOURCES_DIR + "/temp/"
#dh.write_excel(temp_path + file_name)
solutions_dict[file_name] = dh
solutions_dict = MergePararametersOptimizer.make_ind_col_dict(solutions_dict)
with open(pdir.RESOURCES_DIR + "/test/merge_solutions.obj", "wb") as temp_file:
pickle.dump(solutions_dict, temp_file)
with open(pdir.RESOURCES_DIR + "/test/raw_test.obj", "wb") as temp_file:
pickle.dump(raw_dict, temp_file)
def post(self, request):
sr_list = jsonpickle.decode(request.data['sr_list'])
dhName = request.data['dhName']
selected_sheets = request.data['selected_sheets']
data_holder = DataHolder(dhName)
for sr in sr_list:
if sr.sheet_name in selected_sheets:
data_holder.add_sheet(sr.sheet_name,
pd.DataFrame(columns=sr.headers,
data=sr.row_vals),
pd.DataFrame(columns=sr.headers,
data=sr.xls_types),
orig_sheet_name=sr.sheet_name)
encoded = data_holder.encode()
return Response(encoded, status=200)
class DataHolderTest(TestCase):
def setUp(self):
self.names = ["first", "second"]
self.dh = DataHolder("test")
d1 = pd.DataFrame(data={'col1': ["1", "2"], 'col2': ["3", "4"]})
d2 = pd.DataFrame(data={'col1': [1, 2], 'col2': [3, 4]})
self.dh.add_sheet(self.names[0], d1, d2, orig_sheet_name="1")
d1 = pd.DataFrame(data={'col1': ["1", "2"], 'col2': ["1", "1"]})
d2 = pd.DataFrame(data={'col1': [1, 2], 'col2': [1, 1]})
self.dh.add_sheet(self.names[0], d1, d2, orig_sheet_name="2")
d1 = pd.DataFrame(data={'col1': ["1", "2"], 'col2': ["15", "16"]})
d2 = pd.DataFrame(data={'col1': [1, 2], 'col2': [15, 16]})
self.dh.add_sheet(self.names[1], d1, d2, orig_sheet_name="2")
def test_set_card_ids(self):
trngs = []
trngs.append({"group_id": "one"})
trngs.append({"group_id": "two"})
RowParser.set_card_ids(trngs, self.dh)
# print(self.dh)
true_ids = [0, 1, 1]
for ds, ind in zip(self.dh.data_struct_list, true_ids):
self.assertEqual(ds.card_id, ind)
def test_updating(self):
for dh_ind, df_data, df_profiles in self.dh.enumerate():
if dh_ind == 0:
df_data.loc[0, "col2"] = "5"
df_profiles.loc[0, "col2"] = 5
if dh_ind == 1:
df_data.loc[0, "col1"] = "5"
df_profiles.loc[0, "col1"] = 5
if dh_ind == 2:
df_data.loc[1, "col1"] = "19"
df_profiles.loc[1, "col1"] = 19
self.dh.update_with_ind(dh_ind, df_data, df_profiles)
assert_frame_equal(self.dh.data_dict[self.names[0]][0].df_data, self.dh.data_struct_list[0].df_data)
assert_frame_equal(self.dh.data_dict[self.names[0]][1].df_data, self.dh.data_struct_list[1].df_data)
assert_frame_equal(self.dh.data_dict[self.names[1]][0].df_data, self.dh.data_struct_list[2].df_data)
assert_frame_equal(self.dh.data_dict[self.names[0]][0].df_profiles, self.dh.data_struct_list[0].df_profiles)
assert_frame_equal(self.dh.data_dict[self.names[0]][1].df_profiles, self.dh.data_struct_list[1].df_profiles)
assert_frame_equal(self.dh.data_dict[self.names[1]][0].df_profiles, self.dh.data_struct_list[2].df_profiles)
for key in self.dh.data_dict:
for d_struct in self.dh.data_dict[key]:
self.assertEqual(key, d_struct.name)
for ds in self.dh.data_struct_list:
self.assertEqual(ds, self.dh.id_dict[ds.id])
def test_mementos(self):
self.dh.create_memento()
for dh_ind, df_data, df_profiles in self.dh.enumerate():
if dh_ind == 0:
df_data.loc[0, "col2"] = "5"
df_profiles.loc[0, "col2"] = 5
if dh_ind == 1:
df_data.loc[0, "col1"] = "5"
df_profiles.loc[0, "col1"] = 5
if dh_ind == 2:
df_data.loc[1, "col1"] = "19"
df_profiles.loc[1, "col1"] = 19
self.dh.update_with_ind(dh_ind, df_data, df_profiles)
self.dh.create_memento()
diff_dict_list = SheetStateComparer.compare_states(self.dh.mementos[0], self.dh.mementos[1])
#for diff_dict in diff_dict_list:
#pass#diff_
for i in range(2):
for j in range(2):
el = diff_dict_list[0]["diff_array"][i][j]
if i == 0 and j == 1:
self.assertEqual(el.change, "Corrected")
else:
self.assertEqual(el.change, "No change")
def test_serialization(self):
self.dh.data_struct_list[0].roles.append("Claims Paid")
self.dh.data_struct_list[0].df_data.sort_values("col1", ascending=False, inplace=True)
serialized = self.dh.encode()
data_framed = DataHolder.decode(serialized)
assert_frame_equal(self.dh.data_struct_list[0].df_data, data_framed.data_struct_list[0].df_data)
assert_frame_equal(self.dh.data_struct_list[1].df_data, data_framed.data_struct_list[1].df_data)
assert_frame_equal(self.dh.data_struct_list[2].df_data, data_framed.data_struct_list[2].df_data)
assert_frame_equal(self.dh.data_struct_list[0].df_profiles, data_framed.data_struct_list[0].df_profiles)
assert_frame_equal(self.dh.data_struct_list[1].df_profiles, data_framed.data_struct_list[1].df_profiles)
assert_frame_equal(self.dh.data_struct_list[2].df_profiles, data_framed.data_struct_list[2].df_profiles)
self.assertEqual(data_framed.data_struct_list[0].roles[0], "Claims Paid")
# Test conservation of ids
for ind in range(len(self.dh.data_struct_list)):
self.assertEqual(data_framed.data_struct_list[ind].id, self.dh.data_struct_list[ind].id)
def post(self, request):
#Recieve name of file
filename = request.data.get('fileName')
# Build data holder
sr_list = jsonpickle.decode(request.data['sr_list'])
selected_sheets = request.data['selected_sheets']
data_holder = DataHolder(filename)
for sr in sr_list:
if sr.sheet_name in selected_sheets:
data_holder.add_sheet(sr.sheet_name,
pd.DataFrame(columns=sr.headers,
data=sr.row_vals),
pd.DataFrame(columns=sr.headers,
data=sr.xls_types),
orig_sheet_name=sr.sheet_name)
response_data = {}
#dump this shiiiit
#data_holder.to_pickle_file(pdir.TEMP_DIR + "from_views.pickle")
if data_holder is None:
raise ValueError("No data holder found")
elif data_holder.n == 0:
raise ValueError("No sheets in data holder")
#Recieve triangle formats
triangles = request.data.get('triangles')
user_defined_triangles = triangles['templates']
outputFormats = triangles['output_formats']
n_outputs = triangles['number_of_outputs']
input_format = triangles['inputFormat']
tr_type = user_defined_triangles[0]['type']
# if tr_type == "single":
# n_outputs = len(user_defined_triangles)
# else:
# n_outputs = int(len(user_defined_triangles)/2)
try:
if input_format[0] == 'triangle':
#print(tr_type, n_outputs)
data_holder_dict, data_holder = TrianglePipeline.triangle_pipeline_dh(
data_holder, tri_type=tr_type, n_outputs=n_outputs)
else:
data_holder_dict, data_holder = TrianglePipeline.table_triangle_pipeline_dh(
data_holder)
#DataHolder manipulation
data_holder, group_ids, sheet_names = RowParser.set_card_ids(
user_defined_triangles, data_holder)
user_defined_triangles = InputMatcher.match_triangles_to_output(
user_defined_triangles, data_holder)
user_defined_triangles = RowParser.parse_output_from_triangle_forms(
user_defined_triangles, data_holder)
except DataHolderException as err:
data = {}
data['message'] = err.message
data['dh'] = err.dh
return Response({'response_error': data})
SheetWriter.trngs_to_existing_excel(
user_defined_triangles, pdir.TEMP_DIR + ps.OUTPUT_NAME + filename)
#Unsure if all neded
response_data["group_ids"] = group_ids
response_data['output_triangles'] = user_defined_triangles
#Building list for initial rendering
response_data[
"unit_triangles"] = ConnectDataAPIView.make_unit_triangle_list(
data_holder)
response_data["str_data_holder"] = data_holder.encode()
if len(data_holder_dict) > 1:
response_data["str_data_holder_dict"] = {
key: val.encode()
for key, val in data_holder_dict.items()
}
else:
response_data["str_data_holder_dict"] = {
data_holder.name: response_data["str_data_holder"]
}
return Response({'data': response_data})