def create_pck(self):
data = self.transform(self.response["data"], self.survey)
id_dict = self.ids._asdict()
pck_name = CSFormatter.pck_name(id_dict["survey_id"], id_dict["tx_id"])
pck = CSFormatter.get_pck(data, id_dict["inst_id"], id_dict["ru_ref"],
id_dict["ru_check"], id_dict["period"])
return pck_name, pck
def create_zip(self, img_seq=None):
"""Perform transformation on the survey data
and pack the output into a zip file exposed by the image transformer
"""
logger.info("Creating PCK", ru_ref=self.ids["ru_ref"])
pck_name = CSFormatter.pck_name(self.ids["survey_id"], self.ids["seq_nr"])
transformed_data = self.transform()
pck = CSFormatter.get_pck(
transformed_data,
self.idbr_ref[self.ids["instrument_id"]],
self.ids["ru_ref"],
self.ids["ru_check"],
self.ids["period"],
)
logger.info("Creating IDBR receipt", ru_ref=self.ids["ru_ref"])
idbr_name = CSFormatter.idbr_name(self.ids["submitted_at"], self.ids["seq_nr"])
idbr = CSFormatter.get_idbr(
self.ids["survey_id"],
self.ids["ru_ref"],
self.ids["ru_check"],
self.ids["period"],
)
response_json_name = CSFormatter.response_json_name(self.ids["survey_id"], self.ids["seq_nr"])
self.image_transformer.zip.append(os.path.join(SDX_FTP_DATA_PATH, pck_name), pck)
self.image_transformer.zip.append(os.path.join(SDX_FTP_RECEIPT_PATH, idbr_name), idbr)
self.image_transformer.get_zipped_images(img_seq)
self.image_transformer.zip.append(os.path.join(SDX_RESPONSE_JSON_PATH, response_json_name), json.dumps(self.response))
def create_zip(self, img_seq=None):
"""Perform transformation on the survey data
and pack the output into a zip file exposed by the image transformer
"""
data = self.transform(self.response["data"], self.survey)
id_dict = self.ids._asdict()
pck_name = CSFormatter.pck_name(id_dict["survey_id"], id_dict["seq_nr"])
pck = CSFormatter.get_pck(data, id_dict["inst_id"], id_dict["ru_ref"], id_dict["ru_check"], id_dict["period"])
idbr_name = CSFormatter.idbr_name(id_dict["user_ts"], id_dict["seq_nr"])
idbr = CSFormatter.get_idbr(id_dict["survey_id"], id_dict["ru_ref"], id_dict["ru_check"], id_dict["period"])
response_json_name = CSFormatter.response_json_name(id_dict["survey_id"], id_dict["seq_nr"])
self.image_transformer.zip.append(os.path.join(SDX_FTP_DATA_PATH, pck_name), pck)
self.image_transformer.zip.append(os.path.join(SDX_FTP_RECEIPT_PATH, idbr_name), idbr)
self.image_transformer.get_zipped_images(img_seq)
self.image_transformer.zip.append(os.path.join(SDX_RESPONSE_JSON_PATH, response_json_name),
json.dumps(self.response))
def create_pck(self, img_seq=None):
logger.info("Creating PCK", ru_ref=self.ids.ru_ref)
pck_name = CSFormatter.pck_name(self.ids.survey_id, self.ids.tx_id)
transformed_data = self._transform()
pck = CSFormatter.get_pck(
transformed_data,
self.idbr_ref[self.ids.inst_id],
self.ids.ru_ref,
self.ids.ru_check,
self.ids.period,
)
return pck_name, pck
def test_digits_to_onetwo(self):
"""
Test question value in question range return as boolean
"""
digits_ingested_as_bools = [100, 120, 200, 220]
for question_range in digits_ingested_as_bools:
question_id = str(question_range)
with self.subTest(question_range=question_range, question_id=question_id):
return_value = MWSSTransformer.transform({question_id: "64"})
self.assertIs(True, return_value[question_id])
self.assertEqual(1, CSFormatter._pck_value(return_value[question_id]))
return_value = MWSSTransformer.transform({question_id: ""})
self.assertEqual(2, CSFormatter._pck_value(return_value[question_id]))
def test_digits_to_onetwo(self):
"""
Test question value in question range return as boolean
"""
digits_ingested_as_bools = [100, 120, 200, 220]
for question_range in digits_ingested_as_bools:
question_id = str(question_range)
with self.subTest(question_range=question_range, question_id=question_id):
return_value = MWSSTransformer.transform({question_id: "64"})
self.assertIs(True, return_value[question_id])
self.assertEqual(1, CSFormatter._pck_value(return_value[question_id]))
return_value = MWSSTransformer.transform({question_id: ""})
self.assertEqual(2, CSFormatter._pck_value(return_value[question_id]))
def test_dates_to_onetwo(self):
"""
Test dates in question range are returned as boolean
"""
dates_ingested_as_bools = [110, 210]
for question_range in dates_ingested_as_bools:
question_id = str(question_range)
with self.subTest(question_range=question_range, question_id=question_id):
return_value = MWSSTransformer.transform({question_id: "23/4/2017"})
self.assertEqual([datetime.date(2017, 4, 23)], return_value[question_id])
self.assertEqual(1, CSFormatter._pck_value(return_value[question_id]))
return_value = MWSSTransformer.transform({question_id: ""})
self.assertEqual([], return_value[question_id])
self.assertEqual(2, CSFormatter._pck_value(return_value[question_id]))
def test_dates_to_onetwo(self):
"""
Test dates in question range are returned as boolean
"""
dates_ingested_as_bools = [110, 210]
for question_range in dates_ingested_as_bools:
question_id = str(question_range)
with self.subTest(question_range=question_range, question_id=question_id):
return_value = MWSSTransformer.transform({question_id: "23/4/2017"})
self.assertEqual([datetime.date(2017, 4, 23)], return_value[question_id])
self.assertEqual(1, CSFormatter._pck_value(return_value[question_id]))
return_value = MWSSTransformer.transform({question_id: ""})
self.assertEqual([], return_value[question_id])
self.assertEqual(2, CSFormatter._pck_value(return_value[question_id]))
def test_pck_from_transformed_data(self):
"""
Test package from transformer returned data correctly
"""
f = open(self.replies_dir + "eq-mwss.json", "r")
reply = json.loads(f.read())
f.close()
reply["tx_id"] = "27923934-62de-475c-bc01-433c09fd38b8"
reply["survey_id"] = "134"
reply["collection"]["period"] = "200911"
reply["metadata"]["ru_ref"] = "49900001225C"
ids = Survey.identifiers(reply, batch_nr=3866, seq_nr=0)
data = MWSSTransformer.transform(
OrderedDict([
("40", 2),
("140", 124),
("151", 217222)
])
)
id_dict = ids._asdict()
return_value = CSFormatter._pck_lines(data, id_dict["inst_id"], id_dict["ru_ref"], id_dict["ru_check"],
id_dict["period"])
self.assertEqual([
"FV ",
"0005:49900001225C:200911",
"0040 00000000002",
"0130 00000000002",
"0131 00000000002",
"0132 00000000002",
"0140 00000000124",
"0151 00000217222",
], return_value)
def test_idbr_receipt(self):
self.reply["tx_id"] = "27923934-62de-475c-bc01-433c09fd38b8"
ids = Survey.identifiers(self.reply, batch_nr=3866)
id_dict = ids._asdict()
rv = CSFormatter.get_idbr(id_dict["survey_id"], id_dict["ru_ref"],
id_dict["ru_check"], id_dict["period"])
self.assertEqual("12346789012:A:134:201605", rv)
def test_idbr_receipt(self):
self.reply["tx_id"] = "27923934-62de-475c-bc01-433c09fd38b8"
ids = Survey.identifiers(self.reply, batch_nr=3866)
id_dict = ids._asdict()
rv = CSFormatter._idbr_receipt(id_dict["survey_id"], id_dict["ru_ref"], id_dict["ru_check"],
id_dict["period"])
self.assertEqual("12346789012:A:134:201605", rv)
def test_pck_form_header(self):
form_id = 5
ru_ref = 49900001225
check = "C"
period = "200911"
rv = CSFormatter._pck_form_header(form_id, ru_ref, check, period)
self.assertEqual("0005:49900001225C:200911", rv)
def test_pck_form_header(self):
form_id = 5
ru_ref = 49900001225
check = "C"
period = "200911"
rv = CSFormatter._pck_form_header(form_id, ru_ref, check, period)
self.assertEqual("0005:49900001225C:200911", rv)
def test_pck_from_transformed_data(self):
"""
Test package from transformer returned data correctly
"""
src = pkg_resources.resource_string(__name__, "replies/eq-mwss.json")
reply = json.loads(src.decode("utf-8"))
reply["tx_id"] = "27923934-62de-475c-bc01-433c09fd38b8"
reply["survey_id"] = "134"
reply["collection"]["period"] = "200911"
reply["metadata"]["ru_ref"] = "49900001225C"
ids = Survey.identifiers(reply, batch_nr=3866, seq_nr=0)
data = MWSSTransformer.transform(
OrderedDict([
("40", 2),
("140", 124),
("151", 217222)
])
)
id_dict = ids._asdict()
return_value = CSFormatter._pck_lines(data, id_dict["inst_id"], id_dict["ru_ref"], id_dict["ru_check"],
id_dict["period"])
self.assertEqual([
"FV ",
"0005:49900001225C:200911",
"0040 00000000002",
"0130 00000000002",
"0131 00000000002",
"0132 00000000002",
"0140 00000000124",
"0151 00000217222",
], return_value)
def test_currency(self):
"""
Test currency value
"""
return_value = MWSSTransformer.transform({"50": "36852"})
self.assertEqual(36852, return_value["50"])
item = CSFormatter._pck_item("50", return_value["50"])
self.assertEqual(item, "0050 00000036852")
def test_unsigned(self):
"""
Test unsigned integer value
"""
return_value = MWSSTransformer.transform({"40": "33"})
self.assertEqual(33, return_value["40"])
item = CSFormatter._pck_item("40", return_value["40"])
self.assertEqual(item, "0040 00000000033")
def test_currency(self):
"""
Test currency value
"""
return_value = MWSSTransformer.transform({"50": "36852"})
self.assertEqual(36852, return_value["50"])
item = CSFormatter._pck_item("50", return_value["50"])
self.assertEqual(item, "0050 00000036852")
def test_unsigned(self):
"""
Test unsigned integer value
"""
return_value = MWSSTransformer.transform({"40": "33"})
self.assertEqual(33, return_value["40"])
item = CSFormatter._pck_item("40", return_value["40"])
self.assertEqual(item, "0040 00000000033")
def test_pck_form_header(self):
"""
Test package form header
"""
form_id = "MB65B"
ru_ref = 49900108249
check = "D"
period = "1704"
return_value = CSFormatter._pck_form_header(form_id, ru_ref, check, period)
self.assertEqual("MB65B:49900108249D:1704", return_value)
def test_pck_form_header(self):
"""
Test package form header
"""
form_id = "MB65B"
ru_ref = 49900108249
check = "D"
period = "1704"
return_value = CSFormatter._pck_form_header(form_id, ru_ref, check,
period)
self.assertEqual("MB65B:49900108249D:1704", return_value)
def test_pck_form_header(self):
"""
Test package form header
"""
form_id = 5
ru_ref = 49900001225
check = "C"
period = "200911"
return_value = CSFormatter._pck_form_header(form_id, ru_ref, check, period)
self.assertEqual("0005:49900001225C:200911", return_value)
def test_pck_form_header(self):
"""
Test package form header
"""
form_id = 5
ru_ref = 49900001225
check = "C"
period = "200911"
return_value = CSFormatter._pck_form_header(form_id, ru_ref, check, period)
self.assertEqual("0005:49900001225C:200911", return_value)
def test_unsigned_decimals(self):
"""
Test unsigned decimal value
"""
digits_ingested_as_bools = [100, 200]
for question_range in digits_ingested_as_bools:
question_id = str(question_range)
with self.subTest(question_range=question_range, question_id=question_id):
return_value = MWSSTransformer.transform({question_id: "64.0"})
self.assertIs(True, return_value[question_id])
self.assertEqual(1, CSFormatter._pck_value(return_value[question_id]))
def test_unsigned_decimals(self):
"""
Test unsigned decimal value
"""
digits_ingested_as_bools = [100, 200]
for question_range in digits_ingested_as_bools:
question_id = str(question_range)
with self.subTest(question_range=question_range, question_id=question_id):
return_value = MWSSTransformer.transform({question_id: "64.0"})
self.assertIs(True, return_value[question_id])
self.assertEqual(1, CSFormatter._pck_value(return_value[question_id]))
def test_pay_frequency_as_bool(self):
"""
Test pay frequency value in question range returns as boolean
"""
pay_frequencies = {
130: "Calendar monthly",
131: "Four weekly",
132: "Five weekly",
}
for question_id, return_value in pay_frequencies.items():
question_id = str(question_id)
with self.subTest(question_id=question_id, return_value=return_value):
return_value = MWSSTransformer.transform({question_id: return_value})
self.assertIs(True, return_value[question_id])
self.assertEqual(1, CSFormatter._pck_value(return_value[question_id]))
return_value = MWSSTransformer.transform({question_id: ""})
self.assertIs(False, return_value[question_id])
self.assertEqual(2, CSFormatter._pck_value(return_value[question_id]))
return_value = MWSSTransformer.transform({})
self.assertIs(False, return_value[question_id])
self.assertEqual(2, CSFormatter._pck_value(return_value[question_id]))
def test_pay_frequency_as_bool(self):
"""
Test pay frequency value in question range returns as boolean
"""
pay_frequencies = {
130: "Calendar monthly",
131: "Four weekly",
132: "Five weekly",
}
for question_id, return_value in pay_frequencies.items():
question_id = str(question_id)
with self.subTest(question_id=question_id, return_value=return_value):
return_value = MWSSTransformer.transform({question_id: return_value})
self.assertIs(True, return_value[question_id])
self.assertEqual(1, CSFormatter._pck_value(return_value[question_id]))
return_value = MWSSTransformer.transform({question_id: ""})
self.assertIs(False, return_value[question_id])
self.assertEqual(2, CSFormatter._pck_value(return_value[question_id]))
return_value = MWSSTransformer.transform({})
self.assertIs(False, return_value[question_id])
self.assertEqual(2, CSFormatter._pck_value(return_value[question_id]))
def test_idbr_receipt(self):
"""
Tests inter-departmental business register receipt
"""
src = pkg_resources.resource_string(__name__, "replies/eq-mwss.json")
reply = json.loads(src.decode("utf-8"))
reply["tx_id"] = "27923934-62de-475c-bc01-433c09fd38b8"
ids = Survey.identifiers(reply, batch_nr=3866, seq_nr=0)
id_dict = ids._asdict()
return_value = CSFormatter._idbr_receipt(id_dict["survey_id"], id_dict["ru_ref"], id_dict["ru_check"],
id_dict["period"])
self.assertEqual("12346789012:A:134:201605", return_value)
def test_idbr_receipt(self):
"""
Tests inter-departmental business register receipt
"""
f = open(self.replies_dir + "eq-mwss.json", "r")
reply = json.loads(f.read())
f.close()
reply["tx_id"] = "27923934-62de-475c-bc01-433c09fd38b8"
ids = Survey.identifiers(reply, batch_nr=3866, seq_nr=0)
id_dict = ids._asdict()
return_value = CSFormatter._idbr_receipt(id_dict["survey_id"], id_dict["ru_ref"], id_dict["ru_check"],
id_dict["period"])
self.assertEqual("12346789012:A:134:201605", return_value)
def test_pck_lines(self):
"""Tests conversions of various types of values."""
inst_id = "0005"
ru_ref = 49900001225
check = "C"
period = "200911"
data = OrderedDict([("0001", 2), ("0140", 124),
("0146", "This is a comment"), ("0151", 217222),
("0200", {})])
self.assertTrue(isinstance(val, int) for val in data.values())
rv = CSFormatter._pck_lines(data, inst_id, ru_ref, check, period)
self.assertEqual([
"FV ", "0005:49900001225C:200911", "0001 00000000002",
"0140 00000000124", "0146 00000000001", "0151 00000217222",
"0200 ???????????"
], rv)
def test_pck_from_transformed_data(self):
self.reply["tx_id"] = "27923934-62de-475c-bc01-433c09fd38b8"
self.reply["survey_id"] = "134"
self.reply["collection"]["period"] = "200911"
self.reply["metadata"]["ru_ref"] = "49900001225C"
self.reply["data"] = OrderedDict([("0001", 2), ("0140", 124),
("0151", 217222)])
ids = Survey.identifiers(self.reply, batch_nr=3866)
id_dict = ids._asdict()
rv = CSFormatter._pck_lines(self.reply["data"], id_dict["inst_id"],
id_dict["ru_ref"], id_dict["ru_check"],
id_dict["period"])
self.assertEqual([
"FV ",
"0005:49900001225C:200911",
"0001 00000000002",
"0140 00000000124",
"0151 00000217222",
], rv)
def test_pck_from_transformed_data(self):
self.reply["tx_id"] = "27923934-62de-475c-bc01-433c09fd38b8"
self.reply["survey_id"] = "134"
self.reply["collection"]["period"] = "200911"
self.reply["metadata"]["ru_ref"] = "49900001225C"
self.reply["data"] = OrderedDict([
("0001", 2),
("0140", 124),
("0151", 217222)
])
ids = Survey.identifiers(self.reply, batch_nr=3866)
id_dict = ids._asdict()
rv = CSFormatter._pck_lines(self.reply["data"], id_dict["inst_id"], id_dict["ru_ref"], id_dict["ru_check"],
id_dict["period"])
self.assertEqual([
"FV ",
"0005:49900001225C:200911",
"0001 00000000002",
"0140 00000000124",
"0151 00000217222",
], rv)
def test_pck_lines(self):
"""
Tests data in packet is valid
"""
inst_id = "0005"
ru_ref = 49900001225
check = "C"
period = "200911"
data = OrderedDict([
("0001", 2),
("0140", 124),
("0151", 217222)
])
self.assertTrue(isinstance(return_value, int) for return_value in data.values())
return_value = CSFormatter._pck_lines(data, inst_id, ru_ref, check, period)
self.assertEqual([
"FV ",
"0005:49900001225C:200911",
"0001 00000000002",
"0140 00000000124",
"0151 00000217222",
], return_value)
def test_pck_lines(self):
"""
Tests data in packet is valid
"""
inst_id = "0005"
ru_ref = 49900001225
check = "C"
period = "200911"
data = OrderedDict([
("0001", 2),
("0140", 124),
("0151", 217222)
])
self.assertTrue(isinstance(return_value, int) for return_value in data.values())
return_value = CSFormatter._pck_lines(data, inst_id, ru_ref, check, period)
self.assertEqual([
"FV ",
"0005:49900001225C:200911",
"0001 00000000002",
"0140 00000000124",
"0151 00000217222",
], return_value)
def test_pck_lines(self):
"""Tests conversions of various types of values."""
inst_id = "0005"
ru_ref = 49900001225
check = "C"
period = "200911"
data = OrderedDict([
("0001", 2),
("0140", 124),
("0146", "This is a comment"),
("0151", 217222),
("0200", {})
])
self.assertTrue(isinstance(val, int) for val in data.values())
rv = CSFormatter._pck_lines(data, inst_id, ru_ref, check, period)
self.assertEqual([
"FV ",
"0005:49900001225C:200911",
"0001 00000000002",
"0140 00000000124",
"0146 00000000001",
"0151 00000217222",
"0200 ???????????"
], rv)