def __init__(self, response, sequence_no):
self.response = response
self.sequence_no = sequence_no
self.logger = logger
self.ids = Survey.identifiers(response, seq_nr=sequence_no)
self.survey = Survey.load_survey(self.ids)
self.image_transformer = ImageTransformer(
self.logger,
self.survey,
self.response,
sequence_no=self.sequence_no,
base_image_path=SDX_FTP_IMAGE_PATH)
def test_load_survey_miss(self):
ids = Survey.identifiers({
"survey_id": "127",
"tx_id": "27923934-62de-475c-bc01-433c09fd38b8",
"collection": {
"instrument_id": "0001",
"period": "201704"
},
"metadata": {
"user_id": "123456789",
"ru_ref": "12345678901A"
}
})
with pytest.raises(MissingSurveyException):
Survey.load_survey(ids, "./tests/data/{survey_id}.{inst_id}.json")
def test_load_survey(self):
ids = Survey.identifiers({
"survey_id": "134",
"tx_id": "27923934-62de-475c-bc01-433c09fd38b8",
"collection": {
"instrument_id": "0005",
"period": "201704"
},
"metadata": {
"user_id": "123456789",
"ru_ref": "12345678901A"
}
})
rv = Survey.load_survey(ids, "./tests/data/{survey_id}.{inst_id}.json")
self.assertIsNotNone(rv)
def test_load_survey_miss(self):
ids = Survey.identifiers({
"survey_id": "127",
"tx_id": "27923934-62de-475c-bc01-433c09fd38b8",
"collection": {
"instrument_id": "0001",
"period": "201704"
},
"metadata": {
"user_id": "123456789",
"ru_ref": "12345678901A"
}
})
rv = Survey.load_survey(ids, "./tests/data/{survey_id}.{inst_id}.json")
self.assertIsNone(rv)
def test_datetime_ms_with_timezone(self):
"""
Tests datetime with no colon
"""
return_value = Survey.parse_timestamp("2017-01-11T17:18:53.020222+0000")
self.assertIsInstance(return_value, datetime.datetime)
def test_idbr_receipt(self):
self.reply["tx_id"] = "27923934-62de-475c-bc01-433c09fd38b8"
ids = Survey.identifiers(self.reply, batch_nr=3866)
id_dict = ids._asdict()
rv = CSFormatter.get_idbr(id_dict["survey_id"], id_dict["ru_ref"],
id_dict["ru_check"], id_dict["period"])
self.assertEqual("12346789012:A:134:201605", rv)
def test_pck_from_transformed_data(self):
"""
Test package from transformer returned data correctly
"""
f = open(self.replies_dir + "eq-mwss.json", "r")
reply = json.loads(f.read())
f.close()
reply["tx_id"] = "27923934-62de-475c-bc01-433c09fd38b8"
reply["survey_id"] = "134"
reply["collection"]["period"] = "200911"
reply["metadata"]["ru_ref"] = "49900001225C"
ids = Survey.identifiers(reply, batch_nr=3866, seq_nr=0)
data = MWSSTransformer.transform(
OrderedDict([
("40", 2),
("140", 124),
("151", 217222)
])
)
id_dict = ids._asdict()
return_value = CSFormatter._pck_lines(data, id_dict["inst_id"], id_dict["ru_ref"], id_dict["ru_check"],
id_dict["period"])
self.assertEqual([
"FV ",
"0005:49900001225C:200911",
"0040 00000000002",
"0130 00000000002",
"0131 00000000002",
"0132 00000000002",
"0140 00000000124",
"0151 00000217222",
], return_value)
def test_idbr_receipt(self):
self.reply["tx_id"] = "27923934-62de-475c-bc01-433c09fd38b8"
ids = Survey.identifiers(self.reply, batch_nr=3866)
id_dict = ids._asdict()
rv = CSFormatter._idbr_receipt(id_dict["survey_id"], id_dict["ru_ref"], id_dict["ru_check"],
id_dict["period"])
self.assertEqual("12346789012:A:134:201605", rv)
def test_datetime_zulu(self):
"""
Tests zulu datetime
"""
return_value = Survey.parse_timestamp("2017-01-11T17:18:53Z")
self.assertIsInstance(return_value, datetime.datetime)
def test_datetime_zulu(self):
"""
Tests zulu datetime
"""
return_value = Survey.parse_timestamp("2017-01-11T17:18:53Z")
self.assertIsInstance(return_value, datetime.datetime)
def test_datetime_ms_with_timezone(self):
"""
Tests datetime with no colon
"""
return_value = Survey.parse_timestamp("2017-01-11T17:18:53.020222+0000")
self.assertIsInstance(return_value, datetime.datetime)
def test_pck_from_transformed_data(self):
"""
Test package from transformer returned data correctly
"""
src = pkg_resources.resource_string(__name__, "replies/eq-mwss.json")
reply = json.loads(src.decode("utf-8"))
reply["tx_id"] = "27923934-62de-475c-bc01-433c09fd38b8"
reply["survey_id"] = "134"
reply["collection"]["period"] = "200911"
reply["metadata"]["ru_ref"] = "49900001225C"
ids = Survey.identifiers(reply, batch_nr=3866, seq_nr=0)
data = MWSSTransformer.transform(
OrderedDict([
("40", 2),
("140", 124),
("151", 217222)
])
)
id_dict = ids._asdict()
return_value = CSFormatter._pck_lines(data, id_dict["inst_id"], id_dict["ru_ref"], id_dict["ru_check"],
id_dict["period"])
self.assertEqual([
"FV ",
"0005:49900001225C:200911",
"0040 00000000002",
"0130 00000000002",
"0131 00000000002",
"0132 00000000002",
"0140 00000000124",
"0151 00000217222",
], return_value)
def test_date_iso(self):
"""
Tests iso datetime format
"""
return_value = Survey.parse_timestamp("2017-01-11")
self.assertNotIsInstance(return_value, datetime.datetime)
self.assertIsInstance(return_value, datetime.date)
def test_date_diary(self):
"""
Tests date date diary datetime format
"""
return_value = Survey.parse_timestamp("11/07/2017")
self.assertNotIsInstance(return_value, datetime.datetime)
self.assertIsInstance(return_value, datetime.date)
def test_date_iso(self):
"""
Tests iso datetime format
"""
return_value = Survey.parse_timestamp("2017-01-11")
self.assertNotIsInstance(return_value, datetime.datetime)
self.assertIsInstance(return_value, datetime.date)
def test_date_diary(self):
"""
Tests date date diary datetime format
"""
return_value = Survey.parse_timestamp("11/07/2017")
self.assertNotIsInstance(return_value, datetime.datetime)
self.assertIsInstance(return_value, datetime.date)
def test_load_survey_miss(self):
"""
Tests if load data is missed if survey id is not 134
"""
ids = Survey.identifiers({
"survey_id": "127",
"tx_id": "27923934-62de-475c-bc01-433c09fd38b8",
"collection": {
"instrument_id": "0001",
"period": "201704"
},
"metadata": {
"user_id": "123456789",
"ru_ref": "12345678901A"
}
}, batch_nr=0, seq_nr=0)
return_value = Survey.load_survey(ids, MWSSTransformer.pattern)
self.assertIsNone(return_value)
def test_load_survey(self):
"""
Tests if load data passes if survey id is 134
"""
ids = Survey.identifiers({
"survey_id": "134",
"tx_id": "27923934-62de-475c-bc01-433c09fd38b8",
"collection": {
"instrument_id": "0005",
"period": "201704"
},
"metadata": {
"user_id": "123456789",
"ru_ref": "12345678901A"
}
}, batch_nr=0, seq_nr=0)
return_value = Survey.load_survey(ids, MWSSTransformer.pattern)
self.assertIsNotNone(return_value)
def test_load_survey_miss(self):
"""
Tests if load data is missed if survey id is not 134
"""
ids = Survey.identifiers({
"survey_id": "127",
"tx_id": "27923934-62de-475c-bc01-433c09fd38b8",
"collection": {
"instrument_id": "0001",
"period": "201704"
},
"metadata": {
"user_id": "123456789",
"ru_ref": "12345678901A"
}
}, batch_nr=0, seq_nr=0)
with pytest.raises(MissingSurveyException):
Survey.load_survey(ids, MWSSTransformer.pattern)
def __init__(self, response, seq_nr=0, log=None):
"""Create a transformer object to process a survey response."""
self.response = response
self.ids = Survey.identifiers(response, seq_nr=seq_nr)
if self.ids is None:
raise UserWarning("Missing identifiers")
if log is None:
self.log = wrap_logger(logging.getLogger(__name__))
else:
self.log = Survey.bind_logger(log, self.ids)
# Enforce that child classes have defn and pattern attributes
for attr in ("defn", "pattern"):
if not hasattr(self.__class__, attr):
raise UserWarning("Missing class attribute: {0}".format(attr))
self.survey = Survey.load_survey(self.ids, self.pattern)
self.image_transformer = ImageTransformer(self.log, self.survey, self.response,
sequence_no=self.ids.seq_nr, base_image_path=SDX_FTP_IMAGE_PATH)
def test_idbr_receipt(self):
"""
Tests inter-departmental business register receipt
"""
src = pkg_resources.resource_string(__name__, "replies/eq-mwss.json")
reply = json.loads(src.decode("utf-8"))
reply["tx_id"] = "27923934-62de-475c-bc01-433c09fd38b8"
ids = Survey.identifiers(reply, batch_nr=3866, seq_nr=0)
id_dict = ids._asdict()
return_value = CSFormatter._idbr_receipt(id_dict["survey_id"], id_dict["ru_ref"], id_dict["ru_check"],
id_dict["period"])
self.assertEqual("12346789012:A:134:201605", return_value)
def test_identifiers(self):
self.reply["tx_id"] = "27923934-62de-475c-bc01-433c09fd38b8"
self.reply["collection"]["period"] = "200911"
ids = Survey.identifiers(self.reply)
self.assertIsInstance(ids, Survey.Identifiers)
self.assertEqual(0, ids.batch_nr)
self.assertEqual(0, ids.seq_nr)
self.assertEqual(self.reply["tx_id"], ids.tx_id)
self.assertEqual(datetime.date.today(), ids.ts.date())
self.assertEqual("134", ids.survey_id)
self.assertEqual("K5O86M2NU1", ids.user_id)
self.assertEqual("12346789012", ids.ru_ref)
self.assertEqual("A", ids.ru_check)
self.assertEqual("200911", ids.period)
def test_identifiers(self):
self.reply["tx_id"] = "27923934-62de-475c-bc01-433c09fd38b8"
self.reply["collection"]["period"] = "200911"
ids = Survey.identifiers(self.reply)
self.assertIsInstance(ids, Survey.Identifiers)
self.assertEqual(0, ids.batch_nr)
self.assertEqual(0, ids.seq_nr)
self.assertEqual(self.reply["tx_id"], ids.tx_id)
self.assertEqual(datetime.date.today(), ids.ts.date())
self.assertEqual("134", ids.survey_id)
self.assertEqual("K5O86M2NU1", ids.user_id)
self.assertEqual("12346789012", ids.ru_ref)
self.assertEqual("A", ids.ru_check)
self.assertEqual("200911", ids.period)
def test_idbr_receipt(self):
"""
Tests inter-departmental business register receipt
"""
f = open(self.replies_dir + "eq-mwss.json", "r")
reply = json.loads(f.read())
f.close()
reply["tx_id"] = "27923934-62de-475c-bc01-433c09fd38b8"
ids = Survey.identifiers(reply, batch_nr=3866, seq_nr=0)
id_dict = ids._asdict()
return_value = CSFormatter._idbr_receipt(id_dict["survey_id"], id_dict["ru_ref"], id_dict["ru_check"],
id_dict["period"])
self.assertEqual("12346789012:A:134:201605", return_value)
def test_pck_from_transformed_data(self):
self.reply["tx_id"] = "27923934-62de-475c-bc01-433c09fd38b8"
self.reply["survey_id"] = "134"
self.reply["collection"]["period"] = "200911"
self.reply["metadata"]["ru_ref"] = "49900001225C"
self.reply["data"] = OrderedDict([("0001", 2), ("0140", 124),
("0151", 217222)])
ids = Survey.identifiers(self.reply, batch_nr=3866)
id_dict = ids._asdict()
rv = CSFormatter._pck_lines(self.reply["data"], id_dict["inst_id"],
id_dict["ru_ref"], id_dict["ru_check"],
id_dict["period"])
self.assertEqual([
"FV ",
"0005:49900001225C:200911",
"0001 00000000002",
"0140 00000000124",
"0151 00000217222",
], rv)
def test_identifiers(self):
"""
Tests identifiers are valid
"""
src = pkg_resources.resource_string(__name__, "replies/eq-mwss.json")
reply = json.loads(src.decode("utf-8"))
reply["tx_id"] = "27923934-62de-475c-bc01-433c09fd38b8"
reply["collection"]["period"] = "200911"
ids = Survey.identifiers(reply, batch_nr=0, seq_nr=0)
self.assertIsInstance(ids, Survey.Identifiers)
self.assertEqual(0, ids.batch_nr)
self.assertEqual(0, ids.seq_nr)
self.assertEqual(reply["tx_id"], ids.tx_id)
self.assertEqual(datetime.date.today(), ids.ts.date())
self.assertEqual("134", ids.survey_id)
self.assertEqual("K5O86M2NU1", ids.user_id)
self.assertEqual("12346789012", ids.ru_ref)
self.assertEqual("A", ids.ru_check)
self.assertEqual("200911", ids.period)
def __init__(self, response, seq_nr=0):
self.response = response
self.ids = Survey.identifiers(self.response, seq_nr=seq_nr, log=logger)
survey_file = "./transform/surveys/{}.{}.json".format(
self.ids.survey_id, self.ids.inst_id
)
with open(survey_file) as fp:
logger.info("Loading {}".format(survey_file))
self.survey = json.load(fp)
self.image_transformer = ImageTransformer(
logger,
self.survey,
self.response,
sequence_no=self.ids.seq_nr,
base_image_path=SDX_FTP_IMAGE_PATH,
)
def test_identifiers(self):
"""
Tests identifiers are valid
"""
f = open(self.replies_dir + "eq-mwss.json", "r")
reply = json.loads(f.read())
f.close()
reply["tx_id"] = "27923934-62de-475c-bc01-433c09fd38b8"
reply["collection"]["period"] = "200911"
ids = Survey.identifiers(reply, batch_nr=0, seq_nr=0)
self.assertIsInstance(ids, Survey.Identifiers)
self.assertEqual(0, ids.batch_nr)
self.assertEqual(0, ids.seq_nr)
self.assertEqual(reply["tx_id"], ids.tx_id)
self.assertEqual(datetime.date.today(), ids.ts.date())
self.assertEqual("134", ids.survey_id)
self.assertEqual("K5O86M2NU1", ids.user_id)
self.assertEqual("12346789012", ids.ru_ref)
self.assertEqual("A", ids.ru_check)
self.assertEqual("200911", ids.period)
def test_pck_from_transformed_data(self):
self.reply["tx_id"] = "27923934-62de-475c-bc01-433c09fd38b8"
self.reply["survey_id"] = "134"
self.reply["collection"]["period"] = "200911"
self.reply["metadata"]["ru_ref"] = "49900001225C"
self.reply["data"] = OrderedDict([
("0001", 2),
("0140", 124),
("0151", 217222)
])
ids = Survey.identifiers(self.reply, batch_nr=3866)
id_dict = ids._asdict()
rv = CSFormatter._pck_lines(self.reply["data"], id_dict["inst_id"], id_dict["ru_ref"], id_dict["ru_check"],
id_dict["period"])
self.assertEqual([
"FV ",
"0005:49900001225C:200911",
"0001 00000000002",
"0140 00000000124",
"0151 00000217222",
], rv)
def events(qid, data, default, *args, group=[], **kwargs):
"""Return a sequence of time events from a question group.
:param str qid: The question id.
:param data: The full survey data.
:type data: dict(str, str)
:param default: The default value for the question.
:param group: A sequence of question ids.
"""
try:
group_vals = [data.get(qid, None)] + [data.get(q, None) for q in group]
data = sorted(filter(
None, (Survey.parse_timestamp(i) for i in group_vals if i is not None)
))
if all(isinstance(i, type(default)) for i in data):
return data
else:
return type(default)(data)
except (AttributeError, TypeError, ValueError):
return default
def events(qid, data, default, *args, group=[], **kwargs):
"""Return a sequence of time events from a question group.
:param str qid: The question id.
:param data: The full survey data.
:type data: dict(str, str)
:param default: The default value for the question.
:param group: A sequence of question ids.
"""
try:
group_vals = [data.get(qid, None)
] + [data.get(q, None) for q in group]
data = sorted(
filter(None, (Survey.parse_timestamp(i)
for i in group_vals if i is not None)))
if all(isinstance(i, type(default)) for i in data):
return data
else:
return type(default)(data)
except (AttributeError, TypeError, ValueError):
return default
def get_identifiers(self, batch_nr=0, seq_nr=0):
"""Parse common metadata from the survey.
Return a named tuple which code can use to access the various ids and discriminators.
:param dict data: A survey reply.
:param int batch_nr: A batch number for the reply.
:param int seq_nr: An image sequence number for the reply.
"""
logger.info("Parsing data from submission")
ru_ref = self.response.get("metadata", {}).get("ru_ref", "")
ts = datetime.datetime.now(datetime.timezone.utc)
ids = {
"batch_nr": batch_nr,
"seq_nr": seq_nr,
"ts": ts,
"tx_id": self.response.get("tx_id"),
"survey_id": self.response.get("survey_id"),
"instrument_id": self.response.get("collection", {}).get("instrument_id"),
"submitted_at": Survey.parse_timestamp(
self.response.get("submitted_at", ts.isoformat())
),
"user_id": self.response.get("metadata", {}).get("user_id"),
"ru_ref": "".join(i for i in ru_ref if i.isdigit()),
"ru_check": ru_ref[-1] if ru_ref and ru_ref[-1].isalpha() else "",
"period": self.response.get("collection", {}).get("period"),
}
for key, value in ids.items():
if value is None:
logger.error("Missing value for: {0}".format(key))
return None
return ids
def get_identifiers(self, batch_nr=0, seq_nr=0):
"""Parse common metadata from the survey.
Return a named tuple which code can use to access the various ids and discriminators.
:param int batch_nr: A batch number for the reply.
:param int seq_nr: An image sequence number for the reply.
"""
logger.info("Parsing data from submission")
ru_ref = self.response.get("metadata", {}).get("ru_ref", "")
ts = datetime.datetime.now(datetime.timezone.utc)
ids = {
"batch_nr": batch_nr,
"seq_nr": seq_nr,
"ts": ts,
"tx_id": self.response.get("tx_id"),
"survey_id": self.response.get("survey_id"),
"instrument_id": self.response.get("collection", {}).get("instrument_id"),
"submitted_at": Survey.parse_timestamp(
self.response.get("submitted_at", ts.isoformat())
),
"user_id": self.response.get("metadata", {}).get("user_id"),
"ru_ref": "".join(i for i in ru_ref if i.isdigit()),
"ru_check": ru_ref[-1] if ru_ref and ru_ref[-1].isalpha() else "",
"period": self.response.get("collection", {}).get("period"),
}
for key, value in ids.items():
if value is None:
logger.error(f"Missing value for: {key}")
return None
return ids
def test_date_period(self):
rv = Survey.parse_timestamp("201605")
self.assertNotIsInstance(rv, datetime.datetime)
self.assertIsInstance(rv, datetime.date)
def test_date_diary(self):
rv = Survey.parse_timestamp("11/07/2017")
self.assertNotIsInstance(rv, datetime.datetime)
self.assertIsInstance(rv, datetime.date)
def test_date_iso(self):
rv = Survey.parse_timestamp("2017-01-11")
self.assertNotIsInstance(rv, datetime.datetime)
self.assertIsInstance(rv, datetime.date)
def test_datetime_ms_with_colon_in_timezone(self):
rv = Survey.parse_timestamp("2017-01-11T17:18:53.020222+00:00")
self.assertIsInstance(rv, datetime.datetime)
def test_date_period(self):
rv = Survey.parse_timestamp("201605")
self.assertNotIsInstance(rv, datetime.datetime)
self.assertIsInstance(rv, datetime.date)
def test_datetime_zulu(self):
rv = Survey.parse_timestamp("2017-01-11T17:18:53Z")
self.assertIsInstance(rv, datetime.datetime)
def test_date_iso(self):
rv = Survey.parse_timestamp("2017-01-11")
self.assertNotIsInstance(rv, datetime.datetime)
self.assertIsInstance(rv, datetime.date)
def test_date_diary(self):
rv = Survey.parse_timestamp("11/07/2017")
self.assertNotIsInstance(rv, datetime.datetime)
self.assertIsInstance(rv, datetime.date)
def test_datetime_zulu(self):
rv = Survey.parse_timestamp("2017-01-11T17:18:53Z")
self.assertIsInstance(rv, datetime.datetime)
def test_datetime_ms_with_colon_in_timezone(self):
rv = Survey.parse_timestamp("2017-01-11T17:18:53.020222+00:00")
self.assertIsInstance(rv, datetime.datetime)