def fetch_latest_euribor_rates(commit: bool = False,
verbose: bool = False) -> List[EuriborRate]:
feed_url = (
"https://www.suomenpankki.fi/WebForms/ReportViewerPage.aspx?report=/tilastot/markkina-_ja_hallinnolliset_korot/euribor_korot_today_xml_en&output=xml"
)
res = requests.get(feed_url)
if verbose:
logger.info("GET %s HTTP %s\n%s", feed_url, res.status_code,
res.content)
if res.status_code >= 300:
raise Exception(f"Failed to load Euribor rate feed from {feed_url}")
results = xml_to_dict(res.content)
data = results["data"]["period_Collection"]["period"]
rates: List[EuriborRate] = []
if isinstance(data, list): # Sometime get a list or single item
assert len(data) > 0
data = data[len(data) - 1] # Get from newest date
record_date = parse_datetime(data["@value"]).date()
for rate_data in data["matrix1_Title_Collection"]["rate"]:
name = rate_data["@name"]
rate = dec4(rate_data["intr"]["@value"])
obj = EuriborRate(name=name, record_date=record_date, rate=rate)
rates.append(obj)
if commit:
out: List[EuriborRate] = []
for obj in rates:
out.append(
EuriborRate.objects.save_unique(obj.record_date, obj.name,
obj.rate))
return out
return rates
def test_dict_to_xml(self):
from xml.etree.ElementTree import Element
from xml.etree import ElementTree as ET
data = {
'Doc': {
'@version':
'1.2',
'A': [{
'@class': 'x',
'B': {
'@': 'hello',
'@class': 'x2'
}
}, {
'@class': 'y',
'B': {
'@': 'world',
'@class': 'y2'
}
}],
'C':
'value node',
}
}
el = dict_to_element(data)
assert isinstance(el, Element)
xml_str = ET.tostring(el, encoding='utf8', method='xml').decode()
# print(xml_str) # <Doc version="1.2"><C>value node</C><A class="x"><B class="x2">hello</B></A><A class="y"><B class="y2">world</B></A></Doc>
data2 = xml_to_dict(xml_str, document_tag=True)
# print(data)
# print(data2)
self.assertEqual(data2, data)
def camt053_parse_statement_from_file(filename: str) -> dict:
if parse_filename_suffix(
filename).upper() not in CAMT053_STATEMENT_SUFFIXES:
raise ValidationError(
_('File {filename} has unrecognized ({suffixes}) suffix for file type "{file_type}"'
).format(filename=filename,
suffixes=", ".join(CAMT053_STATEMENT_SUFFIXES),
file_type="camt.053"))
with open(filename, "rb") as fp:
data = xml_to_dict(fp.read(),
array_tags=CAMT053_ARRAY_TAGS,
int_tags=CAMT053_INT_TAGS)
return data
def test_parse_xml(self):
# finvoice_201_example1.xml
xml_bytes = open(
join(settings.BASE_DIR, 'data/fi/finvoice_201_example1.xml'),
'rb').read()
data = xml_to_dict(xml_bytes, value_key='value', attribute_prefix='_')
# pprint(data)
self.assertEqual(data['_Version'], '2.01')
self.assertEqual(data['InvoiceRow'][0]['ArticleIdentifier'], '12345')
self.assertEqual(data['InvoiceRow'][0]['DeliveredQuantity']['value'],
'2')
self.assertEqual(
data['InvoiceRow'][0]['DeliveredQuantity']['_QuantityUnitCode'],
'kpl')
self.assertEqual(data['InvoiceRow'][1]['ArticleIdentifier'], '123456')
# parse_xml1.xml
xml_str = open(join(settings.BASE_DIR, 'data/parse_xml1.xml'),
'rt').read()
data = xml_to_dict(xml_str)
# pprint(data)
ref_data = {
'@version':
'1.2',
'A': [{
'@class': 'x',
'B': {
'@': 'hello',
'@class': 'x2'
}
}, {
'@class': 'y',
'B': {
'@': 'world',
'@class': 'y2'
}
}],
'C':
'value node'
}
self.assertEqual(ref_data, data)
# parse_xml1.xml / no attributes
xml_str = open(join(settings.BASE_DIR, 'data/parse_xml1.xml'),
'rt').read()
data = xml_to_dict(xml_str, parse_attributes=False)
# pprint(data)
ref_data = {'A': [{'B': 'hello'}, {'B': 'world'}], 'C': 'value node'}
self.assertEqual(ref_data, data)
# parse_xml2.xml / no attributes
xml_str = open(join(settings.BASE_DIR, 'data/parse_xml2.xml'),
'rt').read()
data = xml_to_dict(xml_str, ['VastausLoki', 'LuottoTietoMerkinnat'],
parse_attributes=False)
# pprint(data)
ref_data = {
'VastausLoki': {
'KysyttyHenkiloTunnus': '020685-1234',
'PaluuKoodi': 'Palveluvastaus onnistui',
'SyyKoodi': '1'
}
}
self.assertEqual(ref_data, data)
def do(self, *args, **options): # pylint: disable=too-many-branches
default_ws = WsEdiConnection.objects.get(
id=options["default_ws"]) if options["default_ws"] else None
assert default_ws is None or isinstance(default_ws, WsEdiConnection)
file_type = options["file_type"]
if not file_type:
print("--file-type required (e.g. XL, NDCORPAYS, pain.001.001.03)")
return
payouts = Payout.objects.all()
if options["payout"]:
payouts = Payout.objects.filter(id=options["payout"])
else:
payouts = payouts.filter(state=PAYOUT_WAITING_UPLOAD)
if options["ws"]:
payouts = payouts.filter(connection_id=options["ws"])
for p in list(payouts):
assert isinstance(p, Payout)
p.refresh_from_db()
ws_connection = p.connection or default_ws
if p.state != PAYOUT_WAITING_UPLOAD:
logger.info(
"Skipping {} since not in state PAYOUT_WAITING_UPLOAD".
format(p))
continue
if ws_connection and not ws_connection.enabled:
logger.info(
"WS connection %s not enabled, skipping payment %s",
ws_connection, p)
continue
response_code = ""
response_text = ""
try:
# upload file
logger.info("Uploading payment id={} {} file {}".format(
p.id, file_type, p.full_path))
with open(p.full_path, "rt", encoding="utf-8") as fp:
file_content = fp.read()
p.state = PAYOUT_UPLOADED
p.save(update_fields=["state"])
if ws_connection:
content = wsedi_execute(
ws_connection,
"UploadFile",
file_content=file_content,
file_type=file_type,
verbose=options["verbose"],
)
data = xml_to_dict(content, array_tags=["FileDescriptor"])
else:
res = wsedi_upload_file(
file_content=file_content,
file_type=file_type,
file_name=p.file_name,
verbose=options["verbose"],
)
logger.info("HTTP response {}".format(res.status_code))
logger.info(res.text)
data = res.json()
# parse response
response_code = data.get("ResponseCode", "")[:4]
response_text = data.get("ResponseText", "")[:255]
if response_code != "00":
msg = "WS-EDI file {} upload failed: {} ({})".format(
p.file_name, response_text, response_code)
logger.error(msg)
raise Exception("Response code {} ({})".format(
response_code, response_text))
if "FileDescriptors" in data:
fds = data.get("FileDescriptors",
{}).get("FileDescriptor", [])
fd = {} if not fds else fds[0]
file_reference = fd.get("FileReference", "")
if file_reference:
p.file_reference = file_reference
p.save(update_fields=["file_reference"])
PayoutStatus.objects.create(
payout=p,
msg_id=p.msg_id,
file_name=p.file_name,
response_code=response_code,
response_text=response_text,
status_reason="File upload OK",
)
except Exception as e:
long_err = "File upload failed ({}): ".format(
p.file_name) + traceback.format_exc()
logger.error(long_err)
short_err = "File upload failed: " + str(e)
p.state = PAYOUT_ERROR
p.save(update_fields=["state"])
PayoutStatus.objects.create(
payout=p,
group_status=PAYOUT_ERROR,
msg_id=p.msg_id,
file_name=p.file_name,
response_code=response_code,
response_text=response_text,
status_reason=short_err[:255],
)
def do(self, *args, **options): # pylint: disable=too-many-statements
ws = WsEdiConnection.objects.get(
id=options["ws"]) if options["ws"] else None
assert ws is None or isinstance(ws, WsEdiConnection)
if ws and not ws.enabled:
logger.info("WS connection %s not enabled, exiting", ws)
return
start_date, end_date = parse_start_and_end_date(
pytz.timezone("Europe/Helsinki"), **options)
path = os.path.abspath(options["path"])
command = "DownloadFileList"
time_now = now()
file_reference = options["file_reference"]
if file_reference:
command = "DownloadFile"
status = options["status"]
file_type = options["file_type"]
if command == "DownloadFileList" and not file_type:
print(
"--file-type required (e.g. TO, SVM, XP, NDCORPAYL, pain.002.001.03)"
)
return
if ws:
content = wsedi_execute(
ws,
command=command,
file_type=file_type,
status=status,
start_date=start_date,
end_date=end_date,
file_reference=file_reference,
verbose=options["verbose"],
)
data = xml_to_dict(content, array_tags=["FileDescriptor"])
else:
res = wsedi_get(
command=command,
file_type=file_type,
status=status,
file_reference=file_reference,
verbose=options["verbose"],
)
data = res.json()
# "FileDescriptors": {
# "FileDescriptor": [
# {
# "FileReference": "535283541",
# "TargetId": "NONE",
# "UserFilename": "STOL001.FMV80KT2.WEBSER.PS",
# "ParentFileReference": "1218",
# "FileType": "TO",
# "FileTimestamp": "2017-12-18T20:33:09.362+02:00",
# "Status": "DLD",
# "LastDownloadTimestamp": "2017-12-19T12:36:34.490+02:00",
# "ForwardedTimestamp": "2017-12-18T20:33:09.362+02:00",
# "Deletable": "false",
# "CustomerNumber": "06720106",
# "Modifier": "06720106",
# "ModifiedTimestamp": "2017-12-19T12:36:34.490+02:00",
# "SourceId": "A",
# "Environment": "PRODUCTION"
# },
# ...
if command == "DownloadFileList":
if "FileDescriptors" in data and data[
"FileDescriptors"] is not None and "FileDescriptor" in data[
"FileDescriptors"]:
for fd in data["FileDescriptors"]["FileDescriptor"]:
file_reference = fd["FileReference"]
file_type = fd["FileType"]
file_basename = file_reference + "." + file_type
file_path = os.path.join(path, file_basename)
if options["list_only"]:
print(
"{file_reference} ({file_type}/{status}): {user_filename} ({timestamp})"
.format(
file_reference=file_reference,
file_type=file_type,
status=fd.get("Status"),
user_filename=fd.get("UserFilename"),
timestamp=fd.get("FileTimestamp"),
))
continue
if options["overwrite"] or not os.path.isfile(file_path):
command = "DownloadFile"
if ws:
content = wsedi_execute(
ws,
command=command,
file_type=file_type,
status="",
file_reference=file_reference,
verbose=options["verbose"],
)
file_data = xml_to_dict(content)
else:
res = wsedi_get(
command=command,
file_type=file_type,
status="",
file_reference=file_reference,
verbose=options["verbose"],
)
file_data = res.json()
if "Content" not in file_data:
logger.error(
"WS-EDI {} Content block missing: {}".format(
command, file_data))
raise Exception(
"WS-EDI {} Content block missing".format(
command))
bcontent = base64.b64decode(file_data["Content"])
with open(file_path, "wb") as fp:
fp.write(bcontent)
logger.info("Wrote file {}".format(file_path))
# process selected files immediately
if options["process_pain002"] and file_type in [
"XP", "pain.002.001.03", "NDCORPAYL"
]:
process_pain002_file_content(bcontent,
file_path,
created=time_now)
else:
print("Skipping old file {}".format(file_path))
else:
print("Empty file list downloaded")
elif command == "DownloadFile":
bcontent = base64.b64decode(data["Content"])
file_path = os.path.join(path, file_reference)
if options["overwrite"] or not os.path.isfile(file_path):
with open(file_path, "wb") as fp:
fp.write(bcontent)
logger.info("Wrote file {}".format(file_path))
else:
print("Skipping old file {}".format(file_path))