def save_progress(self):
self.end_date = udatetime.now_as_utc()
self.time_lapsed = self.end_date - self.start_date
self._write_json()
self._write_text()
self._write_urls()
def convert_ga(self, doc: html.Element) -> Dict:
t = doc.findall(".//table")
if len(t) == 0:
return { "error": "no tables -> page layout changed", "at": udatetime.now_as_utc() }
data = self._htmltable_to_dict(t[0])
if len(data["data"]) != 2:
return { "error": "expected two data rows", "at": udatetime.now_as_utc() }
if data["data"][0]["COVID-19 Confirmed Cases"] != "Total":
return { "error": "first row should be totals", "at": udatetime.now_as_utc() }
if data["data"][1]["COVID-19 Confirmed Cases"] != "Deaths":
return { "error": "second row should be deaths", "at": udatetime.now_as_utc() }
positive = data["data"][0]["No. Cases (%)"]
positive = int(positive[0: positive.index("(")])
deaths = data["data"][1]["No. Cases (%)"]
deaths = int(deaths[0: deaths.index("(")])
data = self._htmltable_to_dict(t[1])
if len(data["data"]) != 2:
return { "error": "expected two data rows", "at": udatetime.now_as_utc() }
if data["data"][0]["Lab"] != "Commercial Lab":
return { "error": "first row should be Commerial Lab", "at": udatetime.now_as_utc() }
if data["data"][1]["Lab"] != "GPHL":
return { "error": "second row should be GPHL", "at": udatetime.now_as_utc() }
lab_1 = int(data["data"][0]["Total Tests"])
lab_2 = int(data["data"][1]["Total Tests"])
tests = lab_1 + lab_2
return {
"positive": positive,
"tests": tests,
"deaths": deaths
}
def start_run(self):
self.load()
self.previous_date = self.start_date
self.start_date = udatetime.now_as_utc()
self.end_date = self.start_date
self.time_lapsed = self.end_date - self.start_date
self.error_message = None
self.complete = False
for x in self._items: x.complete = False
def update_dates(self) -> Tuple[str, str]:
" returns new and old date "
xpath = os.path.join(self.work_dir, "time_stamp.txt")
if os.path.exists(xpath):
with open(xpath, "r") as f:
old_date = f.readline()
else:
old_date = "[NONE]"
dt = udatetime.now_as_utc()
new_date = f"{dt} ({dt.tzname()})"
with open(xpath, "w") as f:
f.write(f"{new_date}\n")
return new_date, old_date
def _convert(self, location: str, content: Union[bytes,str], item: ChangeItem) -> Dict:
if self.trace: logger.info(f"convert ===>\n{content}<===\n")
if content == None or len(content) == 0: return {}
doc = html.fromstring(content)
if location == "GA.html":
return self.convert_ga(doc)
links = doc.get_element_by_id("links")
if links != None:
links.getparent().remove(links)
tables = doc.findall(".//table")
if len(tables) > 0:
logger.warning(f"found tables for {location}")
for t in tables:
logger.info(f" table ===>\n{html.tostring(t, pretty_print=True)}<===\n")
exit(-1)
return { "error": "found tables w/o parser", "at": udatetime.now_as_utc() }
else:
return { "error": "no tables", "at": udatetime.now_as_utc() }
def __init__(self, cache: DirectoryCache):
self.cache = cache
self.start_date = udatetime.now_as_utc()
self.end_date = self.start_date
self.previous_date = self.start_date
self.time_lapsed = self.end_date - self.start_date
self.error_message = None
self.complete = False
self.last_timestamp = ""
self._is_loaded = False
self._items = []
self._lookup = {}
def test_3():
print("==== test_3")
print("")
local = datetime.now().astimezone()
print(f"local = {local} {local.tzinfo}")
utc = udatetime.now_as_utc()
print(f"udatetime.now_as_utc = {utc} {utc.tzinfo}")
assert (utc.tzinfo == pytz.UTC)
local2 = utc.astimezone(local.tzinfo)
print(f"udatetime.now_as_utc in local time = {local2} {local2.tzinfo}")
assert ((local2.hour - local.hour) < 1)
print("")
print("")
def update_status(self, name: str, source: str, status: str, xurl: str, msg: str) -> Tuple[ChangeItem, ChangeItem, str]:
xnow = udatetime.now_as_utc()
if msg == "": msg = None
idx = self._lookup.get(name)
if idx == None:
x = None
y = ChangeItem({ "name": name, "source": source, "status": status, "url": xurl, "msg": msg, "complete": True,
"added": xnow, "checked": None, "updated": None, "failed": None })
self._lookup[name] = len(self._items)
self._items.append(y)
else:
x = self._items[idx]
y = x.copy()
y.status = status
y.source = source
y.url = xurl
y.msg = msg
y.complete = True
self._items[idx] = y
return y, x, xnow
def validate(self, source: UrlSource) -> bool:
self.is_valid = True
self.status_message = f"Url source {source.name} is valid"
if source.error_msg != None:
self.is_valid = False
self.status_message = source.error_msg
return False
error_msg = self._validate(source)
if error_msg != None:
logger.error(f"validate returned: {error_msg}")
source.status = "failed"
source.error_msg = error_msg
source.error_at = udatetime.now_as_utc()
self.is_valid = False
self.status_message = f"Url Source {source.name} is not valid"
return False
source.status = "valid"
source.error_msg = None
source.error_at = None
return True