def get_current_checksum_include_ignore_text(uuid):
import hashlib
from backend import fetch_site_status
# Get the most recent one
newest_history_key = datastore.get_val(uuid, 'newest_history_key')
# 0 means that theres only one, so that there should be no 'unviewed' history availabe
if newest_history_key == 0:
newest_history_key = list(
datastore.data['watching'][uuid]['history'].keys())[0]
if newest_history_key:
with open(datastore.data['watching'][uuid]['history']
[newest_history_key],
encoding='utf-8') as file:
raw_content = file.read()
handler = fetch_site_status.perform_site_check(
datastore=datastore)
stripped_content = handler.strip_ignore_text(
raw_content,
datastore.data['watching'][uuid]['ignore_text'])
checksum = hashlib.md5(stripped_content).hexdigest()
return checksum
return datastore.data['watching'][uuid]['previous_md5']
def test_strip_text_func():
from backend import fetch_site_status
test_content = """
Some content
is listed here
but sometimes we want to remove the lines.
but not always."""
ignore_lines = ["sometimes"]
fetcher = fetch_site_status.perform_site_check(datastore=False)
stripped_content = fetcher.strip_ignore_text(test_content, ignore_lines)
assert b"sometimes" not in stripped_content
assert b"Some content" in stripped_content
def test_strip_regex_text_func():
from backend import fetch_site_status
test_content = """
but sometimes we want to remove the lines.
but 1 lines
but including 1234 lines
igNORe-cAse text we dont want to keep
but not always."""
ignore_lines = ["sometimes", "/\s\d{2,3}\s/", "/ignore-case text/"]
fetcher = fetch_site_status.perform_site_check(datastore=False)
stripped_content = fetcher.strip_ignore_text(test_content, ignore_lines)
assert b"but 1 lines" in stripped_content
assert b"igNORe-cAse text" not in stripped_content
assert b"but 1234 lines" not in stripped_content
def run(self):
from backend import fetch_site_status
update_handler = fetch_site_status.perform_site_check(
datastore=datastore)
while not app.config.exit.is_set():
try:
uuid = self.q.get(block=False)
except queue.Empty:
pass
else:
self.current_uuid = uuid
if uuid in list(datastore.data['watching'].keys()):
try:
changed_detected, result, contents = update_handler.run(
uuid)
except PermissionError as s:
app.logger.error("File permission error updating",
uuid, str(s))
else:
if result:
datastore.update_watch(uuid=uuid,
update_obj=result)
if changed_detected:
# A change was detected
datastore.save_history_text(uuid=uuid,
contents=contents,
result_obj=result)
self.current_uuid = None # Done
self.q.task_done()
app.config.exit.wait(1)
def run(self):
from backend import fetch_site_status
update_handler = fetch_site_status.perform_site_check(
datastore=self.datastore)
while not self.app.config.exit.is_set():
try:
uuid = self.q.get(block=False)
except queue.Empty:
pass
else:
self.current_uuid = uuid
if uuid in list(self.datastore.data['watching'].keys()):
try:
changed_detected, result, contents = update_handler.run(
uuid)
except PermissionError as e:
self.app.logger.error("File permission error updating",
uuid, str(e))
except Exception as e:
self.app.logger.error("Exception reached", uuid,
str(e))
else:
if result:
try:
self.datastore.update_watch(uuid=uuid,
update_obj=result)
if changed_detected:
# A change was detected
newest_version_file_contents = ""
self.datastore.save_history_text(
uuid=uuid,
contents=contents,
result_obj=result)
watch = self.datastore.data['watching'][
uuid]
print(">> Change detected in UUID {} - {}".
format(uuid, watch['url']))
# Get the newest snapshot data to be possibily used in a notification
newest_key = self.datastore.get_newest_history_key(
uuid)
if newest_key:
with open(watch['history'][newest_key],
'r') as f:
newest_version_file_contents = f.read(
).strip()
n_object = {
'watch_url':
watch['url'],
'uuid':
uuid,
'current_snapshot':
newest_version_file_contents
}
# Did it have any notification alerts to hit?
if len(watch['notification_urls']):
print(
">>> Notifications queued for UUID from watch {}"
.format(uuid))
n_object['notification_urls'] = watch[
'notification_urls']
self.notification_q.put(n_object)
# No? maybe theres a global setting, queue them all
elif len(
self.datastore.data['settings']
['application']['notification_urls']):
print(
">>> Watch notification URLs were empty, using GLOBAL notifications for UUID: {}"
.format(uuid))
n_object[
'notification_urls'] = self.datastore.data[
'settings']['application'][
'notification_urls']
self.notification_q.put(n_object)
else:
print(
">>> NO notifications queued, watch and global notification URLs were empty."
)
except Exception as e:
print("!!!! Exception in update_worker !!!\n",
e)
self.current_uuid = None # Done
self.q.task_done()
self.app.config.exit.wait(1)
def run(self):
from backend import fetch_site_status
update_handler = fetch_site_status.perform_site_check(
datastore=self.datastore)
while not self.app.config.exit.is_set():
try:
uuid = self.q.get(block=False)
except queue.Empty:
pass
else:
self.current_uuid = uuid
if uuid in list(self.datastore.data['watching'].keys()):
try:
changed_detected, result, contents = update_handler.run(
uuid)
except PermissionError as s:
self.app.logger.error("File permission error updating",
uuid, str(s))
else:
if result:
try:
self.datastore.update_watch(uuid=uuid,
update_obj=result)
if changed_detected:
# A change was detected
self.datastore.save_history_text(
uuid=uuid,
contents=contents,
result_obj=result)
watch = self.datastore.data['watching'][
uuid]
# Did it have any notification alerts to hit?
if len(watch['notification_urls']):
print(
"Processing notifications for UUID: {}"
.format(uuid))
n_object = {
'watch_url':
self.datastore.data['watching']
[uuid]['url'],
'notification_urls':
watch['notification_urls']
}
self.notification_q.put(n_object)
# No? maybe theres a global setting, queue them all
elif len(
self.datastore.data['settings']
['application']['notification_urls']):
print(
"Processing GLOBAL notifications for UUID: {}"
.format(uuid))
n_object = {
'watch_url':
self.datastore.data['watching']
[uuid]['url'],
'notification_urls':
self.datastore.data['settings']
['application']
['notification_urls']
}
self.notification_q.put(n_object)
except Exception as e:
print("!!!! Exception in update_worker !!!\n",
e)
self.current_uuid = None # Done
self.q.task_done()
self.app.config.exit.wait(1)