def ProcessDataSource(self: Any, source: Dict[str, Any]) -> None:
"""Prepare to diff the provided data source."""
source["hash"] = Utility.MD5(self, source["url"])
source["older"] = {}
source["old"] = {}
source["new"] = {}
older: Dict[str, Any] = source["older"]
old: Dict[str, Any] = source["old"]
new: Dict[str, Any] = source["new"]
format: str = source["contentType"].upper()
allowRevert: bool = source.get("allowRevert", True)
if format == "JSON":
source["ext"] = "json"
source["filename"] = source["hash"] + "." + source["ext"]
old["gist"] = Utility.GetGist(self, source["filename"])
new["raw"] = Utility.FormatJSON(self, Utility.GET(self, source["url"]))
if old["gist"] is False:
return
elif (new["raw"] is not None) and (old["gist"] is not None):
if allowRevert is False:
older["raw"] = Utility.FormatJSON(
self,
Utility.GetGistRaw(self, old["gist"], source["filename"], 1),
)
old["raw"] = Utility.FormatJSON(
self, Utility.GetGistRaw(self, old["gist"], source["filename"])
)
SitRep.DiffJSON(self, source)
elif (new["raw"] is not None) and (old["gist"] is None):
Utility.CreateGist(self, source)
elif format == "IMAGE":
source["ext"] = "txt"
source["filename"] = source["hash"] + "." + source["ext"]
old["gist"] = Utility.GetGist(self, source["filename"])
new["raw"] = Utility.Base64(
self, Utility.GET(self, source["url"], raw=True)
)
if old["gist"] is False:
return
elif (new["raw"] is not None) and (old["gist"] is not None):
if allowRevert is False:
older["raw"] = Utility.GetGistRaw(
self, old["gist"], source["filename"], 1
)
old["raw"] = Utility.GetGistRaw(self, old["gist"], source["filename"])
SitRep.DiffImage(self, source)
elif (new["raw"] is not None) and (old["gist"] is None):
Utility.CreateGist(self, source)
elif format == "TEXT":
source["ext"] = source.get("fileType", "txt")
source["filename"] = source["hash"] + "." + source["ext"]
old["gist"] = Utility.GetGist(self, source["filename"])
new["raw"] = Utility.GET(self, source["url"])
if old["gist"] is False:
return
elif (new["raw"] is not None) and (old["gist"] is not None):
if allowRevert is False:
older["raw"] = Utility.GetGistRaw(
self, old["gist"], source["filename"], 1
)
old["raw"] = Utility.GetGistRaw(self, old["gist"], source["filename"])
SitRep.DiffText(self, source)
elif (new["raw"] is not None) and (old["gist"] is None):
Utility.CreateGist(self, source)
else:
logger.error(f"Data source with content type {format} is not supported")
logger.debug(source)
def DiffText(self: Any, source: Dict[str, Any]) -> None:
"""Diff the provided text data source."""
filename: str = source["filename"]
url: str = source["url"]
allowRevert: bool = source.get("allowRevert", True)
older: Dict[str, Any] = source["older"]
old: Dict[str, Any] = source["old"]
new: Dict[str, Any] = source["new"]
if allowRevert is False:
older["hash"] = Utility.MD5(self, older["raw"])
old["hash"] = Utility.MD5(self, old["raw"])
new["hash"] = Utility.MD5(self, new["raw"])
if old["hash"] == new["hash"]:
logger.info(f"No difference found in {filename} ({url})")
return
elif (allowRevert is False) and (older["hash"] == new["hash"]):
logger.info(f"Ignored revert found in {filename} ({url})")
return
diff: Iterator[str] = Differ().compare(
old["raw"].splitlines(), new["raw"].splitlines()
)
desc: str = ""
additions: int = 0
deletions: int = 0
for line in diff:
if line.startswith("+ "):
additions += 1
desc += f"{line}\n"
elif line.startswith("- "):
deletions += 1
desc += f"{line}\n"
desc = Utility.Truncate(self, desc, 4048, split="\n")
source["urlTrim"] = Utility.Truncate(self, url, 256)
success: bool = SitRep.Notify(
self,
{
"title": source["urlTrim"],
"description": f"```diff\n{desc}```",
"url": url,
"filename": source["filename"],
"additions": f"{additions:,}",
"deletions": f"{deletions:,}",
"diffUrl": source["old"]["gist"].html_url + "/revisions",
},
)
# Ensure no changes go without notification
if success is True:
Utility.UpdateGist(self, source)