def url(filename):
d = {}
f = open(filename,'r')
for line in f:
url = line.split()[0]
name = util.getId(url)
d[name] = url
return d
def url(filename):
d = {}
f = open(filename, 'r')
for line in f:
url = line.split()[0]
name = util.getId(url)
d[name] = url
return d
def download(self):
f = open(self.filename, 'r')
for line in f:
# the first component of the line should be the git url
url = line.split()[0]
name = util.getId(url)
command = self.prefix+url+' '+self.directory + '/' + name
DownloadThread(command).start()
f.close()
def realmFuseWorker(file, ids):
conn, cur = util.open_sqlite(file)
ip = util.extractIp(file)
if ip is None:
logging.error("Could not find Ip in %s" % file)
return
_id = util.getId(ids, ip=ip)
if _id is None:
logging.error("Could not find Id for ip %s" % ip)
return
realm_data = util.query_sqlite(cur,
"SELECT * FROM realm_data",
if_no_table=[])
if len(realm_data) == 0:
logging.warning("%s has no realm_data" % os.path.basename(file))
return
realm_data = [dict(i)
for i in realm_data] # cast to dict for get() function
inserts = []
for row in realm_data:
received = row["isReceived"]
if received != row[
"isDelieverd"]: # isReceived is sometimes 1 while isDelieverd is 0
logging.error(
"isReceived: %s != isDelieverd: %s for %s at %s from %s" %
(row["isReceived"], row["isDelieverd"], row["messageType"],
row["index"], os.path.basename(file)))
continue
other_id = None
messageType = None
lat = row.get("j_latitude", None)
lon = row.get("j_longitude", None)
category = None
text1, text2, text3 = (None, None, None)
if row["messageType"] is not None:
messageType = row["messageType"].replace("_message", "")
if messageType == "hilferuf":
if received == 1:
other_id = util.getId(ids, dtn_id=row["dtn_id"])
if other_id is None:
logging.error(
"Could not find id for %s in %s at %s" %
(row["dtn_id"], os.path.basename(file), row["index"]))
continue
else:
other_id = None # if send its a broadcast
text1 = row["j_passiertText"]
text2 = row["j_beschreibenText"]
text3 = row["j_isVerletzte"]
if row["j_categories"] is not None:
category = {}
category_checks = json.loads(row["j_categories"])
rettungsdienst = []
polizei = []
netzwerk = []
for key in category_checks:
i = category_checks[key]
if i["isChecked"]:
if i["parentCategory"] == "rettungsdiens": # sic
rettungsdienst.append(i["title"])
elif i["parentCategory"] == "smarternetCategory":
netzwerk.append(i["title"])
elif i["parentCategory"] == "polizei":
polizei.append(i["title"])
else:
logging.error(
"Unkown parentCategory %s from %s at %s" %
(i["parentCategory"], os.path.basename(file),
row["index"]))
category["Rettungsdienst"] = rettungsdienst
category["Polizei"] = polizei
category["Netzwerk"] = netzwerk
category = json.dumps(category)
hash_content = "%s%s%s%s%s" % (messageType, category, text1, text2,
text3)
elif messageType == "ressourcenmarkt":
# seams to be always 0, no way to know if this message send by this node or received
if received != 0:
logging.error(
"ressourcenmarkt with received !=0 from %s at %s" %
(os.path.basename(file), row["index"]))
# dtn_id and j_from differ from nodes own dtn_ids so probably received
search_id = util.getId(ids, dtn_id=row["dtn_id"])
if search_id is None:
logging.error(
"Could not find id for %s in %s at %s" %
(row["dtn_id"], os.path.basename(file), row["index"]))
continue
if search_id == _id: # from is own id then this was probably send into the network, but could also be duplicate receive (or does the bundle layer filter these out before)
received = 0
else:
received = 1
text1 = row["j_title"]
text2 = row["j_desc"]
text3 = row["j_Quanitity"] # sic
category = row["j_category"]
hash_content = "%s%s%s%s%s" % (messageType, category, text1, text2,
text3)
# for 0b3e0292e03a9d67fcdfb8d6658e58a8 (#481) all None (realm_creator aslo None) but they clearly dont belong together, no way to generate id
# ccf5d56fb962485f4d9148072d17b51e multiple with received 0 but equal realm_creator one of them has status -1 ???
# cfab094f0ea57d155ff1508ac90a0921 multiple with received 0 and equal realm_creator
elif messageType == "ressourcenmarkt_delete":
# has nearly no data, check if received is set properly, seams to be always 0 -> no way to differ direction
if received != 0:
logging.error(
"ressourcenmarkt_delete with received !=0 from %s at %s" %
(os.path.basename(file), row["index"]))
# hash_content = "%s%s%s" % (messageType, _id, row["timestamp"]) # not usefull unable to detect if the same delte arrives somewhere
hash_content = None
elif messageType == "chat":
if row["j_from"] is None or row["j_to"] is None or row[
"dtn_id"] is None:
logging.error(
"One of the chat message ids is None from %s at %s" %
(os.path.basename(file), row["index"]))
continue
from_id = util.getId(ids, dtn_id=row["j_from"])
if from_id is None:
logging.error(
"Could not find from_id for %s in %s at %s" %
(row["j_from"], os.path.basename(file), row["index"]))
to_id = util.getId(ids, dtn_id=row["j_to"])
if to_id is None:
logging.error(
"Could not find to_id for %s in %s at %s" %
(row["j_to"], os.path.basename(file), row["index"]))
other_id = util.getId(ids, dtn_id=row["dtn_id"])
if other_id is None:
logging.error(
"Could not find other_id for %s in %s at %s" %
(row["dtn_id"], os.path.basename(file), row["index"]))
if received == 0:
if not (_id == from_id and other_id == to_id):
logging.error(
"Ids of sending chat message dont match from %s at %s"
% (os.path.basename(file), row["index"]))
continue
else:
if not (_id == to_id and other_id == from_id):
logging.error(
"Ids of receiving chat message dont match from %s at %s"
% (os.path.basename(file), row["index"]))
continue
text1 = row["j_message"]
if _id is None or other_id is None:
logging.error(
"Could not find all chat message ids, from %s at %s" %
(os.path.basename(file), row["index"]))
continue
ordered_ids = sorted([_id, other_id])
hash_content = "%s%s%s%s" % (messageType, ordered_ids[0],
ordered_ids[1], text1)
# unwanted collisions if the same text is send again, happens ~20 times
elif messageType == "lebenszeichen" or messageType == "personenfinder":
other_id = util.getId(ids, dtn_id=row["dtn_id"])
if other_id is None:
logging.error(
"Could not find id for %s in %s at %s" %
(row["dtn_id"], os.path.basename(file), row["index"]))
continue
if received == 1:
hash_content = "%s%s->%s" % (messageType, other_id, _id)
else:
hash_content = "%s%s->%s" % (messageType, _id, other_id)
# many collisions when received=0, double sends?
else:
logging.error(
"Unkown messageType %s from %s at %s" %
(row["messageType"], os.path.basename(file), row["index"]))
continue
# md5 hash as message id
if hash_content is not None:
md5 = hashlib.md5(hash_content.encode("utf-8")).hexdigest()
else:
md5 = None # for some types it is not possible to generate a meaningful msg id
inserts.append([
row["timestamp"],
row.get("j_sentTime", None), _id, other_id, received, messageType,
md5, row["json_size"], lat, lon, category, text1, text2, text3,
row.get("j_creatorORMMessageId", None),
row.get("status", None)
])
if len(inserts) > 0:
return inserts
else:
return
