def get_wind(json, n=0, t="03:00:00"):
now = datetime.datetime.now()
date = now.day + n
a = "2017-09-0" + str(date) + " " + t
start = json.index(a)
wind = (json.find("speed", start)) + 7
speed = float(json[wind:(json.find(",", wind))])
#print speed
return speed
def get_pressure(json, n=0, t="03:00:00"):
now = datetime.datetime.now()
date = now.day + n
a = "2017-09-0" + str(date) + " " + t
start = json.index(a)
pressure = (json.find("pressure", start)) + 10
pres = float(json[pressure:(json.find(",", pressure))])
#print pres
return pres
def get_humidity(json, n=0, t="03:00:00"):
now = datetime.datetime.now()
date = now.day + n
a = "2017-09-0" + str(date) + " " + t
start = json.index(a)
humidity = (json.find("humid", start)) + 10
humid = json[humidity:(json.find(",", humidity))]
humid = float(humid)
#print humid
return humid
def parsePageContent(data):
newDict = {}
for json in data:
if json.find('{"status":') != -1:
tempDict = eval(json)
newDict['status'] = tempDict.get('status')
elif json.find('{"content":') != -1:
tempDict = eval(json)
newDict['content'] = tempDict.get('content')
return newDict
def get_sealevel(json, n=0, t="03:00:00"):
now = datetime.datetime.now()
date = now.day + n
a = "2017-09-0" + str(date) + " " + t
start = json.index(a)
sea = (json.find("sea_level", start)) + 11
seal = float(json[sea:(json.find(",", sea))])
#print seal
return seal
#get_sealevel(a,2,"09:00:00")
def isTrendyTag(tag, json):
for service in json.find('hashtags'):
for t in service.find('hashtags').findall("tag"):
print(service.get("value"))
if service.get("value") == tag:
print("TAG : " + tag)
return True
def process(outfd, file_to_send):
global DBNAME
if DBNAME != None:
return process_sql(outfd, file_to_send)
f = open(file_to_send, "r")
file_contents = f.read()
f.close()
files = [("file", os.path.basename(file_to_send), file_contents)]
fields = [("uploadedFile", file_to_send),
("filename", os.path.basename(file_to_send))]
json = post_multipart(host, selector, fields, files)
if json.find("suspiciousfile") == -1:
print "Unable to submit sample"
print json
return -1
jsondict = simplejson.loads(json)
for key, val in jsondict.items():
outfd.write("*" * 72 + "\n")
outfd.write("{0:30} {1}\n".format("Filename:",
os.path.basename(file_to_send)))
outfd.write("{0:30} {1}\n".format("md5:", val['md5']))
outfd.write("{0:30} {1}\n".format("sha1:", val['sha1']))
outfd.write("{0:30} {1}\n".format("Start Time:", val['starttime']))
outfd.write("{0:30} {1}\n".format("End Time:", val['endtime']))
outfd.write("{0:30} {1}\n".format("Final Result:", val['finalresult']))
if val['finalthreatfound'] == "":
val['finalthreatfound'] = "None"
outfd.write("{0:30} {1}\n".format("Final Threat Found:",
val['finalthreatfound']))
outfd.write("*" * 72 + "\n")
if val['finalthreatfound'] == "None":
continue
for item in val['avresults']:
outfd.write("{0:30} {1}\n".format(
"AV Name:", item['avname'] + " " + item['avversion'] + " " +
item['avdefversiondate']))
if item['scanresult'] == 'Failed':
item['scanresult'] = item['scanresult'].upper()
outfd.write("{0:30} {1}\n".format("Scan Result:",
item['scanresult']))
if item['threatsfound'] == "":
item['threatsfound'] = "None"
else:
item['threatsfound'] += " [!!]"
outfd.write("{0:30} {1}\n".format("Threats Found:",
item['threatsfound']))
outfd.write("{0:30} {1}\n".format(
"AV DefSignature/DefVersion:",
item['avdefsignature'] + "/" + item['avdefversion']))
outfd.write("-" * 72 + "\n")
outfd.write("\n\n")
return 0
def has_error(loca, json):
cityin = json.index("name") + 7
city = json[cityin:((json.find(",", -57)) - 1)]
#print city
if (loca.lower() == city.lower()):
#print False
return False
else:
#print True
return True
def get_temperature(json, n=0, t="03:00:00"):
now = datetime.datetime.now()
date = now.day + n
a = "2017-09-0" + str(date) + " " + t
start = json.index(a)
temperature = (json.find("temp", start)) + 6
temp = json[temperature:(json.index(",", temperature))]
temp = float(temp)
#print temp
return temp
def extract_json_value(key, json):
key = "\"" + key + "\":"
idx = json.find(key)
val = None
if idx >= 0:
val = json[idx + len(key):]
val = val.strip(" ")
val = val[1:] # remove opening quote
idx = val.find("\"")
val = val[0:idx]
return val
def simplifyJSON(self, json):
"""Takes json data from vision AI and shortens to important information
Arguments:
json: The JSON data straight out of Vision AI API
Returns:
Just the textual data found in the image without the necessary locations
"""
json = str(json)
data = ""
index = json.find("description:") + 12
final = json.find("bounding")
while json[index] != '\"':
index += 1
index += 1
while json[index] != '\"' and index < final:
data += json[index]
index += 1
return data
def process(outfd, file_to_send):
global DBNAME
if DBNAME != None:
return process_sql(outfd, file_to_send)
f = open(file_to_send, "r")
file_contents = f.read()
f.close()
files = [("file", os.path.basename(file_to_send), file_contents)]
fields = [("uploadedFile", file_to_send), ("filename", os.path.basename(file_to_send))]
json = post_multipart(host, selector, fields, files)
if json.find("suspiciousfile") == -1:
print "Unable to submit sample"
print json
return -1
jsondict = simplejson.loads(json)
for key, val in jsondict.items():
outfd.write("*" * 72 + "\n")
outfd.write("{0:30} {1}\n".format("Filename:", os.path.basename(file_to_send)))
outfd.write("{0:30} {1}\n".format("md5:", val['md5']))
outfd.write("{0:30} {1}\n".format("sha1:", val['sha1']))
outfd.write("{0:30} {1}\n".format("Start Time:", val['starttime']))
outfd.write("{0:30} {1}\n".format("End Time:", val['endtime']))
outfd.write("{0:30} {1}\n".format("Final Result:", val['finalresult']))
if val['finalthreatfound'] == "":
val['finalthreatfound'] = "None"
outfd.write("{0:30} {1}\n".format("Final Threat Found:", val['finalthreatfound']))
outfd.write("*" * 72 + "\n")
if val['finalthreatfound'] == "None":
continue
for item in val['avresults']:
outfd.write("{0:30} {1}\n".format("AV Name:", item['avname'] + " " + item['avversion'] + " " + item['avdefversiondate']))
if item['scanresult'] == 'Failed':
item['scanresult'] = item['scanresult'].upper()
outfd.write("{0:30} {1}\n".format("Scan Result:", item['scanresult']))
if item['threatsfound'] == "":
item['threatsfound'] = "None"
else:
item['threatsfound'] += " [!!]"
outfd.write("{0:30} {1}\n".format("Threats Found:", item['threatsfound']))
outfd.write("{0:30} {1}\n".format("AV DefSignature/DefVersion:", item['avdefsignature'] + "/" + item['avdefversion']))
outfd.write("-" * 72 + "\n")
outfd.write("\n\n")
return 0
def filterIn(json: str) -> bool:
return json.find("msgType_") > 0 and not json.find('"msgType_":11') > 0
def process_sql(outfd, file_to_send):
global DBNAME
conn = sqlite3.connect(DBNAME)
cur = conn.cursor()
f = open(file_to_send, "r")
file_contents = f.read()
f.close()
files = [("file", os.path.basename(file_to_send), file_contents)]
fields = [("uploadedFile", file_to_send), ("filename", os.path.basename(file_to_send))]
json = post_multipart(host, selector, fields, files)
if json.find("suspiciousfile") == -1:
print "Unable to submit sample"
print json
conn.close()
return -1
jsondict = simplejson.loads(json)
for key, val in jsondict.items():
outfd.write("*" * 72 + "\n")
outfd.write("{0:30} {1}\n".format("Filename:", os.path.basename(file_to_send)))
outfd.write("{0:30} {1}\n".format("Final Result:", val['finalresult']))
if val['finalthreatfound'] == "":
val['finalthreatfound'] = "None"
outfd.write("{0:30} {1}\n".format("Final Threat Found:", val['finalthreatfound']))
cur.execute("SELECT COUNT(*) FROM opswat WHERE md5 = ?", [val['md5']])
count = cur.fetchone()[0]
if count > 0:
outfd.write("Sample {0} already exists in DB... not dumped\n".format(val['md5']))
outfd.write("*" * 72 + "\n")
continue
outfd.write("*" * 72 + "\n")
cur.execute("INSERT INTO opswat VALUES(null, ?,?,?,?,?,?,?)", ("", os.path.basename(file_to_send),
val['md5'], val['sha1'], val['starttime'], val['endtime'], val['finalthreatfound']))
conn.commit()
cur.execute("SELECT id FROM opswat WHERE md5 = ?", [val['md5']])
id = cur.fetchone()[0]
if val['finalthreatfound'] == "None":
continue
cur.execute("SELECT count(*) FROM artifacts WHERE file LIKE ?", [os.path.basename(file_to_send)])
count = cur.fetchone()[0]
if count == 0:
try:
if os.path.basename(file_to_send).find(".exe") != -1 and q[0].find(".dmp") == -1:
cur.execute("SELECT pname FROM procdump WHERE dump_file = ?", [os.path.basename(file_to_send)])
artifact = "Executable: " + cur.fetchone()[0]
elif os.path.basename(file_to_send).find(".dll") != -1:
cur.execute("SELECT path FROM dlldump WHERE dump_file = ?", [os.path.basename(file_to_send)])
artifact = "DLL: " + cur.fetchone()[0]
elif os.path.basename(file_to_send).find(".dmp") != -1:
cur.execute("SELECT pname FROM vaddump WHERE dump_file like ?", [os.path.basename(file_to_send)])
artifact = "Vaddump from process: " + cur.fetchone()[0]
else:
cur.execute("SELECT name FROM moddump WHERE dump_file = ?", [os.path.basename(file_to_send)])
artifact = "Module: " + cur.fetchone()[0]
cur.execute("INSERT INTO artifacts VALUES(null,?,?,?)", (artifact, os.path.basename(file_to_send), "opswat"))
conn.commit()
except:
pass
for item in val['avresults']:
if item['scanresult'] == 'Failed':
item['scanresult'] = item['scanresult'].upper()
if item['threatsfound'] == "":
item['threatsfound'] = "None"
cur.execute("INSERT INTO opswat_avscans VALUES(null, ?,?,?,?,?,?,?,?)", (id, item['avname'], item['avversion'], item['avdefversiondate'],
item['scanresult'], item['threatsfound'], item['avdefsignature'], item['avdefversion']))
conn.commit()
outfd.write("*" * 72 + "\n\n")
conn.close()
return 0
def process_sql(outfd, file_to_send):
global DBNAME
conn = sqlite3.connect(DBNAME)
cur = conn.cursor()
f = open(file_to_send, "r")
file_contents = f.read()
f.close()
files = [("file", os.path.basename(file_to_send), file_contents)]
fields = [("uploadedFile", file_to_send),
("filename", os.path.basename(file_to_send))]
json = post_multipart(host, selector, fields, files)
if json.find("suspiciousfile") == -1:
print "Unable to submit sample"
print json
conn.close()
return -1
jsondict = simplejson.loads(json)
for key, val in jsondict.items():
outfd.write("*" * 72 + "\n")
outfd.write("{0:30} {1}\n".format("Filename:",
os.path.basename(file_to_send)))
outfd.write("{0:30} {1}\n".format("Final Result:", val['finalresult']))
if val['finalthreatfound'] == "":
val['finalthreatfound'] = "None"
outfd.write("{0:30} {1}\n".format("Final Threat Found:",
val['finalthreatfound']))
cur.execute("SELECT COUNT(*) FROM opswat WHERE md5 = ?", [val['md5']])
count = cur.fetchone()[0]
if count > 0:
outfd.write(
"Sample {0} already exists in DB... not dumped\n".format(
val['md5']))
outfd.write("*" * 72 + "\n")
continue
outfd.write("*" * 72 + "\n")
cur.execute(
"INSERT INTO opswat VALUES(null, ?,?,?,?,?,?,?)",
("", os.path.basename(file_to_send), val['md5'], val['sha1'],
val['starttime'], val['endtime'], val['finalthreatfound']))
conn.commit()
cur.execute("SELECT id FROM opswat WHERE md5 = ?", [val['md5']])
id = cur.fetchone()[0]
if val['finalthreatfound'] == "None":
continue
cur.execute("SELECT count(*) FROM artifacts WHERE file LIKE ?",
[os.path.basename(file_to_send)])
count = cur.fetchone()[0]
if count == 0:
try:
if os.path.basename(file_to_send).find(
".exe") != -1 and q[0].find(".dmp") == -1:
cur.execute(
"SELECT pname FROM procdump WHERE dump_file = ?",
[os.path.basename(file_to_send)])
artifact = "Executable: " + cur.fetchone()[0]
elif os.path.basename(file_to_send).find(".dll") != -1:
cur.execute("SELECT path FROM dlldump WHERE dump_file = ?",
[os.path.basename(file_to_send)])
artifact = "DLL: " + cur.fetchone()[0]
elif os.path.basename(file_to_send).find(".dmp") != -1:
cur.execute(
"SELECT pname FROM vaddump WHERE dump_file like ?",
[os.path.basename(file_to_send)])
artifact = "Vaddump from process: " + cur.fetchone()[0]
else:
cur.execute("SELECT name FROM moddump WHERE dump_file = ?",
[os.path.basename(file_to_send)])
artifact = "Module: " + cur.fetchone()[0]
cur.execute(
"INSERT INTO artifacts VALUES(null,?,?,?)",
(artifact, os.path.basename(file_to_send), "opswat"))
conn.commit()
except:
pass
for item in val['avresults']:
if item['scanresult'] == 'Failed':
item['scanresult'] = item['scanresult'].upper()
if item['threatsfound'] == "":
item['threatsfound'] = "None"
cur.execute(
"INSERT INTO opswat_avscans VALUES(null, ?,?,?,?,?,?,?,?)",
(id, item['avname'], item['avversion'],
item['avdefversiondate'], item['scanresult'],
item['threatsfound'], item['avdefsignature'],
item['avdefversion']))
conn.commit()
outfd.write("*" * 72 + "\n\n")
conn.close()
return 0
def LocalJsonHandler(self, handler, is_2d=False, json_version=1):
"""Handle GET request for JSON file for plugin."""
if not handler.IsValidRequest():
raise tornado.web.HTTPError(404)
current_globe = ""
globe_request_name = self.ParseGlobeReqName(handler.request.uri)
if globe_request_name != -1 and globe_request_name != 1:
# Requested globe name is valid, so select it
current_globe = tornado.web.globe_.GlobeName()
globe_path = "%s%s%s" % (tornado.web.globe_.GlobeBaseDirectory(),
os.sep, globe_request_name)
tornado.web.globe_.ServeGlobe(globe_path)
# Get to end of serverUrl so we can add globe name.
# This will fail if serverDefs are requested for a glc file
try:
if is_2d:
# TODO: Add real layer support for mbtiles.
if tornado.web.globe_.IsMbtiles():
json = MBTILES_JSON
else:
# Portable seems to believe that 2D files are 3D when they
# are not actively being viewed by a client, so handle
# both possibilities in either case.
try:
json = tornado.web.globe_.ReadFile("maps/map.json")
except:
json = tornado.web.globe_.ReadFile("earth/earth.json")
else:
try:
json = tornado.web.globe_.ReadFile("earth/earth.json")
except:
json = tornado.web.globe_.ReadFile("maps/map.json")
except:
handler.write("var geeServerDefs = {};")
return
host = handler.request.host
json = json.replace("localhost:9335", host)
json = json.replace(" : ", ": ")
WHITE_SPACE_ALLOWED = 3
index0 = json.find("serverUrl")
if index0 == -1:
print "Non-standard 2d map json."
handler.write(json)
return
else:
index0 += 9
index1 = json.find(":", index0)
if index1 == -1 or index1 > index0 + WHITE_SPACE_ALLOWED:
print "Non-standard 2d map json."
handler.write(json)
return
else:
index1 += 1
index2 = json.find('"', index1)
if index2 == -1 or index2 > index1 + WHITE_SPACE_ALLOWED:
print "Non-standard 2d map json."
handler.write(json)
return
else:
index2 += 1
index3 = json.find('"', index2)
if index3 == -1:
print "Non-standard 2d map json."
handler.write(json)
return
json_start = json[:index3].strip()
json_end = json[index3:]
# Get rid of the end of structure, so we can add to it.
json_end = json_end[:json_end.rfind("}")].strip()
# If not from a remote server, show available globes.
if handler.IsLocalhost():
# Add information about available globes
json_end = (("%s,\n\"selectedGlobe\": \"%s\"\n, "
"\"globes\": [\"%%s\"]};") %
(json_end, tornado.web.globe_.GlobeName()))
json_end %= "\", \"".join(
portable_web_interface.SetUpHandler.GlobeNameList(
tornado.web.globe_.GlobeBaseDirectory(),
[".glc", ".glb", ".glm"]))
else:
json_end += "};"
# Adding globe name helps ensure clearing of cache for new globes.
json_text = "%s/%s%s" % (json_start,
tornado.web.globe_.GlobeShortName(), json_end)
if json_version == 2:
json_text = self.JStoJson(json_text)
handler.write(json_text)
# If we switched globes, switch back
if len(current_globe):
globe_path = "%s%s%s" % (tornado.web.globe_.GlobeBaseDirectory(),
os.sep, globe_request_name)
tornado.web.globe_.ServeGlobe(globe_path)
def fix_json(self, json):
"""remove slashes since they can break download process"""
json = json.replace('\'', '"')
sep_index = json.find('<!>')
json = json[:sep_index]
return json