exposed = True

def GET(self):

return open(os.path.join(insiderer_dir, 'static/index.html'), 'r').read()

@cherrypy.tools.json_out()

def POST(self, **kwargs):

metadata_files = []

for key, postfiles in kwargs.items():

if not isinstance(postfiles, list):

postfiles = [postfiles]

for postfile in postfiles:

tmp_path = None

try:

try:

tmp_path = tempfile.mkstemp(dir=TMP_DIR)[1]

except IOError as e:

cherrypy.log("FATAL ERROR: Unable to write to " + TMP_DIR)

raise e

handle = open(tmp_path, 'wb')

handle.write(postfile.file.read())

handle.close()

#cherrypy.log("md")

metadata = get_metadata(tmp_path, postfile.filename)

metadata_files.append(metadata)

except Exception as e:

cherrypy.log("[insiderer.py] exception", e)

finally:

if tmp_path is not None:

safedelete(tmp_path)

exposed = True

def GET(self, mimetype):

icons_dir = "static/icons/"

icons_local_dir = os.path.join(insiderer_dir, icons_dir)

chosen_icon = "text.svg"

parts = mimetype.split("/")

parts[0] = sanitise(parts[0])

parts[1] = sanitise(parts[1])

exact_mimetype = parts[0] + "_" + parts[1] + ".svg"

basic_mimetype = parts[0] + ".svg"

if os.path.exists(os.path.join(icons_local_dir, exact_mimetype)):

raise cherrypy.HTTPRedirect(icons_dir + exact_mimetype)

elif os.path.exists(os.path.join(icons_local_dir, basic_mimetype)):

raise cherrypy.HTTPRedirect(icons_dir + basic_mimetype)

else:

exposed = True

def GET(self, path="", **kwargs):

if path == "":

return open(os.path.join(insiderer_dir, "static/tests/index.html"), 'r').read()

elif path == "list":

list = []

test_pattern = os.path.join(insiderer_dir, "tests") + "/*"

paths = glob.glob(test_pattern)

for path in paths:

if not path.endswith(".json"):

list.append(os.path.basename(path))

cherrypy.response.headers['Content-Type'] = "application/json"

return json.dumps(list).encode("utf-8")

test = {}

actual_metadata = None

tmp_path = None

try:

tmp_path = tempfile.mkstemp(dir=TMP_DIR)[1]

source = os.path.join(insiderer_dir, "tests", path)

shutil.copyfile(source, tmp_path)

actual_metadata = json.dumps(normalize(get_metadata(tmp_path, os.path.basename(path))))

finally:

if tmp_path is not None:

safedelete(tmp_path)

cherrypy.log(actual_metadata)

expected_metadata = open(os.path.join(insiderer_dir, "tests", path + ".json"), 'rb').read().decode('utf-8')

cherrypy.response.headers['Content-Type'] = "application/json"

def get_mime(path):

import magic

mimey_the_mimetype = magic.from_file(path, mime=True).decode('utf-8')

if mimey_the_mimetype == "application/octet-stream": #well that's useless

description = magic.from_file(path).decode('utf-8').lower()

if "audio " in description:

mimey_the_mimetype = "audio/mpeg"

return mimey_the_mimetype

def sha1OfFile(filepath):

sha = hashlib.sha1()

if os.path.isdir(filepath):

return None

with open(filepath, 'rb') as f:

while True:

block = f.read(2**10) # Magic number: one-megabyte blocks.

if not block: break

sha.update(block)

return sha.hexdigest()

#cherrypy.log("a1")

mimetype = get_mime(path)

#cherrypy.log("a2", mimetype)

mime_app_name = sanitise(mimetype.split(";")[0])

filedata = {}

filedata['filename'] = filename;

filedata['mimetype'] = mimetype.split(";")[0];

filedata['sha1'] = sha1OfFile(path);

filedata['filesize_bytes'] = os.path.getsize(path);

filedata['children'] = [];

mime_app = None;

import_obj = 'mimes.%s' % mime_app_name

if not os.path.exists(os.path.join(insiderer_dir, import_obj.replace('.', '/') + ".py")):

mime_app_name = sanitise(mimetype.split("/")[0])

import_obj = 'mimes.%s' % mime_app_name

if not os.path.exists(os.path.join(insiderer_dir, import_obj.replace('.', '/') + ".py")):

import_obj = None

else:

cherrypy.log(import_obj + " exists2")

else:

cherrypy.log(import_obj + " exists1")

if import_obj:

#cherrypy.log("import", import_obj)

mime_app = __import__(import_obj)

if mime_app: #if there was a handler for this mimetype

mime_app_method = getattr(mime_app, mime_app_name)

filedata['metadata'] = {}

#cherrypy.log(mime_app_name)

getattr(mime_app_method, mime_app_name)(path, filedata['metadata'], filedata['children'])

if len(filedata['children']) == 0:

del filedata['children']

if os.path.isdir(path):

cherrypy.log("Can't safedelete directory", path)

return

sector = 4096

data = ""

try:

filesize = os.path.getsize(path)

cycles = math.ceil(filesize / sector) + 1

with open(path, 'wb') as safehandle:

safehandle.seek(0)

for x in range(0, cycles):

safehandle.write(os.urandom(sector))

except Exception as e:

cherrypy.log(e)

finally:

headers = cherrypy.response.headers

headers['X-Frame-Options'] = 'SAMEORIGIN'

headers['X-XSS-Protection'] = '1; mode=block'

newobj = None

if isinstance(obj, dict):

newobj = dict()

keys = list(obj.keys())

keys.sort()

for key in keys:

newkey = str(key)

newkey = re.sub(r'[\.,_-]', ' ', newkey).replace("@", "").replace("#","").replace("/","").lower().strip()

if newkey.startswith("xmlns"):

continue

if ":" in newkey:

newkey = newkey[newkey.rfind(":") + 1:]

newkey = de_dup(newkey, newobj)

response = normalize(obj[key])

if contains_values(response):

newobj[newkey] = response

elif isinstance(obj, list):

newobj = list()

for i in range(len(obj)):

response = normalize(obj[i])

if contains_values(response):

newobj.append(response)

elif isinstance(obj, int) or obj is None:

newobj = obj

else:

newobj = str(obj).replace("\n", " ")

newobj = normalize_date(newobj)

if len(datestring) < 8: # Unfortunately dateutil.parser.parse will think strings like "319/1" are dates, so to avoid that we filter by length. If it's shorter than "YYYYMMDD" then we won't possibly normalizing it into a date.

return datestring

try:

datestring = normalize_malformed_date(datestring)

datetime = dateutil.parser.parse(datestring)

isoformat = datetime.isoformat()

if datetime.isoformat() != "1972-01-19T00:00:00":

datestring = isoformat

except Exception as e:

if datestring.startswith("D:"):

return normalize_date(datestring[2:].replace("'", ""))

if (datestring.startswith("19") or datestring.startswith("20")) and datestring.count(" ") == 1 and datestring.count(":") == 4:

# Unfortunately the dateutil parser will read "2014:09:15 14:06:02" as "2015-04-21T14:06:02" (as of today) because it doesn't like colons between year:month:day so we detect that scenario and change it

parts = datestring.split(" ")

datestring = parts[0].replace(":", "/") + " " + parts[1]

return datestring

if isinstance(obj, dict):

return len(obj.keys()) > 0

elif isinstance(obj, list):

return len(obj) > 0

# Note that key order must be sorted before using de_dup or else two keys that are "description" and "exif:description" will be be insiderer.normalize()d and de_duped to description and description2 in random order, and randomly break tests

addon = ""

while (key + str(addon)) in obj:

if addon == "":

addon = 2

else:

addon += 1