def get(self, file):
if 'restrict' in self.application.restrict_source_files.keys():
if 'exempt' not in self.application.restrict_source_files.keys() or \
file not in self.application.restrict_source_files['exempt']:
srx = r'^/files/\.data/.*'
if not re.match(srx, self.request.uri):
if self.application.do_get_status(
self
) in self.application.restrict_source_files[
'restrict']:
res = Result()
res.result = 403
self.set_status(res.result)
self.finish(res.emit())
return
url = "%s%s" % (buildServerURL(), self.request.uri)
if DEBUG:
print url
r = requests.get(url, verify=False)
self.set_header("Content-Type", r.headers['content-type'])
self.finish(r.content)
def get_task_pool(self):
try:
return json.loads(requests.get("%s/get_task_pool/" % buildServerURL()).content)['data']
except Exception as e:
print e, type(e)
return None
def do_open_drive_file(self, handler):
if DEBUG: print "opening this drive file in unveillance annex"
status = self.do_get_status(handler)
if status not in [2,3]:
if DEBUG: print "NO-ACCESS TO THIS METHOD (\"do_open_drive_file\")"
return None
files = None
self.initDriveClient(handler)
for _id in parseRequestEntity(handler.request.query)['_ids']:
_id = urllib.unquote(_id).replace("'", "")[1:]
file_name = self.drive_client.getFileName(_id)
if file_name is None: return None
url = "%s/documents/?file_name=%s" % (buildServerURL(), file_name)
entry = None
handled_file = None
if DEBUG: print url
# look up the file in annex. (annex/documents/?file_name=file)
# if this file exists in annex, return its _id for opening in-app
try:
entry = json.loads(requests.get(
url, verify=False).content)['data']['documents'][0]
except Exception as e:
if DEBUG: print "COULD NOT GET ENTRY:\n%s" % e
if entry is not None:
if DEBUG: print type(entry['_id'])
handled_file = { '_id' : entry['_id'] }
else:
if status != 3:
if DEBUG:
print "** at this point, we would process file if you were admin"
print "** but you are not admin."
return None
entry = self.drive_client.download(_id,
save_as=file_name, save=False, return_content=True)
if entry is not None:
self.addToNetcatQueue({
'file' : entry[0],
'save_as' : entry[1],
'importer_source' : "google_drive"
})
handled_file = { 'file_name' : entry[1] }
if handled_file is not None:
if files is None: files = []
files.append(handled_file)
return files
def get(self, file):
url = "%s%s" % (buildServerURL(), self.request.uri)
if DEBUG:
print url
r = requests.get(url, verify=False)
self.set_header("Content-Type", r.headers['content-type'])
'''
self.set_header("Content-Disposition",
'attachment; filename="%s"' % self.request.uri.split('/')[-1])
'''
self.finish(r.content)
def getDocWrapper(self,param):
doc = J3mCache.getWrapFromCache(param)
if doc is None :
url = "%s%s%s" % (buildServerURL(),"/documents/?_id=" ,param)
if DEBUG: print str(self) +"SENDING REQUEST TO %s" % url
http_client = AsyncHTTPClient()
future = http_client.fetch(url)
J3mCache.putWrapInCache(param,future)
response = yield future
J3mCache.putWrapInCache(param,response)
raise gen.Return(response.body)
elif isinstance(doc, Future) :
if DEBUG: print str(self) +"is future"
response = yield doc
J3mCache.putWrapInCache(param,response)
raise gen.Return(response.body)
else :
if DEBUG: print str(self) +"got otherwise: " + str(doc)
raise gen.Return(doc.body)
def checkForDuplicate(self, _id, from_file=None): if from_file is not None and _id is None: _id = self.get_new_hash(file) url = "%s/documents/?_id=%s" % (buildServerURL(), _id) try: r = requests.get(url, verify=False) except Exception as e: if DEBUG: print e return None try: r = json.loads(r.content) if 'data' in r.keys(): return r['data'] except Exception as e: if DEBUG: print e return None
def post(self):
res = Result()
res.result = 412
query = parseRequestEntity(self.request.body)
if query is None or len(query.keys()) != 1 or '_id' not in query.keys():
self.set_status(res.result)
self.finish(res.emit())
return
r = requests.post("%stask/" % buildServerURL(),
data={ '_id' : query['_id'] }, verify=False)
try:
res.data = json.loads(r.content)['data']
res.result = 200
except Exception as e:
if DEBUG: print e
self.set_status(res.result)
self.finish(res.emit())
def getJ3mDoc(self,param):
j3mDoc = J3mCache.getJ3mFromCache(param)
if j3mDoc is None :
handle = yield getDocWrapper(self,param)
self.objectHandle = json_decode(handle)
url = "%s%s%s%s%s" % (buildServerURL(),"/documents/?doc_type=ic_j3m&_id=" ,self.objectHandle['data']['j3m_id'], '&media_id=', self.objectHandle['data']['_id'])
if DEBUG: print str(self) +"SENDING REQUEST TO %s" % url
http_client = AsyncHTTPClient()
future = http_client.fetch(url)
J3mCache.putJ3mInCache(param,future)
response = yield future
raise gen.Return(response.body)
elif isinstance(j3mDoc, Future) :
if DEBUG: print str(self) + "is j3mDoc future"
response = yield j3mDoc
J3mCache.putJ3mInCache(param,response)
raise gen.Return(response.body)
else :
if DEBUG: print str(self) +"got j3mDoc otherwise"
raise gen.Return(j3mDoc.body)
def checkForDuplicate(self, _id, from_file=None):
if from_file is not None and _id is None:
_id = self.get_new_hash(file)
url = "%s/documents/?_id=%s" % (buildServerURL(), _id)
try:
r = requests.get(url, verify=False)
except Exception as e:
if DEBUG:
print e
return None
try:
r = json.loads(r.content)
if 'data' in r.keys():
return r['data']
except Exception as e:
if DEBUG:
print e
return None
def passToAnnex(self, handler, uri=None):
handler_status = self.do_get_status(handler)
if handler.request.body != "":
ref = "?%s" % handler.request.body
else:
ref = handler.request.headers['Referer']
query = ""
try:
query += ref[ref.index("?"):]
except ValueError as e:
pass
print self.sanitizeQuery(query, handler_status)
if uri is None: uri = handler.request.uri
url = "%s%s%s" % (buildServerURL(), uri,
self.sanitizeQuery(query, handler_status))
# TODO: verify=False ... hmm.... no.
if DEBUG:
print "SENDING REQUEST TO %s" % url
try:
r = self.uv_cache.get(url, verify=False)
except Exception as e:
if DEBUG: print e
if type(e) is ConnectionError:
if DEBUG: print type(e.message)
return {'result': 404, 'error': e.message[0]}
try:
return json.loads(r.content)['data']
except Exception as e:
if DEBUG: print e
return None
def post(self):
res = Result()
res.result = 412
query = parseRequestEntity(self.request.body)
if query is None or len(
query.keys()) != 1 or '_id' not in query.keys():
self.set_status(res.result)
self.finish(res.emit())
return
r = requests.post("%stask/" % buildServerURL(),
data={'_id': query['_id']},
verify=False)
try:
res.data = json.loads(r.content)['data']
res.result = 200
except Exception as e:
if DEBUG: print e
self.set_status(res.result)
self.finish(res.emit())
def passToAnnex(self, handler, uri=None):
handler_status = self.do_get_status(handler)
if handler.request.body != "":
ref = "?%s" % handler.request.body
else:
ref = handler.request.headers['Referer']
query = ""
try:
query += ref[ref.index("?"):]
except ValueError as e: pass
print self.sanitizeQuery(query, handler_status)
if uri is None: uri = handler.request.uri
url = "%s%s%s" % (buildServerURL(), uri, self.sanitizeQuery(query, handler_status))
# TODO: verify=False ... hmm.... no.
if DEBUG:
print "SENDING REQUEST TO %s" % url
try:
r = self.uv_cache.get(url, verify=False)
except Exception as e:
if DEBUG: print e
if type(e) is ConnectionError:
if DEBUG: print type(e.message)
return { 'result' : 404, 'error' : e.message[0] }
try:
return json.loads(r.content)['data']
except Exception as e:
if DEBUG: print e
return None
def get(self, file):
if 'restrict' in self.application.restrict_source_files.keys():
if 'exempt' not in self.application.restrict_source_files.keys() or \
file not in self.application.restrict_source_files['exempt']:
srx = r'^/files/\.data/.*'
if not re.match(srx, self.request.uri):
if self.application.do_get_status(self) in self.application.restrict_source_files['restrict']:
res = Result()
res.result = 403
self.set_status(res.result)
self.finish(res.emit())
return
url = "%s%s" % (buildServerURL(), self.request.uri)
if DEBUG:
print url
r = requests.get(url, verify=False)
self.set_header("Content-Type", r.headers['content-type'])
self.finish(r.content)
def do_open_drive_file(self, handler):
if DEBUG: print "opening this drive file in unveillance annex"
status = self.do_get_status(handler)
if status not in [2, 3]:
if DEBUG: print "NO-ACCESS TO THIS METHOD (\"do_open_drive_file\")"
return None
files = None
self.initDriveClient(handler)
for _id in parseRequestEntity(handler.request.query)['_ids']:
_id = urllib.unquote(_id).replace("'", "")[1:]
file_name = self.drive_client.getFileName(_id)
if file_name is None: return None
url = "%s/documents/?file_name=%s" % (buildServerURL(), file_name)
entry = None
handled_file = None
if DEBUG: print url
# look up the file in annex. (annex/documents/?file_name=file)
# if this file exists in annex, return its _id for opening in-app
try:
entry = json.loads(requests.get(
url, verify=False).content)['data']['documents'][0]
except Exception as e:
if DEBUG: print "COULD NOT GET ENTRY:\n%s" % e
if entry is not None:
if DEBUG: print type(entry['_id'])
handled_file = {'_id': entry['_id']}
else:
if status != 3:
if DEBUG:
print "** at this point, we would process file if you were admin"
print "** but you are not admin."
return None
entry = self.drive_client.download(_id,
save_as=file_name,
save=False,
return_content=True)
if entry is not None:
self.addToNetcatQueue({
'file': entry[0],
'save_as': entry[1],
'importer_source': "google_drive"
})
handled_file = {'file_name': entry[1]}
if handled_file is not None:
if files is None: files = []
files.append(handled_file)
return files
