def get(self, file): if 'restrict' in self.application.restrict_source_files.keys(): if 'exempt' not in self.application.restrict_source_files.keys() or \ file not in self.application.restrict_source_files['exempt']: srx = r'^/files/\.data/.*' if not re.match(srx, self.request.uri): if self.application.do_get_status( self ) in self.application.restrict_source_files[ 'restrict']: res = Result() res.result = 403 self.set_status(res.result) self.finish(res.emit()) return url = "%s%s" % (buildServerURL(), self.request.uri) if DEBUG: print url r = requests.get(url, verify=False) self.set_header("Content-Type", r.headers['content-type']) self.finish(r.content)
def get_task_pool(self): try: return json.loads(requests.get("%s/get_task_pool/" % buildServerURL()).content)['data'] except Exception as e: print e, type(e) return None
def do_open_drive_file(self, handler): if DEBUG: print "opening this drive file in unveillance annex" status = self.do_get_status(handler) if status not in [2,3]: if DEBUG: print "NO-ACCESS TO THIS METHOD (\"do_open_drive_file\")" return None files = None self.initDriveClient(handler) for _id in parseRequestEntity(handler.request.query)['_ids']: _id = urllib.unquote(_id).replace("'", "")[1:] file_name = self.drive_client.getFileName(_id) if file_name is None: return None url = "%s/documents/?file_name=%s" % (buildServerURL(), file_name) entry = None handled_file = None if DEBUG: print url # look up the file in annex. (annex/documents/?file_name=file) # if this file exists in annex, return its _id for opening in-app try: entry = json.loads(requests.get( url, verify=False).content)['data']['documents'][0] except Exception as e: if DEBUG: print "COULD NOT GET ENTRY:\n%s" % e if entry is not None: if DEBUG: print type(entry['_id']) handled_file = { '_id' : entry['_id'] } else: if status != 3: if DEBUG: print "** at this point, we would process file if you were admin" print "** but you are not admin." return None entry = self.drive_client.download(_id, save_as=file_name, save=False, return_content=True) if entry is not None: self.addToNetcatQueue({ 'file' : entry[0], 'save_as' : entry[1], 'importer_source' : "google_drive" }) handled_file = { 'file_name' : entry[1] } if handled_file is not None: if files is None: files = [] files.append(handled_file) return files
def get(self, file): url = "%s%s" % (buildServerURL(), self.request.uri) if DEBUG: print url r = requests.get(url, verify=False) self.set_header("Content-Type", r.headers['content-type']) ''' self.set_header("Content-Disposition", 'attachment; filename="%s"' % self.request.uri.split('/')[-1]) ''' self.finish(r.content)
def getDocWrapper(self,param): doc = J3mCache.getWrapFromCache(param) if doc is None : url = "%s%s%s" % (buildServerURL(),"/documents/?_id=" ,param) if DEBUG: print str(self) +"SENDING REQUEST TO %s" % url http_client = AsyncHTTPClient() future = http_client.fetch(url) J3mCache.putWrapInCache(param,future) response = yield future J3mCache.putWrapInCache(param,response) raise gen.Return(response.body) elif isinstance(doc, Future) : if DEBUG: print str(self) +"is future" response = yield doc J3mCache.putWrapInCache(param,response) raise gen.Return(response.body) else : if DEBUG: print str(self) +"got otherwise: " + str(doc) raise gen.Return(doc.body)
def checkForDuplicate(self, _id, from_file=None): if from_file is not None and _id is None: _id = self.get_new_hash(file) url = "%s/documents/?_id=%s" % (buildServerURL(), _id) try: r = requests.get(url, verify=False) except Exception as e: if DEBUG: print e return None try: r = json.loads(r.content) if 'data' in r.keys(): return r['data'] except Exception as e: if DEBUG: print e return None
def post(self): res = Result() res.result = 412 query = parseRequestEntity(self.request.body) if query is None or len(query.keys()) != 1 or '_id' not in query.keys(): self.set_status(res.result) self.finish(res.emit()) return r = requests.post("%stask/" % buildServerURL(), data={ '_id' : query['_id'] }, verify=False) try: res.data = json.loads(r.content)['data'] res.result = 200 except Exception as e: if DEBUG: print e self.set_status(res.result) self.finish(res.emit())
def getJ3mDoc(self,param): j3mDoc = J3mCache.getJ3mFromCache(param) if j3mDoc is None : handle = yield getDocWrapper(self,param) self.objectHandle = json_decode(handle) url = "%s%s%s%s%s" % (buildServerURL(),"/documents/?doc_type=ic_j3m&_id=" ,self.objectHandle['data']['j3m_id'], '&media_id=', self.objectHandle['data']['_id']) if DEBUG: print str(self) +"SENDING REQUEST TO %s" % url http_client = AsyncHTTPClient() future = http_client.fetch(url) J3mCache.putJ3mInCache(param,future) response = yield future raise gen.Return(response.body) elif isinstance(j3mDoc, Future) : if DEBUG: print str(self) + "is j3mDoc future" response = yield j3mDoc J3mCache.putJ3mInCache(param,response) raise gen.Return(response.body) else : if DEBUG: print str(self) +"got j3mDoc otherwise" raise gen.Return(j3mDoc.body)
def passToAnnex(self, handler, uri=None): handler_status = self.do_get_status(handler) if handler.request.body != "": ref = "?%s" % handler.request.body else: ref = handler.request.headers['Referer'] query = "" try: query += ref[ref.index("?"):] except ValueError as e: pass print self.sanitizeQuery(query, handler_status) if uri is None: uri = handler.request.uri url = "%s%s%s" % (buildServerURL(), uri, self.sanitizeQuery(query, handler_status)) # TODO: verify=False ... hmm.... no. if DEBUG: print "SENDING REQUEST TO %s" % url try: r = self.uv_cache.get(url, verify=False) except Exception as e: if DEBUG: print e if type(e) is ConnectionError: if DEBUG: print type(e.message) return {'result': 404, 'error': e.message[0]} try: return json.loads(r.content)['data'] except Exception as e: if DEBUG: print e return None
def post(self): res = Result() res.result = 412 query = parseRequestEntity(self.request.body) if query is None or len( query.keys()) != 1 or '_id' not in query.keys(): self.set_status(res.result) self.finish(res.emit()) return r = requests.post("%stask/" % buildServerURL(), data={'_id': query['_id']}, verify=False) try: res.data = json.loads(r.content)['data'] res.result = 200 except Exception as e: if DEBUG: print e self.set_status(res.result) self.finish(res.emit())
def passToAnnex(self, handler, uri=None): handler_status = self.do_get_status(handler) if handler.request.body != "": ref = "?%s" % handler.request.body else: ref = handler.request.headers['Referer'] query = "" try: query += ref[ref.index("?"):] except ValueError as e: pass print self.sanitizeQuery(query, handler_status) if uri is None: uri = handler.request.uri url = "%s%s%s" % (buildServerURL(), uri, self.sanitizeQuery(query, handler_status)) # TODO: verify=False ... hmm.... no. if DEBUG: print "SENDING REQUEST TO %s" % url try: r = self.uv_cache.get(url, verify=False) except Exception as e: if DEBUG: print e if type(e) is ConnectionError: if DEBUG: print type(e.message) return { 'result' : 404, 'error' : e.message[0] } try: return json.loads(r.content)['data'] except Exception as e: if DEBUG: print e return None
def get(self, file): if 'restrict' in self.application.restrict_source_files.keys(): if 'exempt' not in self.application.restrict_source_files.keys() or \ file not in self.application.restrict_source_files['exempt']: srx = r'^/files/\.data/.*' if not re.match(srx, self.request.uri): if self.application.do_get_status(self) in self.application.restrict_source_files['restrict']: res = Result() res.result = 403 self.set_status(res.result) self.finish(res.emit()) return url = "%s%s" % (buildServerURL(), self.request.uri) if DEBUG: print url r = requests.get(url, verify=False) self.set_header("Content-Type", r.headers['content-type']) self.finish(r.content)
def do_open_drive_file(self, handler): if DEBUG: print "opening this drive file in unveillance annex" status = self.do_get_status(handler) if status not in [2, 3]: if DEBUG: print "NO-ACCESS TO THIS METHOD (\"do_open_drive_file\")" return None files = None self.initDriveClient(handler) for _id in parseRequestEntity(handler.request.query)['_ids']: _id = urllib.unquote(_id).replace("'", "")[1:] file_name = self.drive_client.getFileName(_id) if file_name is None: return None url = "%s/documents/?file_name=%s" % (buildServerURL(), file_name) entry = None handled_file = None if DEBUG: print url # look up the file in annex. (annex/documents/?file_name=file) # if this file exists in annex, return its _id for opening in-app try: entry = json.loads(requests.get( url, verify=False).content)['data']['documents'][0] except Exception as e: if DEBUG: print "COULD NOT GET ENTRY:\n%s" % e if entry is not None: if DEBUG: print type(entry['_id']) handled_file = {'_id': entry['_id']} else: if status != 3: if DEBUG: print "** at this point, we would process file if you were admin" print "** but you are not admin." return None entry = self.drive_client.download(_id, save_as=file_name, save=False, return_content=True) if entry is not None: self.addToNetcatQueue({ 'file': entry[0], 'save_as': entry[1], 'importer_source': "google_drive" }) handled_file = {'file_name': entry[1]} if handled_file is not None: if files is None: files = [] files.append(handled_file) return files