def initDriveClient(self, restart=False): if DEBUG: print "INITING DRIVE CLIENT" if not hasattr(self, "drive_client") or restart: self.drive_client = UnveillanceAnnexClient() sleep(2) return self.do_get_drive_status()
class UnveillanceAPI(): def __init__(self): print "Stock Unveillance Frontend API started..." self.UNVEILLANCE_LM_VARS = {} def do_documents(self, handler): return self.passToAnnex(handler) def do_list(self, handler): return self.passToAnnex(handler) def do_cluster(self, handler): return self.passToAnnex(handler) def do_reindex(self, handler): req = parseRequestEntity(handler.request.body) if req is None: return None if '_id' not in req.keys(): return None ''' we might have to replace parts of the request with secret values. ''' if 'task_path' in req.keys(): req_keys = list(chain.from_iterable( [k.keys() for k in TASK_REQUIREMENTS])) if req['task_path'] in req_keys: try: req_field = [k for k in TASK_REQUIREMENTS if k.keys()[0] == req['task_path']][0][req['task_path']] print req_field except Exception as e: if DEBUG: print "Could not resolve required task info. %s" % e for repl in req_field.iteritems(): if repl[0] in req.keys(): handler.request.body = handler.request.body.replace( repl[1], getSecrets(repl[1])) return self.passToAnnex(handler) def do_import(self, handler): status = self.do_get_status(handler) if status not in PERMISSIONS['upload_local']: return for s_a in handler.request.files.keys(): if s_a != "uv_import": save_as = handler.request.files[s_a][0] break try: entry = BytesIO() entry.write(save_as['body']) except Exception as e: if DEBUG: print "COULD NOT CREATE FILE FROM BLOB %s" % e return None netcat_stub = { 'save_as' : save_as['filename'], 'file' : entry, 'importer_source' : "web_frontend_status_%d" % status } if status not in PERMISSIONS['upload_global']: # add this filename to the restricted list in handler if DEBUG: print "SINCE STATUS == %d, this file will be restricted locally" % status #netcat_stub['for_local_use_only'] = True res = self.checkForDuplicate(self.get_new_hash(netcat_stub['file'])) if res is None: res = { 'file_name' : save_as['filename'], 'result' : 404, 'duplicate_attempt' : False } else: res.update({ 'result' : 200, # or?? 'duplicate_attempt' : True }) return res else: try: res = self.addToNetcatQueue(netcat_stub) if 'duplicate_attempt' not in res.keys(): res.update({ 'file_name' : save_as['filename'], 'result' : 200 if res['uploaded'] else 403 }) return res except Exception as e: if DEBUG: print "ERROR IN NETCAT: %s" % e return None def do_open_drive_file(self, handler): if DEBUG: print "opening this drive file in unveillance annex" status = self.do_get_status(handler) if status not in [2,3]: if DEBUG: print "NO-ACCESS TO THIS METHOD (\"do_open_drive_file\")" return None files = None self.initDriveClient(handler) for _id in parseRequestEntity(handler.request.query)['_ids']: _id = urllib.unquote(_id).replace("'", "")[1:] file_name = self.drive_client.getFileName(_id) if file_name is None: return None url = "%s/documents/?file_name=%s" % (buildServerURL(), file_name) entry = None handled_file = None if DEBUG: print url # look up the file in annex. (annex/documents/?file_name=file) # if this file exists in annex, return its _id for opening in-app try: entry = json.loads(requests.get( url, verify=False).content)['data']['documents'][0] except Exception as e: if DEBUG: print "COULD NOT GET ENTRY:\n%s" % e if entry is not None: if DEBUG: print type(entry['_id']) handled_file = { '_id' : entry['_id'] } else: if status != 3: if DEBUG: print "** at this point, we would process file if you were admin" print "** but you are not admin." return None entry = self.drive_client.download(_id, save_as=file_name, save=False, return_content=True) if entry is not None: self.addToNetcatQueue({ 'file' : entry[0], 'save_as' : entry[1], 'importer_source' : "google_drive" }) handled_file = { 'file_name' : entry[1] } if handled_file is not None: if files is None: files = [] files.append(handled_file) return files def do_get_admin_party_status(self, handler): status = self.do_get_status(handler) if status == 0: return None from conf import USER_ROOT for _, _, files in os.walk(USER_ROOT): for f in files: return False return True def do_create_new_user(self, handler): status = self.do_get_status(handler) if status != 3: return None new_user = parseRequestEntity(handler.request.body) if new_user is None: return None if DEBUG: print new_user n_set = sorted(['nu_username', 'nu_password']) if sorted(list(set(n_set) & set(new_user.keys()))) != n_set: return None from Utils.funcs import createNewUser return createNewUser(new_user['nu_username'], new_user['nu_password']) def do_get_user_status(self, handler): status = self.do_get_status(handler) if status == 0: return None if self.do_get_admin_party_status(handler): return 4 return status def do_get_status(self, handler): try: for cookie in handler.request.cookies: if cookie == UnveillanceCookie.PUBLIC: print "**** VISITOR OF TYPE ZERO ENCOUNTERED: %s *****" % UnveillanceCookie.PUBLIC return 0 except KeyError as e: pass access = handler.get_secure_cookie(UnveillanceCookie.USER) if access is not None: if handler.get_secure_cookie(UnveillanceCookie.ADMIN) is not None: return 3 return 2 return 1 def do_get_drive_status(self, handler=None): if handler is not None: if self.do_get_status(handler) == 0: return None # TODO: actually, if not 3 if hasattr(self, "drive_client"): if hasattr(self.drive_client, "service"): return True return False def do_logout(self, handler): status = self.do_get_status(handler) if status not in [2, 3]: if DEBUG: print "CANNOT LOG IN USER, DON'T EVEN TRY (status %d)" % status return None credentials = parseRequestEntity(handler.request.body) if credentials is None: return None if DEBUG: print credentials return self.logoutUser(credentials, handler) def do_login(self, handler): status = self.do_get_status(handler) if status != 1: if DEBUG: print "CANNOT LOG IN USER, DON'T EVEN TRY (status %d)" % status return None credentials = parseRequestEntity(handler.request.body) if credentials is None: return None if DEBUG: print credentials try: return self.loginUser(credentials['username'], credentials['password'], handler) except KeyError as e: if DEBUG: print "CANNOT LOG IN USER: %s missing" % e return None def initDriveClient(self, restart=False): if DEBUG: print "INITING DRIVE CLIENT" if not hasattr(self, "drive_client") or restart: self.drive_client = UnveillanceAnnexClient() sleep(2) return self.do_get_drive_status() def logoutUser(self, credentials, handler): handler.clear_cookie(UnveillanceCookie.USER) handler.clear_cookie(UnveillanceCookie.ADMIN) try: username = credentials['username'] except KeyError as e: return None try: password = credentials['password'] except KeyError as e: return True try: IV = getConfig('encryption.iv') SALT = getConfig('encryption.salt') USER_SALT = getConfig('encyption.user_salt') except Exception as e: if DEBUG: print e return None try: from Utils.funcs import decryptUserData, encryptUserData except Exception as e: if DEBUG: print e from lib.Frontend.Utils.funcs import decryptUserData, encryptUserData user_root = "%s.txt" % generateMD5Hash(content=username,salt=USER_SALT) with open(os.path.join(USER_ROOT, user_root), 'rb') as UD: user_data = decryptUserData(UD.read, password, p_salt=SALT) if user_data is None: return None new_data = copy.deepcopy(user_data) new_data['saved_searches'] = credentials['save_data']['saved_searches'] try: new_data['annex_key_sent'] = credentials['save_data']['annex_key_sent'] except KeyError as e: pass with open(os.path.join(USER_ROOT, user_root), 'wb+') as UD: UD.write(encryptUserData(new_data, password, iv=IV, p_salt=SALT)) return True return None def loginUser(self, username, password, handler): try: SALT = getConfig('encryption.salt') USER_SALT = getConfig('encyption.user_salt') except Exception as e: if DEBUG: print e return None from base64 import b64encode try: from Utils.funcs import decryptUserData except Exception as e: if DEBUG: print e from lib.Frontend.Utils.funcs import decryptUserData try: user_root = "%s.txt" % generateMD5Hash(content=username, salt=USER_SALT) with open(os.path.join(USER_ROOT, user_root), 'rb') as UD: user_data = decryptUserData(UD.read(), password, p_salt=SALT) if user_data is None: return None try: if user_data['admin']: del user_data['admin'] handler.set_secure_cookie(UnveillanceCookie.ADMIN, "true", path="/", expires_days=1) if not self.do_get_drive_status(): self.initDriveClient() if "annex_key_sent" not in user_data.keys() or not user_data['annex_key_sent']: if self.drive_client.checkAnnexKeyStatus(): user_data['annex_key_sent'] = True except KeyError as e: if DEBUG: print e pass handler.set_secure_cookie(UnveillanceCookie.USER, b64encode(json.dumps(user_data)), path="/", expires_days=1) return user_data except Exception as e: if DEBUG: print e return None
class UnveillanceAPI(): def __init__(self): print "Stock Unveillance Frontend API started..." self.UNVEILLANCE_LM_VARS = {} def do_documents(self, handler): return self.passToAnnex(handler) def do_list(self, handler): return self.passToAnnex(handler) def do_cluster(self, handler): return self.passToAnnex(handler) def do_reindex(self, handler): req = parseRequestEntity(handler.request.body) if req is None: return None if '_id' not in req.keys(): return None ''' we might have to replace parts of the request with secret values. ''' if 'task_path' in req.keys(): req_keys = list( chain.from_iterable( [k.keys() for k in MIME_TYPE_TASK_REQUIREMENTS])) if req['task_path'] in req_keys: try: req_field = [ k for k in MIME_TYPE_TASK_REQUIREMENTS if k.keys()[0] == req['task_path'] ][0][req['task_path']] print req_field except Exception as e: if DEBUG: print "Could not resolve required task info. %s" % e for repl in req_field.iteritems(): if repl[0] in req.keys(): handler.request.body = handler.request.body.replace( repl[1], getSecrets(repl[1])) return self.passToAnnex(handler) def do_import(self, handler): status = self.do_get_status(handler) if status not in PERMISSIONS['upload_local']: return for s_a in handler.request.files.keys(): if s_a != "uv_import": save_as = handler.request.files[s_a][0] break try: entry = BytesIO() entry.write(save_as['body']) except Exception as e: if DEBUG: print "COULD NOT CREATE FILE FROM BLOB %s" % e return None netcat_stub = { 'save_as': save_as['filename'], 'file': entry, 'importer_source': "web_frontend_status_%d" % status } if status not in PERMISSIONS['upload_global']: # add this filename to the restricted list in handler if DEBUG: print "SINCE STATUS == %d, this file will be restricted locally" % status #netcat_stub['for_local_use_only'] = True res = self.checkForDuplicate(self.get_new_hash( netcat_stub['file'])) if res is None: res = { 'file_name': save_as['filename'], 'result': 403, 'duplicate_attempt': False } else: res.update({ 'result': 200, # or?? 'duplicate_attempt': True }) return res else: try: res = self.addToNetcatQueue(netcat_stub) if 'duplicate_attempt' not in res.keys(): res.update({ 'file_name': save_as['filename'], 'result': 200 if res['uploaded'] else 403 }) return res except Exception as e: if DEBUG: print "ERROR IN NETCAT: %s" % e return None def do_open_drive_file(self, handler): if DEBUG: print "opening this drive file in unveillance annex" status = self.do_get_status(handler) if status not in [2, 3]: if DEBUG: print "NO-ACCESS TO THIS METHOD (\"do_open_drive_file\")" return None files = None self.initDriveClient(handler) for _id in parseRequestEntity(handler.request.query)['_ids']: _id = urllib.unquote(_id).replace("'", "")[1:] file_name = self.drive_client.getFileName(_id) if file_name is None: return None url = "%s/documents/?file_name=%s" % (buildServerURL(), file_name) entry = None handled_file = None if DEBUG: print url # look up the file in annex. (annex/documents/?file_name=file) # if this file exists in annex, return its _id for opening in-app try: entry = json.loads(requests.get( url, verify=False).content)['data']['documents'][0] except Exception as e: if DEBUG: print "COULD NOT GET ENTRY:\n%s" % e if entry is not None: if DEBUG: print type(entry['_id']) handled_file = {'_id': entry['_id']} else: if status != 3: if DEBUG: print "** at this point, we would process file if you were admin" print "** but you are not admin." return None entry = self.drive_client.download(_id, save_as=file_name, save=False, return_content=True) if entry is not None: self.addToNetcatQueue({ 'file': entry[0], 'save_as': entry[1], 'importer_source': "google_drive" }) handled_file = {'file_name': entry[1]} if handled_file is not None: if files is None: files = [] files.append(handled_file) return files def do_get_admin_party_status(self, handler): status = self.do_get_status(handler) if status == 0: return None from conf import USER_ROOT for _, _, files in os.walk(USER_ROOT): for f in files: return False return True def do_create_new_user(self, handler): status = self.do_get_status(handler) if status != 3: return None new_user = parseRequestEntity(handler.request.body) if new_user is None: return None if DEBUG: print new_user n_set = sorted(['nu_username', 'nu_password']) if sorted(list(set(n_set) & set(new_user.keys()))) != n_set: return None from Utils.funcs import createNewUser return createNewUser(new_user['nu_username'], new_user['nu_password']) def do_get_user_status(self, handler): status = self.do_get_status(handler) if status == 0: return None if self.do_get_admin_party_status(handler): return 4 return status def do_get_status(self, handler): try: for cookie in handler.request.cookies: if cookie == UnveillanceCookie.PUBLIC: print "**** VISITOR OF TYPE ZERO ENCOUNTERED: %s *****" % UnveillanceCookie.PUBLIC return 0 except KeyError as e: pass access = handler.get_secure_cookie(UnveillanceCookie.USER) if access is not None: if handler.get_secure_cookie(UnveillanceCookie.ADMIN) is not None: return 3 return 2 return 1 def do_get_drive_status(self, handler=None): if handler is not None: if self.do_get_status(handler) == 0: return None # TODO: actually, if not 3 if hasattr(self, "drive_client"): if hasattr(self.drive_client, "service"): return True return False def do_logout(self, handler): status = self.do_get_status(handler) if status not in [2, 3]: if DEBUG: print "CANNOT LOG IN USER, DON'T EVEN TRY (status %d)" % status return None credentials = parseRequestEntity(handler.request.body) if credentials is None: return None if DEBUG: print credentials return self.logoutUser(credentials, handler) def do_login(self, handler): status = self.do_get_status(handler) if status != 1: if DEBUG: print "CANNOT LOG IN USER, DON'T EVEN TRY (status %d)" % status return None credentials = parseRequestEntity(handler.request.body) if credentials is None: return None if DEBUG: print credentials try: return self.loginUser(credentials['username'], credentials['password'], handler) except KeyError as e: if DEBUG: print "CANNOT LOG IN USER: %s missing" % e return None def initDriveClient(self, restart=False): if DEBUG: print "INITING DRIVE CLIENT" if not hasattr(self, "drive_client") or restart: self.drive_client = UnveillanceAnnexClient() sleep(2) return self.do_get_drive_status() def logoutUser(self, credentials, handler): handler.clear_cookie(UnveillanceCookie.USER) handler.clear_cookie(UnveillanceCookie.ADMIN) try: username = credentials['username'] except KeyError as e: return None try: password = credentials['password'] except KeyError as e: return True try: IV = getConfig('encryption.iv') SALT = getConfig('encryption.salt') USER_SALT = getConfig('encyption.user_salt') except Exception as e: if DEBUG: print e return None try: from Utils.funcs import decryptUserData, encryptUserData except Exception as e: if DEBUG: print e from lib.Frontend.Utils.funcs import decryptUserData, encryptUserData user_root = "%s.txt" % generateMD5Hash(content=username, salt=USER_SALT) with open(os.path.join(USER_ROOT, user_root), 'rb') as UD: user_data = decryptUserData(UD.read, password, p_salt=SALT) if user_data is None: return None new_data = copy.deepcopy(user_data) new_data['saved_searches'] = credentials['save_data'][ 'saved_searches'] try: new_data['annex_key_sent'] = credentials['save_data'][ 'annex_key_sent'] except KeyError as e: pass with open(os.path.join(USER_ROOT, user_root), 'wb+') as UD: UD.write(encryptUserData(new_data, password, iv=IV, p_salt=SALT)) return True return None def loginUser(self, username, password, handler): try: SALT = getConfig('encryption.salt') USER_SALT = getConfig('encyption.user_salt') except Exception as e: if DEBUG: print e return None from base64 import b64encode try: from Utils.funcs import decryptUserData except Exception as e: if DEBUG: print e from lib.Frontend.Utils.funcs import decryptUserData try: user_root = "%s.txt" % generateMD5Hash(content=username, salt=USER_SALT) with open(os.path.join(USER_ROOT, user_root), 'rb') as UD: user_data = decryptUserData(UD.read(), password, p_salt=SALT) if user_data is None: return None try: if user_data['admin']: del user_data['admin'] handler.set_secure_cookie(UnveillanceCookie.ADMIN, "true", path="/", expires_days=1) if not self.do_get_drive_status(): self.initDriveClient() if "annex_key_sent" not in user_data.keys( ) or not user_data['annex_key_sent']: if self.drive_client.checkAnnexKeyStatus(): user_data['annex_key_sent'] = True except KeyError as e: if DEBUG: print e pass handler.set_secure_cookie(UnveillanceCookie.USER, b64encode(json.dumps(user_data)), path="/", expires_days=1) return user_data except Exception as e: if DEBUG: print e return None