def initDriveClient(self, restart=False):
if DEBUG: print "INITING DRIVE CLIENT"
if not hasattr(self, "drive_client") or restart:
self.drive_client = UnveillanceAnnexClient()
sleep(2)
return self.do_get_drive_status()
def initDriveClient(self, restart=False): if DEBUG: print "INITING DRIVE CLIENT" if not hasattr(self, "drive_client") or restart: self.drive_client = UnveillanceAnnexClient() sleep(2) return self.do_get_drive_status()
class UnveillanceAPI():
def __init__(self):
print "Stock Unveillance Frontend API started..."
self.UNVEILLANCE_LM_VARS = {}
def do_documents(self, handler): return self.passToAnnex(handler)
def do_list(self, handler): return self.passToAnnex(handler)
def do_cluster(self, handler): return self.passToAnnex(handler)
def do_reindex(self, handler):
req = parseRequestEntity(handler.request.body)
if req is None:
return None
if '_id' not in req.keys():
return None
'''
we might have to replace parts of the request with
secret values.
'''
if 'task_path' in req.keys():
req_keys = list(chain.from_iterable(
[k.keys() for k in TASK_REQUIREMENTS]))
if req['task_path'] in req_keys:
try:
req_field = [k for k in TASK_REQUIREMENTS if k.keys()[0] == req['task_path']][0][req['task_path']]
print req_field
except Exception as e:
if DEBUG: print "Could not resolve required task info. %s" % e
for repl in req_field.iteritems():
if repl[0] in req.keys():
handler.request.body = handler.request.body.replace(
repl[1], getSecrets(repl[1]))
return self.passToAnnex(handler)
def do_import(self, handler):
status = self.do_get_status(handler)
if status not in PERMISSIONS['upload_local']:
return
for s_a in handler.request.files.keys():
if s_a != "uv_import":
save_as = handler.request.files[s_a][0]
break
try:
entry = BytesIO()
entry.write(save_as['body'])
except Exception as e:
if DEBUG:
print "COULD NOT CREATE FILE FROM BLOB %s" % e
return None
netcat_stub = {
'save_as' : save_as['filename'],
'file' : entry,
'importer_source' : "web_frontend_status_%d" % status
}
if status not in PERMISSIONS['upload_global']:
# add this filename to the restricted list in handler
if DEBUG:
print "SINCE STATUS == %d, this file will be restricted locally" % status
#netcat_stub['for_local_use_only'] = True
res = self.checkForDuplicate(self.get_new_hash(netcat_stub['file']))
if res is None:
res = {
'file_name' : save_as['filename'],
'result' : 404,
'duplicate_attempt' : False
}
else:
res.update({
'result' : 200, # or??
'duplicate_attempt' : True
})
return res
else:
try:
res = self.addToNetcatQueue(netcat_stub)
if 'duplicate_attempt' not in res.keys():
res.update({
'file_name' : save_as['filename'],
'result' : 200 if res['uploaded'] else 403
})
return res
except Exception as e:
if DEBUG:
print "ERROR IN NETCAT: %s" % e
return None
def do_open_drive_file(self, handler):
if DEBUG: print "opening this drive file in unveillance annex"
status = self.do_get_status(handler)
if status not in [2,3]:
if DEBUG: print "NO-ACCESS TO THIS METHOD (\"do_open_drive_file\")"
return None
files = None
self.initDriveClient(handler)
for _id in parseRequestEntity(handler.request.query)['_ids']:
_id = urllib.unquote(_id).replace("'", "")[1:]
file_name = self.drive_client.getFileName(_id)
if file_name is None: return None
url = "%s/documents/?file_name=%s" % (buildServerURL(), file_name)
entry = None
handled_file = None
if DEBUG: print url
# look up the file in annex. (annex/documents/?file_name=file)
# if this file exists in annex, return its _id for opening in-app
try:
entry = json.loads(requests.get(
url, verify=False).content)['data']['documents'][0]
except Exception as e:
if DEBUG: print "COULD NOT GET ENTRY:\n%s" % e
if entry is not None:
if DEBUG: print type(entry['_id'])
handled_file = { '_id' : entry['_id'] }
else:
if status != 3:
if DEBUG:
print "** at this point, we would process file if you were admin"
print "** but you are not admin."
return None
entry = self.drive_client.download(_id,
save_as=file_name, save=False, return_content=True)
if entry is not None:
self.addToNetcatQueue({
'file' : entry[0],
'save_as' : entry[1],
'importer_source' : "google_drive"
})
handled_file = { 'file_name' : entry[1] }
if handled_file is not None:
if files is None: files = []
files.append(handled_file)
return files
def do_get_admin_party_status(self, handler):
status = self.do_get_status(handler)
if status == 0: return None
from conf import USER_ROOT
for _, _, files in os.walk(USER_ROOT):
for f in files: return False
return True
def do_create_new_user(self, handler):
status = self.do_get_status(handler)
if status != 3: return None
new_user = parseRequestEntity(handler.request.body)
if new_user is None: return None
if DEBUG: print new_user
n_set = sorted(['nu_username', 'nu_password'])
if sorted(list(set(n_set) & set(new_user.keys()))) != n_set:
return None
from Utils.funcs import createNewUser
return createNewUser(new_user['nu_username'], new_user['nu_password'])
def do_get_user_status(self, handler):
status = self.do_get_status(handler)
if status == 0: return None
if self.do_get_admin_party_status(handler): return 4
return status
def do_get_status(self, handler):
try:
for cookie in handler.request.cookies:
if cookie == UnveillanceCookie.PUBLIC:
print "**** VISITOR OF TYPE ZERO ENCOUNTERED: %s *****" % UnveillanceCookie.PUBLIC
return 0
except KeyError as e: pass
access = handler.get_secure_cookie(UnveillanceCookie.USER)
if access is not None:
if handler.get_secure_cookie(UnveillanceCookie.ADMIN) is not None:
return 3
return 2
return 1
def do_get_drive_status(self, handler=None):
if handler is not None:
if self.do_get_status(handler) == 0: return None
# TODO: actually, if not 3
if hasattr(self, "drive_client"):
if hasattr(self.drive_client, "service"):
return True
return False
def do_logout(self, handler):
status = self.do_get_status(handler)
if status not in [2, 3]:
if DEBUG: print "CANNOT LOG IN USER, DON'T EVEN TRY (status %d)" % status
return None
credentials = parseRequestEntity(handler.request.body)
if credentials is None: return None
if DEBUG: print credentials
return self.logoutUser(credentials, handler)
def do_login(self, handler):
status = self.do_get_status(handler)
if status != 1:
if DEBUG: print "CANNOT LOG IN USER, DON'T EVEN TRY (status %d)" % status
return None
credentials = parseRequestEntity(handler.request.body)
if credentials is None: return None
if DEBUG: print credentials
try:
return self.loginUser(credentials['username'],
credentials['password'], handler)
except KeyError as e:
if DEBUG: print "CANNOT LOG IN USER: %s missing" % e
return None
def initDriveClient(self, restart=False):
if DEBUG: print "INITING DRIVE CLIENT"
if not hasattr(self, "drive_client") or restart:
self.drive_client = UnveillanceAnnexClient()
sleep(2)
return self.do_get_drive_status()
def logoutUser(self, credentials, handler):
handler.clear_cookie(UnveillanceCookie.USER)
handler.clear_cookie(UnveillanceCookie.ADMIN)
try:
username = credentials['username']
except KeyError as e: return None
try:
password = credentials['password']
except KeyError as e: return True
try:
IV = getConfig('encryption.iv')
SALT = getConfig('encryption.salt')
USER_SALT = getConfig('encyption.user_salt')
except Exception as e:
if DEBUG: print e
return None
try:
from Utils.funcs import decryptUserData, encryptUserData
except Exception as e:
if DEBUG: print e
from lib.Frontend.Utils.funcs import decryptUserData, encryptUserData
user_root = "%s.txt" % generateMD5Hash(content=username,salt=USER_SALT)
with open(os.path.join(USER_ROOT, user_root), 'rb') as UD:
user_data = decryptUserData(UD.read, password, p_salt=SALT)
if user_data is None: return None
new_data = copy.deepcopy(user_data)
new_data['saved_searches'] = credentials['save_data']['saved_searches']
try:
new_data['annex_key_sent'] = credentials['save_data']['annex_key_sent']
except KeyError as e:
pass
with open(os.path.join(USER_ROOT, user_root), 'wb+') as UD:
UD.write(encryptUserData(new_data, password, iv=IV, p_salt=SALT))
return True
return None
def loginUser(self, username, password, handler):
try:
SALT = getConfig('encryption.salt')
USER_SALT = getConfig('encyption.user_salt')
except Exception as e:
if DEBUG: print e
return None
from base64 import b64encode
try:
from Utils.funcs import decryptUserData
except Exception as e:
if DEBUG: print e
from lib.Frontend.Utils.funcs import decryptUserData
try:
user_root = "%s.txt" % generateMD5Hash(content=username, salt=USER_SALT)
with open(os.path.join(USER_ROOT, user_root), 'rb') as UD:
user_data = decryptUserData(UD.read(), password, p_salt=SALT)
if user_data is None: return None
try:
if user_data['admin']:
del user_data['admin']
handler.set_secure_cookie(UnveillanceCookie.ADMIN,
"true", path="/", expires_days=1)
if not self.do_get_drive_status():
self.initDriveClient()
if "annex_key_sent" not in user_data.keys() or not user_data['annex_key_sent']:
if self.drive_client.checkAnnexKeyStatus():
user_data['annex_key_sent'] = True
except KeyError as e:
if DEBUG: print e
pass
handler.set_secure_cookie(UnveillanceCookie.USER,
b64encode(json.dumps(user_data)), path="/", expires_days=1)
return user_data
except Exception as e:
if DEBUG: print e
return None
class UnveillanceAPI():
def __init__(self):
print "Stock Unveillance Frontend API started..."
self.UNVEILLANCE_LM_VARS = {}
def do_documents(self, handler):
return self.passToAnnex(handler)
def do_list(self, handler):
return self.passToAnnex(handler)
def do_cluster(self, handler):
return self.passToAnnex(handler)
def do_reindex(self, handler):
req = parseRequestEntity(handler.request.body)
if req is None:
return None
if '_id' not in req.keys():
return None
'''
we might have to replace parts of the request with
secret values.
'''
if 'task_path' in req.keys():
req_keys = list(
chain.from_iterable(
[k.keys() for k in MIME_TYPE_TASK_REQUIREMENTS]))
if req['task_path'] in req_keys:
try:
req_field = [
k for k in MIME_TYPE_TASK_REQUIREMENTS
if k.keys()[0] == req['task_path']
][0][req['task_path']]
print req_field
except Exception as e:
if DEBUG:
print "Could not resolve required task info. %s" % e
for repl in req_field.iteritems():
if repl[0] in req.keys():
handler.request.body = handler.request.body.replace(
repl[1], getSecrets(repl[1]))
return self.passToAnnex(handler)
def do_import(self, handler):
status = self.do_get_status(handler)
if status not in PERMISSIONS['upload_local']:
return
for s_a in handler.request.files.keys():
if s_a != "uv_import":
save_as = handler.request.files[s_a][0]
break
try:
entry = BytesIO()
entry.write(save_as['body'])
except Exception as e:
if DEBUG:
print "COULD NOT CREATE FILE FROM BLOB %s" % e
return None
netcat_stub = {
'save_as': save_as['filename'],
'file': entry,
'importer_source': "web_frontend_status_%d" % status
}
if status not in PERMISSIONS['upload_global']:
# add this filename to the restricted list in handler
if DEBUG:
print "SINCE STATUS == %d, this file will be restricted locally" % status
#netcat_stub['for_local_use_only'] = True
res = self.checkForDuplicate(self.get_new_hash(
netcat_stub['file']))
if res is None:
res = {
'file_name': save_as['filename'],
'result': 403,
'duplicate_attempt': False
}
else:
res.update({
'result': 200, # or??
'duplicate_attempt': True
})
return res
else:
try:
res = self.addToNetcatQueue(netcat_stub)
if 'duplicate_attempt' not in res.keys():
res.update({
'file_name': save_as['filename'],
'result': 200 if res['uploaded'] else 403
})
return res
except Exception as e:
if DEBUG:
print "ERROR IN NETCAT: %s" % e
return None
def do_open_drive_file(self, handler):
if DEBUG: print "opening this drive file in unveillance annex"
status = self.do_get_status(handler)
if status not in [2, 3]:
if DEBUG: print "NO-ACCESS TO THIS METHOD (\"do_open_drive_file\")"
return None
files = None
self.initDriveClient(handler)
for _id in parseRequestEntity(handler.request.query)['_ids']:
_id = urllib.unquote(_id).replace("'", "")[1:]
file_name = self.drive_client.getFileName(_id)
if file_name is None: return None
url = "%s/documents/?file_name=%s" % (buildServerURL(), file_name)
entry = None
handled_file = None
if DEBUG: print url
# look up the file in annex. (annex/documents/?file_name=file)
# if this file exists in annex, return its _id for opening in-app
try:
entry = json.loads(requests.get(
url, verify=False).content)['data']['documents'][0]
except Exception as e:
if DEBUG: print "COULD NOT GET ENTRY:\n%s" % e
if entry is not None:
if DEBUG: print type(entry['_id'])
handled_file = {'_id': entry['_id']}
else:
if status != 3:
if DEBUG:
print "** at this point, we would process file if you were admin"
print "** but you are not admin."
return None
entry = self.drive_client.download(_id,
save_as=file_name,
save=False,
return_content=True)
if entry is not None:
self.addToNetcatQueue({
'file': entry[0],
'save_as': entry[1],
'importer_source': "google_drive"
})
handled_file = {'file_name': entry[1]}
if handled_file is not None:
if files is None: files = []
files.append(handled_file)
return files
def do_get_admin_party_status(self, handler):
status = self.do_get_status(handler)
if status == 0: return None
from conf import USER_ROOT
for _, _, files in os.walk(USER_ROOT):
for f in files:
return False
return True
def do_create_new_user(self, handler):
status = self.do_get_status(handler)
if status != 3: return None
new_user = parseRequestEntity(handler.request.body)
if new_user is None: return None
if DEBUG: print new_user
n_set = sorted(['nu_username', 'nu_password'])
if sorted(list(set(n_set) & set(new_user.keys()))) != n_set:
return None
from Utils.funcs import createNewUser
return createNewUser(new_user['nu_username'], new_user['nu_password'])
def do_get_user_status(self, handler):
status = self.do_get_status(handler)
if status == 0: return None
if self.do_get_admin_party_status(handler): return 4
return status
def do_get_status(self, handler):
try:
for cookie in handler.request.cookies:
if cookie == UnveillanceCookie.PUBLIC:
print "**** VISITOR OF TYPE ZERO ENCOUNTERED: %s *****" % UnveillanceCookie.PUBLIC
return 0
except KeyError as e:
pass
access = handler.get_secure_cookie(UnveillanceCookie.USER)
if access is not None:
if handler.get_secure_cookie(UnveillanceCookie.ADMIN) is not None:
return 3
return 2
return 1
def do_get_drive_status(self, handler=None):
if handler is not None:
if self.do_get_status(handler) == 0: return None
# TODO: actually, if not 3
if hasattr(self, "drive_client"):
if hasattr(self.drive_client, "service"):
return True
return False
def do_logout(self, handler):
status = self.do_get_status(handler)
if status not in [2, 3]:
if DEBUG:
print "CANNOT LOG IN USER, DON'T EVEN TRY (status %d)" % status
return None
credentials = parseRequestEntity(handler.request.body)
if credentials is None: return None
if DEBUG: print credentials
return self.logoutUser(credentials, handler)
def do_login(self, handler):
status = self.do_get_status(handler)
if status != 1:
if DEBUG:
print "CANNOT LOG IN USER, DON'T EVEN TRY (status %d)" % status
return None
credentials = parseRequestEntity(handler.request.body)
if credentials is None: return None
if DEBUG: print credentials
try:
return self.loginUser(credentials['username'],
credentials['password'], handler)
except KeyError as e:
if DEBUG: print "CANNOT LOG IN USER: %s missing" % e
return None
def initDriveClient(self, restart=False):
if DEBUG: print "INITING DRIVE CLIENT"
if not hasattr(self, "drive_client") or restart:
self.drive_client = UnveillanceAnnexClient()
sleep(2)
return self.do_get_drive_status()
def logoutUser(self, credentials, handler):
handler.clear_cookie(UnveillanceCookie.USER)
handler.clear_cookie(UnveillanceCookie.ADMIN)
try:
username = credentials['username']
except KeyError as e:
return None
try:
password = credentials['password']
except KeyError as e:
return True
try:
IV = getConfig('encryption.iv')
SALT = getConfig('encryption.salt')
USER_SALT = getConfig('encyption.user_salt')
except Exception as e:
if DEBUG: print e
return None
try:
from Utils.funcs import decryptUserData, encryptUserData
except Exception as e:
if DEBUG: print e
from lib.Frontend.Utils.funcs import decryptUserData, encryptUserData
user_root = "%s.txt" % generateMD5Hash(content=username,
salt=USER_SALT)
with open(os.path.join(USER_ROOT, user_root), 'rb') as UD:
user_data = decryptUserData(UD.read, password, p_salt=SALT)
if user_data is None: return None
new_data = copy.deepcopy(user_data)
new_data['saved_searches'] = credentials['save_data'][
'saved_searches']
try:
new_data['annex_key_sent'] = credentials['save_data'][
'annex_key_sent']
except KeyError as e:
pass
with open(os.path.join(USER_ROOT, user_root), 'wb+') as UD:
UD.write(encryptUserData(new_data, password, iv=IV, p_salt=SALT))
return True
return None
def loginUser(self, username, password, handler):
try:
SALT = getConfig('encryption.salt')
USER_SALT = getConfig('encyption.user_salt')
except Exception as e:
if DEBUG: print e
return None
from base64 import b64encode
try:
from Utils.funcs import decryptUserData
except Exception as e:
if DEBUG: print e
from lib.Frontend.Utils.funcs import decryptUserData
try:
user_root = "%s.txt" % generateMD5Hash(content=username,
salt=USER_SALT)
with open(os.path.join(USER_ROOT, user_root), 'rb') as UD:
user_data = decryptUserData(UD.read(), password, p_salt=SALT)
if user_data is None: return None
try:
if user_data['admin']:
del user_data['admin']
handler.set_secure_cookie(UnveillanceCookie.ADMIN,
"true",
path="/",
expires_days=1)
if not self.do_get_drive_status():
self.initDriveClient()
if "annex_key_sent" not in user_data.keys(
) or not user_data['annex_key_sent']:
if self.drive_client.checkAnnexKeyStatus():
user_data['annex_key_sent'] = True
except KeyError as e:
if DEBUG: print e
pass
handler.set_secure_cookie(UnveillanceCookie.USER,
b64encode(json.dumps(user_data)),
path="/",
expires_days=1)
return user_data
except Exception as e:
if DEBUG: print e
return None
