async def analytics_payload_tree_api(request, user): # each payload is the root of a tree, all of the corresponding callbacks that use it are under that tree dbpayloads = await db_objects.execute(Payload.select()) display_config = {} display_config['inactive'] = False # by default, include only active callbacks display_config['strikethrough'] = False if request.method == 'POST': data = request.json if 'inactive' in data: display_config['inactive'] = data['inactive'] if 'strikethrough' in data: display_config['strikethrough'] = data['strikethrough'] tree = [] for p in dbpayloads: display = await analytics_payload_tree_api_function(p, display_config) ptree = Node(str(p.id), display=display) # now get all callbacks that have this payload tied to it if display_config['inactive']: # we want to display the inactive ones as well using_callbacks = await db_objects.execute(Callback.select().where(Callback.registered_payload==p)) else: using_callbacks = await db_objects.execute(Callback.select().where( (Callback.registered_payload==p) & (Callback.active == True))) tree.append(ptree) for c in using_callbacks: # each of these callbacks has ptree as an associated payload callback_display = await analytics_callback_tree_api_function(c.to_json(), display_config) Node(str(c.id), parent=ptree, display=callback_display) output = "" for t in tree: # this is iterating over each payload-based tree output += str(RenderTree(t, style=DoubleStyle).by_attr("display")) + "\n" return text(output)
async def get_all_tasks_by_callback_in_current_operation(request, user): try: operation = await db_objects.get(Operation, name=user['current_operation']) except Exception as e: return json({'status': 'error', 'error': 'Not part of an operation'}) output = [] callbacks = await db_objects.execute(Callback.select().where( Callback.operation == operation).order_by(Callback.id)) for callback in callbacks: c = callback.to_json( ) # hold this callback, task, and response info to push to our output stack c['tasks'] = [] tasks = await db_objects.execute( Task.select().where(Task.callback == callback).order_by(Task.id)) for t in tasks: t_data = t.to_json() t_data['responses'] = [] t_data['attackids'] = [ ] # display the att&ck id numbers associated with this task if there are any responses = await db_objects.execute(Response.select().where( Response.task == t).order_by(Response.id)) for r in responses: t_data['responses'].append(r.to_json()) attackids = await db_objects.execute(ATTACKId.select().where(( ATTACKId.task == t) | (ATTACKId.cmd == t.command)).order_by( ATTACKId.id)) for a in attackids: t_data['attackids'].append() # make it a set so we don't have duplicates from the command and some other method t_data['attackids'] = set(t_data['attackids']) c['tasks'].append(t_data) output.append(c) return json({'status': 'success', 'output': output})
async def list_all_screencaptures_per_operation(request, user): if user['auth'] not in ['access_token', 'apitoken']: abort(status_code=403, message= "Cannot access via Cookies. Use CLI or access via JS in browser") if user['current_operation'] != "": query = await db_model.operation_query() operation = await db_objects.get(query, name=user['current_operation']) query = await db_model.filemeta_query() screencaptures = await db_objects.prefetch( query.where( FileMeta.path.regexp(".*{}/downloads/.*/screenshots/".format( operation.name))), Task.select(), Command.select(), Callback.select()) screencapture_paths = [] for s in screencaptures: screencapture_paths.append(s.to_json()) return json({'status': 'success', 'files': screencapture_paths}) else: return json({ "status": 'error', 'error': 'must be part of a current operation to see an operation\'s screencaptures' })
async def ws_tasks(request, ws): try: async with aiopg.create_pool(apfell.config['DB_POOL_CONNECT_STRING']) as pool: async with pool.acquire() as conn: async with conn.cursor() as cur: await cur.execute('LISTEN "newtask";') # before we start getting new things, update with all of the old data callbacks = Callback.select() operators = Operator.select() tasks = Task.select() tasks_with_all_info = await db_objects.prefetch(tasks, callbacks, operators) # callbacks_with_operators = await db_objects.prefetch(callbacks, operators) for task in tasks_with_all_info: await ws.send(js.dumps(task.to_json())) await ws.send("") # now pull off any new tasks we got queued up while processing the old data while True: try: msg = conn.notifies.get_nowait() id = (msg.payload) tsk = await db_objects.get(Task, id=id) await ws.send(js.dumps(tsk.to_json())) except asyncio.QueueEmpty as e: await asyncio.sleep(2) await ws.send("") # this is our test to see if the client is still there continue except Exception as e: print(e) return finally: # print("closed /ws/tasks") pool.close()
async def ws_callbacks_current_operation(request, ws, user): try: async with aiopg.create_pool(apfell.config['DB_POOL_CONNECT_STRING']) as pool: async with pool.acquire() as conn: async with conn.cursor() as cur: await cur.execute('LISTEN "newcallback";') if user['current_operation'] != "": # before we start getting new things, update with all of the old data operation = await db_objects.get(Operation, name=user['current_operation']) callbacks = Callback.select().where(Callback.operation == operation).order_by(Callback.id) operators = Operator.select() callbacks_with_operators = await db_objects.prefetch(callbacks, operators) for cb in callbacks_with_operators: await ws.send(js.dumps(cb.to_json())) await ws.send("") # now pull off any new callbacks we got queued up while processing the old data while True: # msg = await conn.notifies.get() try: msg = conn.notifies.get_nowait() id = (msg.payload) cb = await db_objects.get(Callback, id=id, operation=operation) await ws.send(js.dumps(cb.to_json())) except asyncio.QueueEmpty as e: await asyncio.sleep(0.5) await ws.send("") # this is our test to see if the client is still there continue except Exception as e: print(e) return finally: pool.close()
async def get_all_callbacks(request, user): if user['current_operation'] != "": operation = await db_objects.get(Operation, name=user['current_operation']) callbacks = await db_objects.execute(Callback.select().where(Callback.operation == operation)) return json([c.to_json() for c in callbacks]) else: return json([])
async def list_all_screencaptures_per_callback(request, user, id): try: query = await db_model.callback_query() callback = await db_objects.get(query, id=id) except Exception as e: print(e) return json({'status': 'error', 'error': 'failed to find callback'}) screencapture_paths = [] if callback.operation.name in user['operations']: query = await db_model.filemeta_query() screencaptures = await db_objects.prefetch( query.where( FileMeta.path.regexp(".*{}/downloads/.*/screenshots/".format( callback.operation.name))), Task.select(), Command.select(), Callback.select()) for s in screencaptures: if s.task.callback == callback: screencapture_paths.append(s.to_json()) return json({ 'status': 'success', 'callback': callback.id, 'files': screencapture_paths }) else: return json({ 'status': 'error', 'error': 'must be part of that callback\'s operation to see its screenshots' })
async def get_all_callbacks(request): #callbacks = await db_objects.execute(Callback.select(Callback, Operator).join(Operator)) #return json([c.to_json() for c in callbacks]) callbacks = Callback.select() operators = Operator.select() callbacks_with_operators = await db_objects.prefetch(callbacks, operators) return json([c.to_json() for c in callbacks_with_operators])
async def get_all_tasks(request): callbacks = Callback.select() operators = Operator.select() tasks = Task.select() # callbacks_with_operators = await db_objects.prefetch(callbacks, operators) full_task_data = await db_objects.prefetch(tasks, callbacks, operators) return json([c.to_json() for c in full_task_data])
async def get_all_tasks_by_callback_in_current_operation(request, user): try: operation = await db_objects.get(Operation, name=user['current_operation']) except Exception as e: return json({'status': 'error', 'error': 'Not part of an operation'}) output = [] callbacks = await db_objects.execute(Callback.select().where( Callback.operation == operation).order_by(Callback.id)) for callback in callbacks: c = callback.to_json( ) # hold this callback, task, and response info to push to our output stack c['tasks'] = [] tasks = await db_objects.execute( Task.select().where(Task.callback == callback).order_by(Task.id)) for t in tasks: t_data = t.to_json() t_data['responses'] = [] responses = await db_objects.execute(Response.select().where( Response.task == t).order_by(Response.id)) for r in responses: t_data['responses'].append(r.to_json()) c['tasks'].append(t_data) output.append(c) return json({'status': 'success', 'output': output})
async def get_all_files_meta(request, user): try: query = await db_model.filemeta_query() files = await db_objects.prefetch(query, Task.select(), Command.select(), Callback.select()) except Exception as e: return json({'status': 'error', 'error': 'failed to get files'}) return json( [f.to_json() for f in files if f.operation.name in user['operations']])
async def get_all_files_meta(request, user): if user['auth'] not in ['access_token', 'apitoken']: abort(status_code=403, message="Cannot access via Cookies. Use CLI or access via JS in browser") try: query = await db_model.filemeta_query() files = await db_objects.prefetch(query, Task.select(), Command.select(), Callback.select()) except Exception as e: return json({'status': 'error', 'error': 'failed to get files'}) return json([f.to_json() for f in files if f.operation.name in user['operations']])
async def get_all_tasks(request, user): callbacks = Callback.select() operators = Operator.select() tasks = Task.select() full_task_data = await db_objects.prefetch(tasks, callbacks, operators) if user['admin']: # callbacks_with_operators = await db_objects.prefetch(callbacks, operators) return json([c.to_json() for c in full_task_data]) elif user['current_operation'] != "": operation = await db_objects.get(Operation, name=user['current_operation']) return json([c.to_json() for c in full_task_data if c.callback.operation == operation]) else: return json({'status': 'error', 'error': 'must be admin to see all tasks or part of a current operation'})
async def get_all_tasks(request, user): if user['admin']: callbacks = Callback.select() operators = Operator.select() tasks = Task.select() # callbacks_with_operators = await db_objects.prefetch(callbacks, operators) full_task_data = await db_objects.prefetch(tasks, callbacks, operators) return json([c.to_json() for c in full_task_data]) else: return json({ 'status': 'error', 'error': 'must be admin to see all tasks' })
async def get_all_responses(request, user): try: responses = [] operation = await db_objects.get(Operation, name=user['current_operation']) callbacks = await db_objects.execute(Callback.select().where(Callback.operation == operation)) for c in callbacks: tasks = await db_objects.execute(Task.select().where(Task.callback == c)) for t in tasks: task_responses = await db_objects.execute(Response.select().where(Response.task == t)) responses += [r.to_json() for r in task_responses] except Exception as e: return json({'status': 'error', 'error': 'Cannot get responses'}) return json(responses)
async def analytics_callback_tree_api(request, user): # look at the current callbacks and return their data in a more manageable tree format # http://anytree.readthedocs.io/en/latest/ operation = await db_objects.get(Operation, name=user['current_operation']) dbcallbacks = await db_objects.execute(Callback.select().where(Callback.operation == operation)) callbacks = [] # Default values here display_config = {} display_config['inactive'] = True # by default, include all callbacks display_config['strikethrough'] = False # The POST version of this API function will provide modifiers for what specific information to provide in the callback tree, but the main logic remains the same if request.method == 'POST': data = request.json if 'inactive' in data: display_config['inactive'] = data['inactive'] if 'strikethrough' in data: display_config['strikethrough'] = data['strikethrough'] for dbc in dbcallbacks: if display_config['inactive']: # include everything callbacks.append(dbc.to_json()) elif dbc.active: callbacks.append(dbc.to_json()) elif not dbc.active: json_val = dbc.to_json() json_val['description'] = "CALLBACK DEAD " + json_val['description'] callbacks.append(json_val) # every callback with a pcallback of null should be at the root (remove them from list as we place them) tree = [] while len(callbacks) != 0: # when we hit 0 we are done processing for c in callbacks: # this is the root of a 'tree' if c['pcallback'] == 'null': display = await analytics_callback_tree_api_function(c, display_config) tree.append(Node(str(c['id']), display=display)) callbacks.remove(c) # remove the one we just processed from our list else: for t in tree: # for each tree in our list, see if we can find the parent leaf = find_by_attr(t, str(c['pcallback'])) if leaf: display = await analytics_callback_tree_api_function(c, display_config) Node(str(c['id']), parent=leaf, display=display) callbacks.remove(c) break output = "" for t in tree: output += str(RenderTree(t, style=DoubleStyle).by_attr("display")) + "\n" return text(output)
async def get_all_files_meta(request, user): if user["auth"] not in ["access_token", "apitoken"]: abort( status_code=403, message= "Cannot access via Cookies. Use CLI or access via JS in browser", ) try: query = await db_model.filemeta_query() files = await db_objects.prefetch(query, Task.select(), Command.select(), Callback.select()) except Exception as e: return json({"status": "error", "error": "failed to get files"}) return json( [f.to_json() for f in files if f.operation.name in user["operations"]])
async def get_all_artifact_tasks(request, user): # get all of the artifact tasks for the current operation try: operation = await db_objects.get(Operation, name=user['current_operation']) except: return json({ 'status': 'error', 'error': "failed to get current operation" }) callbacks = Callback.select().where(Callback.operation == operation) artifact_tasks = await db_objects.execute( TaskArtifact.select().join(Task).where(Task.callback.in_(callbacks))) return json({ 'status': 'success', 'tasks': [a.to_json() for a in artifact_tasks] })
async def get_current_operations_files_meta(request, user): if user['current_operation'] != "": try: query = await db_model.operation_query() operation = await db_objects.get(query, name=user['current_operation']) query = await db_model.filemeta_query() files = await db_objects.prefetch( query.where(FileMeta.operation == operation), Task.select(), Command.select(), Callback.select()) except Exception as e: return json({'status': 'error', 'error': 'failed to get files'}) return json( [f.to_json() for f in files if not "screenshots" in f.path]) else: return json({ "status": 'error', 'error': 'must be part of an active operation' })
async def update_active_callbacks(request, user): # Add this as a 'Task' in Sanic's loop so it repeatedly get calls to update this behind the scenes # It can also be done manually at any time via this GET request to update all callback statuses try: all_callbacks = await db_objects.execute(Callback.select().where(Callback.active == True)) # if a callback is more than 3x late for a checkin, it's considered inactive # if/when it does finally callback, its status will be updated to active again # There's no need to look at callbacks already set to 'inactive' # TODO finish this part by adding a task to periodically do this to the event loop for c in all_callbacks: if (c.callback_interval * 3 + c.last_checkin) > datetime.now(): c.active = False try: await db_objects.update(c) except Exception as e: print("Failed to update callback to inactive") print(e) except Exception as e: print(e) return json({'status': 'error', 'error': str(e)})
async def ws_responses_current_operation(request, ws, user): try: async with aiopg.create_pool( apfell.config['DB_POOL_CONNECT_STRING']) as pool: async with pool.acquire() as conn: async with conn.cursor() as cur: await cur.execute('LISTEN "newresponse";') if user['current_operation'] != "": operation = await db_objects.get( Operation, name=user['current_operation']) responses = Response.select() tasks = Task.select() callbacks = Callback.select().where( Callback.operation == operation) responses_with_tasks = await db_objects.prefetch( responses, tasks, callbacks) for resp in responses_with_tasks: await ws.send(js.dumps(resp.to_json())) await ws.send("") # now pull off any new responses we got queued up while processing old responses while True: try: msg = conn.notifies.get_nowait() id = (msg.payload) rsp = await db_objects.get(Response, id=id) if rsp.task.callback.operation.name == user[ 'current_operation']: await ws.send(js.dumps(rsp.to_json())) # print(msg.payload) except asyncio.QueueEmpty as e: await asyncio.sleep(2) await ws.send( "" ) # this is our test to see if the client is still there continue except Exception as e: print(e) return finally: # print("closed /ws/task_updates") pool.close()
async def get_current_operations_files_meta(request, user): if user['auth'] not in ['access_token', 'apitoken']: abort(status_code=403, message= "Cannot access via Cookies. Use CLI or access via JS in browser") if user['current_operation'] != "": try: query = await db_model.operation_query() operation = await db_objects.get(query, name=user['current_operation']) query = await db_model.filemeta_query() files = await db_objects.prefetch( query.where(FileMeta.operation == operation), Task.select(), Command.select(), Callback.select()) except Exception as e: return json({'status': 'error', 'error': 'failed to get files'}) return json( [f.to_json() for f in files if not "screenshots" in f.path]) else: return json({ "status": 'error', 'error': 'must be part of an active operation' })
async def analytics_callback_tree_api(request): # look at the current callbacks and return their data in a more manageable tree format # http://anytree.readthedocs.io/en/latest/ dbcallbacks = await db_objects.execute(Callback.select()) callbacks = [] for dbc in dbcallbacks: callbacks.append(dbc.to_json()) # every callback with a pcallback of null should be at the root (remove them from list as we place them) tree = set() while len(callbacks) != 0: # when we hit 0 we are done processing for c in callbacks: # this is the root of a 'tree' if c['pcallback'] == 'null': tree.add( Node( str(c['id']), display=str(c['user'] + "@" + c['host'] + "(" + str(c['pid']) + "): " + c['description']))) callbacks.remove( c) # remove the one we just processed from our list else: for t in tree: # for each tree in our list, see if we can find the parent leaf = find_by_attr(t, str(c['pcallback'])) if leaf: Node(str(c['id']), parent=leaf, display=str(c['user'] + "@" + c['host'] + "(" + str(c['pid']) + "): " + c['description'])) callbacks.remove(c) break output = "" for t in tree: output += str(RenderTree(t, style=DoubleStyle).by_attr("display")) + "\n" return text(output)
async def get_all_callbacks(request, user): callbacks = Callback.select() return json([c.to_json() for c in callbacks])
async def database_clears(request, user): try: operator = await db_objects.get(Operator, username=user['username']) operation = await db_objects.get(Operation, name=user['current_operation']) if operation.name not in user['admin_operations']: return json({'status': 'error', 'error': "you must be the admin of the operation to clear the database"}) except Exception as e: return json({'status': 'error', 'error': "failed to get the operation and operation: " + str(e)}) data = request.json if 'object' not in data: return json({'status': 'error', 'error': '"object" is a required parameter'}) deleted_obj_info = {'dbnumber': 0} if data['object'] == "payloads": payloads = await db_objects.execute(Payload.select().where(Payload.operation == operation)) for p in payloads: try: os.remove(p.location) # delete it from disk first except Exception as e: print(e) await db_objects.delete(p, recursive=True) # then delete it and everything it relies on from the db deleted_obj_info['dbnumber'] = deleted_obj_info['dbnumber'] + 1 elif data['object'] == "callbacks": callbacks = await db_objects.execute(Callback.select().where(Callback.operation == operation)) for c in callbacks: await db_objects.delete(c, recursive=True) deleted_obj_info['dbnumber'] = deleted_obj_info['dbnumber'] + 1 elif data['object'] == "screencaptures": screencaptures = await db_objects.execute(FileMeta.select().where( (FileMeta.operation == operation) & (FileMeta.path.contains("/screenshots/")) )) for s in screencaptures: try: os.remove(s.path) except Exception as e: print(e) await db_objects.delete(s, recursive=True) deleted_obj_info['dbnumber'] = deleted_obj_info['dbnumber'] + 1 elif data['object'] == "downloads": downloads = await db_objects.execute(FileMeta.select().where( (FileMeta.operation == operation) & (FileMeta.path.contains("/downloads/")) )) for d in downloads: try: os.remove(d.path) except Exception as e: print(e) await db_objects.delete(d, recursive=True) deleted_obj_info['dbnumber'] = deleted_obj_info['dbnumber'] + 1 shutil.rmtree("./app/files/{}/downloads".format(operation.name)) # remove the downloads folder from disk elif data['object'] == "uploads": uploads = await db_objects.execute(FileMeta.select().where( (FileMeta.operation == operation) & (FileMeta.path.contains("/{}/".format(operation.name))) & ~(FileMeta.path.contains("/downloads")) )) for u in uploads: try: os.remove(u.path) except Exception as e: print(e) await db_objects.delete(u, recursive=True) deleted_obj_info['dbnumber'] = deleted_obj_info['dbnumber'] + 1 elif data['object'] == "keylogs": keylogs = await db_objects.execute(Keylog.select().where(Keylog.operation == operation)) for k in keylogs: await db_objects.delete(k, recursive=True) deleted_obj_info['dbnumber'] = deleted_obj_info['dbnumber'] + 1 elif data['object'] == "credentials": credentials = await db_objects.execute(Credential.select().where(Credential.operation == operation)) for c in credentials: await db_objects.delete(c, recursive=True) deleted_obj_info['dbnumber'] = deleted_obj_info['dbnumber'] + 1 elif data['object'] == "tasks": callbacks = Callback.select().where(Callback.operation == operation) tasks = await db_objects.prefetch(Task.select(), callbacks) for t in tasks: await db_objects.delete(t, recursive=True) deleted_obj_info['dbnumber'] = deleted_obj_info['dbnumber'] + 1 elif data['object'] == "responses": callbacks = Callback.select().where(Callback.operation == operation) tasks = Task.select() responses = await db_objects.prefetch(Response.select(), tasks, callbacks) for r in responses: await db_objects.delete(r, recursive=True) deleted_obj_info['dbnumber'] = deleted_obj_info['dbnumber'] + 1 return json({"status": "success", 'stats': deleted_obj_info})