def convert_to_objectid(id): try: #the ObjecId was dumped using bson dumps, BUT JSON was used to load. so reload it using bson loads if isinstance(id, dict): # and '$oid' in id: #the wrong funciton was used to load this variable: #json.loads of an object id results in a dictionary where keys are {'oid':} #therefore, we need to first json.dumps back to a string, and THEN loads using bson loads return bson_loads(json.dumps(id)) #ObjectId(id['$oid']) elif isinstance(id, basestring): if '$oid' in id: #the ObjecId was dumped using bson dumps, so reload it using bson loads #if bson_dumps is used,but it is not turned into dict using josn.loads. instead it is just a string. So we need to convert a bson_dumps string to object id, then we need to slightly modify it..hack for special situations #id = id.replace("{","").replace("}","").replace('"','') #id = id.split(':')[1] #print id return bson_loads(id.strip()) else: return ObjectId(id.strip()) else: return ObjectId(id) except Exception as e: raise Exception( 'There is something wrong with the provided id: {0}\nThe following error was reported: {1}' .format(str(id), str(e)))
def deserialize(to_deserialize, *args, **kwargs): """ Deserializes a string into a PyMongo BSON """ if isinstance(to_deserialize, string_types): try: return ObjectId(to_deserialize) except Exception: return bson_loads(to_deserialize, *args, **kwargs) else: return bson_loads(bson_dumps(to_deserialize), *args, **kwargs)
def deserialize(to_deserialize, *args, **kwargs): """ Deserializes a string into a PyMongo BSON """ if isinstance(to_deserialize, string_types): if re.match('^[0-9a-f]{24}$', to_deserialize): return ObjectId(to_deserialize) try: return bson_loads(to_deserialize, *args, **kwargs) except: return bson_loads(bson_dumps(to_deserialize), *args, **kwargs) else: return bson_loads(bson_dumps(to_deserialize), *args, **kwargs)
def insert_db(jsonpath): '''inserts contents of jsonpath into db.specialdisco.collection''' from bson.json_util import loads as bson_loads with open(jsonpath, 'r') as datafile: data = json.loads(datafile.read()) schools.drop() for record in data: # data is a dict - must be re-strung, then inserted as bson result = schools.insert_one(bson_loads(json.dumps(record)))
def convert_to_objectid(id): ''' This function will convert a string or properly formatted dictionary into a MongoDB object ID. There could be a few ways that an object ID was mal-transformed while querying the database: Examples: 1) str(ObjectId) will convert into a string. In order to convert it back, we simply do ObjectID(str) 2) bson_dumps(ObjectId) will create a bson dumped string variable of an object ID. As a string it looks like '{$oid: id }' But, if we did json.loads on that value, then we dont get an objectId. Instead we get a dictionary {'$oid': id} So this function will attempt to convert such examples back into an ObjectId variable Inputs ------ id: string, dict, or ObjectId variable Outputs ------- id converted into an ObjectId variable ''' try: # the ObjecId was dumped using bson dumps, BUT JSON was used to load. so reload it using bson loads if isinstance(id, dict): # the wrong funciton was used to load this variable: # json.loads of an object id results in a dictionary where keys are {'oid':} # therefore, we need to first json.dumps back to a string, and THEN loads using bson loads return bson_loads(json.dumps(id)) elif isinstance(id, basestring): if '$oid' in id: # the ObjecId was dumped using bson dumps, so reload it using bson loads # if bson_dumps is used,but it is not turned into dict using josn.loads. # instead it is just a string. So we need to convert a bson_dumps string to object id, then we need to slightly modify it..hack for special situations return bson_loads(id.strip()) else: return ObjectId(id.strip()) else: return ObjectId(id) except Exception as e: raise Exception( 'There is something wrong with the provided id: {0}\nThe following error was reported: {1}' .format(str(id), str(e)))
def convert_to_objectid(id): ''' This function will convert a string or properly formatted dictionary into a MongoDB object ID. There could be a few ways that an object ID was mal-transformed while querying the database: Examples: 1) str(ObjectId) will convert into a string. In order to convert it back, we simply do ObjectID(str) 2) bson_dumps(ObjectId) will create a bson dumped string variable of an object ID. As a string it looks like '{$oid: id }' But, if we did json.loads on that value, then we dont get an objectId. Instead we get a dictionary {'$oid': id} So this function will attempt to convert such examples back into an ObjectId variable Inputs ------ id: string, dict, or ObjectId variable Outputs ------- id converted into an ObjectId variable ''' try: # the ObjecId was dumped using bson dumps, BUT JSON was used to load. so reload it using bson loads if isinstance(id, dict): # the wrong funciton was used to load this variable: # json.loads of an object id results in a dictionary where keys are {'oid':} # therefore, we need to first json.dumps back to a string, and THEN loads using bson loads return bson_loads(json.dumps(id)) elif isinstance(id, basestring): if '$oid' in id: # the ObjecId was dumped using bson dumps, so reload it using bson loads # if bson_dumps is used,but it is not turned into dict using josn.loads. # instead it is just a string. So we need to convert a bson_dumps string to object id, then we need to slightly modify it..hack for special situations return bson_loads(id.strip()) else: return ObjectId(id.strip()) else: return ObjectId(id) except Exception as e: raise Exception('There is something wrong with the provided id: {0}\nThe following error was reported: {1}'.format(str(id), str(e)))
def extract(self, name, meta, asname=None, store=None): store = store or self.store lpaths = self._local_paths(self.path, name, store, expect_exist=True) with lpaths.meta.open('r') as fin: meta_dict = bson_loads(fin.read()) meta.attributes.update(meta_dict['attributes']) meta.kind_meta.update(meta_dict['kind_meta']) meta.kind = meta_dict['kind'] if lpaths.gridfile.exists(): with lpaths.gridfile.open('rb') as fin: file_backend = store.get_backend_byobj(fin) meta.gridfile = file_backend._store_to_file(store, fin, lpaths.key) if lpaths.collection.exists(): with lpaths.collection.open('r') as fin: data = bson_loads(fin.read()) collection = store.collection(name) collection.insert_many(data) meta.collection = collection.name if asname: meta.name = asname return meta.save()
def load_module_entry(self, session_id, module_name): entry = self.modules.find_one({ 'session_id': session_id, 'name': module_name }) if not entry or entry.get( 'timeout', datetime.datetime.utcnow() - datetime.timedelta(hours=24)) < datetime.datetime.utcnow(): return None else: return bson_loads(entry['entry'])
def convert_to_objectid(id): try: #the ObjecId was dumped using bson dumps, BUT JSON was used to load. so reload it using bson loads if isinstance(id,dict):# and '$oid' in id: #the wrong funciton was used to load this variable: #json.loads of an object id results in a dictionary where keys are {'oid':} #therefore, we need to first json.dumps back to a string, and THEN loads using bson loads return bson_loads(json.dumps(id)) #ObjectId(id['$oid']) elif isinstance(id,basestring): if '$oid' in id: #the ObjecId was dumped using bson dumps, so reload it using bson loads #if bson_dumps is used,but it is not turned into dict using josn.loads. instead it is just a string. So we need to convert a bson_dumps string to object id, then we need to slightly modify it..hack for special situations #id = id.replace("{","").replace("}","").replace('"','') #id = id.split(':')[1] #print id return bson_loads(id.strip()) else: return ObjectId(id.strip()) else: return ObjectId(id) except Exception as e: raise Exception('There is something wrong with the provided id: {0}\nThe following error was reported: {1}'.format(str(id),str(e)))
def create_example_portfolio(): print('Task: Create example portfolio') try: example_portfolio_content = open('example-portfolio.json').read() portfolio_bson = bson_loads(example_portfolio_content) example_portfolio = portfolios_col.find_one( {'_id': portfolio_bson['_id']}) if example_portfolio: print('Example portfolio already exists') else: print('Creating example portfolio') portfolios_col.insert_one(portfolio_bson) print('Successfully inserted example portfolio') except Exception as e: print('Unable to insert example portfolio')
def _type_transform(self, value): """ Transforms ObjectId types to str type and vice versa. Any ObjectId types present are serialized to a str. The same str is converted back to an ObjectId while de-serializing. :param value: the object to be transformed :type value: Object :returns: recursively transformed object :rtype: Object """ # Encoding ObjectId to str if isinstance(value, ObjectId): return bson_dumps(value) # Recursive checks inside dict if isinstance(value, dict): if len(value) == 0: return value # Decoding '$oid' back to ObjectId if '$oid' in value.keys(): return bson_loads(value) return dict((self._type_transform(k), self._type_transform(v)) for k, v in value.iteritems()) # Recursive checks inside a list if isinstance(value, list): if len(value) == 0: return value for i, val in enumerate(value): value[i] = self._type_transform(val) return value # Recursive checks inside a tuple if isinstance(value, tuple): if len(value) == 0: return value return tuple([self._type_transform(val) for val in value]) return value
def content(self): # Check if the session packets are set to 0 (i.e. session packets are not loaded in memory) if not self.dns_requests: # Try to load results from database debug_output("Loading entry from DB") self.dns_requests = self.load_entry() if not self.dns_requests: debug_output("No results in DB, processing PCAP") filename = self.session.pcap_filename self.session.pkts = sniff(stopper=self.session.stop_sniffing, filter=self.session.filter, prn=self.on_packet, stopperTimeout=1, offline=self.session.engine.setup['SNIFFER_DIR']+"/"+filename) # now that everything has been processed, save the results to DB self.save_entry(bson_dumps(self.dns_requests)) else: self.dns_requests = bson_loads(self.dns_requests) content = "<table class='table table-condensed'><tr><th>Query</th><th>Answers</th><th>Count</th></tr>" for q in self.dns_requests: content += "<tr><td>{}</td><td>{}</td><td>{}</td></tr>".format(q, ", ".join(self.dns_requests[q]['answers']), self.dns_requests[q]['count']) content += "</table>" return content
def fetch_sniffer_session(self, session_id): try: debug_output("Fetching session {} from memory".format(session_id)) session = self.sessions.get(ObjectId(session_id)) except Exception as e: debug_output("An {} error occurred when fetching session '{}': {}".format(type(e).__name__, session_id, e), 'error') return # if not found, recreate it from the DB if not session: debug_output("Fetching session {} from DB".format(session_id)) s = self.model.get_sniffer_session(session_id) if not s: return None # TLS interception only possible if PCAP hasn't been generated yet intercept_tls = s['intercept_tls'] and not s['pcap'] if s: session = SnifferSession(s['name'], None, None, self, id=s['_id'], filter_restore=s['filter'], intercept_tls=intercept_tls) session.pcap = s['pcap'] session.public = s['public'] session.date_created = s['date_created'] self.sessions[session.id] = session session_data = bson_loads(s['session_data']) session.nodes = session_data['nodes'] session.edges = session_data['edges'] session.packet_count = s['packet_count'] session.flows = {} for flow in session_data['flows']: f = Flow.load_flow(flow) session.flows[f.fid] = f return session
def queryMongo(host, port, database, collection, query): ''' ----------- PARAMS ----------- host : The hostname of the database to connect to port : The port that accepts connections user : username that has permission to execute queries password : The password for authentication database : The database one the Postgres instance to run the query against query : The query to execute ''' try: mongoURL = 'mongodb://' + host + ':' + port client = pm.MongoClient(mongoURL) db = client.get_database(database) col = db.get_collection(collection) curs = col.find(bson_loads(query)) data = pd.DataFrame(list(curs)) return data except ValueError as e: raise Exception("ValueError: Most likely no rows were returned from database.")
def command_handler(self, msg): self.command_lock.acquire() msg = json.loads(msg) cmd = msg["msg"] params = msg.get("params", {}) src = msg["src"] queryid = msg["queryid"] final_msg = None if cmd == "newsession": _id = self.snifferengine.new_session(params) final_msg = bson_dumps(_id) if cmd == "sessionlist": session_list = [] user = params.get("user", None) page = params.get("page", 0) private = params.get("private", False) for session in self.snifferengine.model.get_sniffer_sessions(private=private, username=user, page=page): if session["_id"] in self.snifferengine.sessions: session = self.snifferengine.sessions[session["_id"]] active = session.status() session = session.__dict__ else: active = False session_data = bson_loads(session["session_data"]) session["nodes"] = session_data["nodes"] session["edges"] = session_data["edges"] session["id"] = session["_id"] session_list.append( { "id": str(session.get("id")), "date_created": session.get("date_created"), "name": session.get("name"), "packets": session.get("packet_count"), "nodes": len(session.get("nodes")), "edges": len(session.get("edges")), "status": "Running" if active else "Stopped", "public": session.get("public"), } ) final_msg = bson_dumps(session_list) if params.get("session_id", False): session = self.snifferengine.fetch_sniffer_session(params["session_id"]) if not session: final_msg = "notfound" if session: if cmd == "sessioninfo": final_msg = { "name": session.name, "filter": session.filter, "pcap": session.pcap, "packet_count": session.packet_count, "pcap_filename": session.pcap_filename, "id": str(session.id), "public": session.public, "status": session.status(), "node_list": session.get_nodes(), "modules": [ (session.modules[module_name].display_name, module_name) for module_name in session.modules ], } if cmd == "sniffstatus": final_msg = {"status": session.status()} if cmd == "sniffupdate": # this needs to be stringyfied, or else encoding errors will ensue final_msg = session.update_nodes() final_msg = json.dumps(final_msg, default=json_util.default) if cmd == "sniffstart": # self.snifferengine.start_session(params['session_name'], params['remote_addr']) session.start() final_msg = True if cmd == "sniffstop": session.stop() final_msg = True if cmd == "flowstatus": flow = session.flow_status() # this needs to be stringyfied, or else encoding errors will ensue final_msg = flow if cmd == "flow_statistics_update": print "Received 'flow_statistics_update' message. Please implement me? " if cmd == "get_flow_payload": if params["flowid"] in session.flows: final_msg = session.flows[params["flowid"]].get_payload(encoding="base64") else: final_msg = False if cmd == "sniffdelete": result = self.snifferengine.delete_session(params["session_id"]) final_msg = result if cmd == "sniffpcap": result = self.snifferengine.commit_to_db(session) final_msg = result if cmd == "call_module_function": module_name = params["module_name"] function = params["function"] args = params["args"] module = session.modules.get(module_name, None) if module: try: final_msg = getattr(module, function)(args) except Exception, e: import traceback final_msg = "[{} in function {}] Module error: {}\n".format(module_name, function, e) final_msg += traceback.format_exc() final_msg = "<pre>{}</pre>".format(final_msg) else: final_msg = False
def load_module_entry(self, session_id, module_name): entry = self.modules.find_one({'session_id': session_id, 'name': module_name}) if entry: return bson_loads(entry['entry']) else: return {}
def get_query(self): """Loads the given json-encoded query.""" query = request.GET.get("query") return bson_loads(query) if query else {}
def command_handler(self, msg): self.command_lock.acquire() msg = json.loads(msg) cmd = msg['msg'] params = msg.get('params', {}) src = msg['src'] queryid = msg['queryid'] final_msg = None if cmd == 'newsession': _id = self.snifferengine.new_session(params) final_msg = bson_dumps(_id) if cmd == 'sessionlist': session_list = [] user = params.get('user', None) page = params.get('page', 0) private = params.get('private', False) for session in self.snifferengine.model.get_sniffer_sessions(private=private, username=user, page=page): if session['_id'] in self.snifferengine.sessions: session = self.snifferengine.sessions[session['_id']] active = session.status() session = session.__dict__ else: active = False session_data = bson_loads(session['session_data']) session['nodes'] = session_data['nodes'] session['edges'] = session_data['edges'] session_list.append( { 'id': str(session.get('_id')), 'date_created': session.get('date_created'), 'name': session.get('name'), 'packets': session.get('packet_count'), 'nodes': len(session.get('nodes')), 'edges': len(session.get('edges')), 'status': "Running" if active else "Stopped", 'public': session.get('public'), } ) final_msg = bson_dumps(session_list) if params.get('session_id', False): session = self.snifferengine.fetch_sniffer_session(params['session_id']) if not session: final_msg = False if session: if cmd == 'sessioninfo': final_msg = { 'name' : session.name, 'filter' : session.filter, 'pcap' : session.pcap, 'pcap_filename': session.pcap_filename, 'id' : str(session.id), 'public': session.public, 'status': session.status(), } if cmd == 'sniffstatus': final_msg = {'status': session.status()} if cmd == 'sniffupdate': # this needs to be stringyfied, or else encoding errors will ensue final_msg = session.update_nodes() final_msg = json.dumps(final_msg, default=json_util.default) if cmd == 'sniffstart': #self.snifferengine.start_session(params['session_name'], params['remote_addr']) session.start(params['remote_addr']) final_msg = True if cmd == 'sniffstop': session.stop() final_msg = True if cmd == 'flowstatus': flow = session.flow_status() # this needs to be stringyfied, or else encoding errors will ensue final_msg = flow if cmd == 'flow_statistics_update': print "Received 'flow_statistics_update' message. Please implement me? " if cmd == 'get_flow_payload': if params['flowid'] in session.flows: final_msg = session.flows[params['flowid']].get_payload(encoding='base64') else: final_msg = False if cmd == 'sniffdelete': result = self.snifferengine.delete_session(params['session_id']) final_msg = result if cmd == 'sniffpcap': result = self.snifferengine.commit_to_db(session) final_msg = result if final_msg != None: reply = {'msg': final_msg, 'queryid': queryid, 'dst': src, 'src':self.name} self.publish_to_channel('sniffer-commands', json.dumps(reply)) self.command_lock.release() return
def command_handler(self, msg): self.command_lock.acquire() msg = json.loads(msg) cmd = msg['msg'] params = msg.get('params', {}) src = msg['src'] queryid = msg['queryid'] final_msg = None if cmd == 'newsession': _id = self.snifferengine.new_session(params) final_msg = bson_dumps(_id) if cmd == 'sessionlist': session_list = [] user = params.get('user', None) page = params.get('page', 0) private = params.get('private', False) for session in self.snifferengine.model.get_sniffer_sessions( private=private, username=user, page=page): if session['_id'] in self.snifferengine.sessions: session = self.snifferengine.sessions[session['_id']] active = session.status() session = session.__dict__ else: active = False session_data = bson_loads(session['session_data']) session['nodes'] = session_data['nodes'] session['edges'] = session_data['edges'] session['id'] = session['_id'] session_list.append({ 'id': str(session.get('id')), 'date_created': session.get('date_created'), 'name': session.get('name'), 'packets': session.get('packet_count'), 'nodes': len(session.get('nodes')), 'edges': len(session.get('edges')), 'status': "Running" if active else "Stopped", 'public': session.get('public'), }) final_msg = bson_dumps(session_list) if params.get('session_id', False): session = self.snifferengine.fetch_sniffer_session( params['session_id']) if not session: final_msg = 'notfound' if session: if cmd == 'sessioninfo': final_msg = { 'name': session.name, 'filter': session.filter, 'pcap': session.pcap, 'packet_count': session.packet_count, 'pcap_filename': session.pcap_filename, 'id': str(session.id), 'public': session.public, 'status': session.status(), 'node_list': session.get_nodes(), 'modules': [(session.modules[module_name].display_name, module_name) for module_name in session.modules], } if cmd == 'sniffstatus': final_msg = {'status': session.status()} if cmd == 'sniffupdate': # this needs to be stringyfied, or else encoding errors will ensue final_msg = session.update_nodes() final_msg = json.dumps(final_msg, default=json_util.default) if cmd == 'sniffstart': #self.snifferengine.start_session(params['session_name'], params['remote_addr']) session.start() final_msg = True if cmd == 'sniffstop': session.stop() final_msg = True if cmd == 'flowstatus': flow = session.flow_status() # this needs to be stringyfied, or else encoding errors will ensue final_msg = flow if cmd == 'flow_statistics_update': print "Received 'flow_statistics_update' message. Please implement me? " if cmd == 'get_flow_payload': if params['flowid'] in session.flows: final_msg = session.flows[ params['flowid']].get_payload(encoding='base64') else: final_msg = False if cmd == 'sniffdelete': result = self.snifferengine.delete_session( params['session_id']) final_msg = result if cmd == 'sniffpcap': result = self.snifferengine.commit_to_db(session) final_msg = result if cmd == 'call_module_function': module_name = params['module_name'] function = params['function'] args = params['args'] module = session.modules.get(module_name, None) if module: try: final_msg = getattr(module, function)(args) except Exception, e: import traceback final_msg = "[{} in function {}] Module error: {}\n".format( module_name, function, e) final_msg += traceback.format_exc() final_msg = "<pre>{}</pre>".format(final_msg) else: final_msg = False
def load_module_entry(self, session_id, module_name): entry = self.modules.find_one({'session_id': session_id, 'name': module_name}) if not entry or entry.get('timeout', datetime.datetime.utcnow() - datetime.timedelta(hours=24)) < datetime.datetime.utcnow(): return None else: return bson_loads(entry['entry'])
def command_handler(self, msg): self.command_lock.acquire() msg = json.loads(msg) cmd = msg['msg'] params = msg.get('params', {}) src = msg['src'] queryid = msg['queryid'] final_msg = None if cmd == 'newsession': _id = self.snifferengine.new_session(params) final_msg = bson_dumps(_id) if cmd == 'sessionlist': session_list = [] user = params.get('user', None) page = params.get('page', 0) private = params.get('private', False) for session in self.snifferengine.model.get_sniffer_sessions( private=private, username=user, page=page): if session['_id'] in self.snifferengine.sessions: session = self.snifferengine.sessions[session['_id']] active = session.status() session = session.__dict__ else: active = False session_data = bson_loads(session['session_data']) session['nodes'] = session_data['nodes'] session['edges'] = session_data['edges'] session['id'] = session['_id'] session_list.append({ 'id': str(session.get('id')), 'date_created': session.get('date_created'), 'name': session.get('name'), 'packets': session.get('packet_count'), 'nodes': len(session.get('nodes')), 'edges': len(session.get('edges')), 'status': "Running" if active else "Stopped", 'public': session.get('public'), }) final_msg = bson_dumps(session_list) if params.get('session_id', False): session = self.snifferengine.fetch_sniffer_session( params['session_id']) if not session: final_msg = False if session: if cmd == 'sessioninfo': final_msg = { 'name': session.name, 'filter': session.filter, 'pcap': session.pcap, 'pcap_filename': session.pcap_filename, 'id': str(session.id), 'public': session.public, 'status': session.status(), } if cmd == 'sniffstatus': final_msg = {'status': session.status()} if cmd == 'sniffupdate': # this needs to be stringyfied, or else encoding errors will ensue final_msg = session.update_nodes() final_msg = json.dumps(final_msg, default=json_util.default) if cmd == 'sniffstart': #self.snifferengine.start_session(params['session_name'], params['remote_addr']) session.start(params['remote_addr']) final_msg = True if cmd == 'sniffstop': session.stop() final_msg = True if cmd == 'flowstatus': flow = session.flow_status() # this needs to be stringyfied, or else encoding errors will ensue final_msg = flow if cmd == 'flow_statistics_update': print "Received 'flow_statistics_update' message. Please implement me? " if cmd == 'get_flow_payload': if params['flowid'] in session.flows: final_msg = session.flows[ params['flowid']].get_payload(encoding='base64') else: final_msg = False if cmd == 'sniffdelete': result = self.snifferengine.delete_session( params['session_id']) final_msg = result if cmd == 'sniffpcap': result = self.snifferengine.commit_to_db(session) final_msg = result if final_msg != None: reply = { 'msg': final_msg, 'queryid': queryid, 'dst': src, 'src': self.name } self.publish_to_channel('sniffer-commands', json.dumps(reply)) self.command_lock.release() return