class Utils: def __init__(self): self.Storage = Storage() def matchLFCChannel(self): def does_channel_match(ctx): lfc_settings = self.Storage.get_lfc_settings(ctx.guild) if lfc_settings['status']: if ctx.channel in lfc_settings['channels'] or len( lfc_settings['channels']) == 0: return True return False return commands.check(does_channel_match) def matchProfileChannel(self): def does_channel_match(ctx): profile_settings = self.Storage.get_profile_settings(ctx.guild) if profile_settings['status']: if ctx.channel in profile_settings['channels'] or len( profile_settings['channels']) == 0: return True return False return commands.check(does_channel_match) def activtyLoggingEnabled(self): def has_guild_enabled(ctx): return self.Storage.get_activity_logging_status(ctx.guild) return commands.check(has_guild_enabled)
def sample(self): env = self.env state = np.array(env.reset()) self.storage = Storage(self.config.rollout_length) cumm_return = 0 for _ in range(self.config.rollout_length): self.env.render() preditions = self.network(state) next_state, rewards, done, _ = env.step(preditions['actions']) #reward = reward_normlizer(reward) self.storage.add(preditions) cumm_return += rewards self.storage.add({'rewards': torch.from_numpy(np.asarray(rewards)).unsqueeze(-1).float(), 'mask': torch.from_numpy(np.asarray(1 - done)).unsqueeze(-1).float(), 'states': torch.from_numpy(state).float().unsqueeze(0)}) # if done : # state = np.array(env.reset()) # else : state = np.array(next_state) print('cummulative return', cumm_return) print('avg return', cumm_return / self.config.rollout_length) prediction = self.network(state) self.storage.add(prediction) self.storage.placeholder() advantages = torch.Tensor(np.zeros((1))) returns = preditions['values'].detach() return compute_gae(advantages, returns, self.storage, self.config)
class AutoVoice(commands.Cog, name='Auto-Voice'): def __init__(self, client): self.client = client self.Storage = Storage() self.created_channels = [] @commands.Cog.listener() async def on_voice_state_update(self, user: discord.Member, before, after): if after.channel: auto_voice_channel = self.Storage.get_auto_voice_channel( user.guild) if after.channel == auto_voice_channel: #Joined Get Ship Voice category = self.client.get_channel(after.channel.category_id) names = self.Storage.get_auto_voice_names(user.guild) names = channel_names if len(names) == 0 else names name = choice(names) pos = auto_voice_channel.position + 1 voice_channel = await category.create_voice_channel( name, reason='Created ship channel.', position=pos) await voice_channel.edit(position=pos, reason='Moved Ship Channel.') self.created_channels.append(voice_channel) await user.move_to(voice_channel, reason='Moved to created channel.') if before.channel in self.created_channels and len( before.channel.members ) == 0: #Left any of the created Voice Channels try: await before.channel.delete(reason='Ship unmanned.') except discord.errors.NotFound: pass self.created_channels.remove(before.channel)
def on_open(self, sid, data): self.init() data = json.loads(data) self.config.write("version", data["version"]) self.config.write("port", data["port"]) storage = Storage() last = storage.fetch_last_measurement_by_name(data["name"]) if last: now = arrow.now() if now.timestamp - int(last["timestamp"]) > 3600: match = re.match( ".+( [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2})$", data["name"]) if match: data["name"] = data["name"][:-len(match.group(1))] data["name"] += " " + arrow.now().format("YYYY-MM-DD HH:mm") self.config.write("name", data["name"]) try: self.config.write("rate", float(data["rate"])) except ValueError: pass self.emit("connecting") self.daemon.start()
def run(self): self.storage = Storage() self.config = Config() interface = Interface(port=self.config.read("port")) try: self.log("Connecting") interface.connect() self.emit("connected") self.log("Connected") while self.running: data = interface.read() self.log(json.dumps(data)) if data: data["name"] = self.config.read("name") self.update(data) self.storage.store_measurement(data) time.sleep(self.config.read("rate")) except (KeyboardInterrupt, SystemExit): raise except: logging.exception(sys.exc_info()[0]) self.emit("log", traceback.format_exc()) finally: interface.close() self.emit("disconnected") self.log("Disconnected") self.thread = None
def run(self): self.storage = Storage() self.config = Config() self.interface = Wrapper() try: self.log("Connecting") self.retry(self.interface.connect) self.emit("connected") self.log("Connected") while self.running: data = self.retry(self.interface.read) if isinstance(data, str): if data in ["disconnected", "connected"]: self.disconnect() return raise Exception(data) else: self.log(json.dumps(data)) if data: data["name"] = self.config.read("name") self.update(data) self.storage.store_measurement(data) sleep(self.config.read("rate")) except (KeyboardInterrupt, SystemExit): raise except: logging.exception(sys.exc_info()[0]) self.emit("log", traceback.format_exc()) self.emit("log-error") finally: self.disconnect()
def __init__(self, backend, on_receive, on_receive_interval): self.backed = backend self.on_receive = on_receive self.on_receive_interval = on_receive_interval self.storage = Storage() if self.storage.fetch_status() != "disconnected": self.storage.update_status("disconnected") self.loop = asyncio.new_event_loop()
def __init__(self, stop_crit, stop_value=None, group_by_keys=None, window_id=None, storage_mode = None): self.stop_crit = stop_crit self.window_id = cuid.cuid() if not window_id else window_id self.group_by_keys = group_by_keys if group_by_keys else [] self.stop_value = stop_value self.cache = Storage(storage_mode) self.existing_keys = set() self._count = None self.reset_window()
def run(self): self.storage = Storage() self.config = Config() interface = None version = self.config.read("version") if version.startswith("TC"): ble = True self.interface = self.backed.ble else: ble = False self.interface = interface = Interface( port=self.config.read("port")) if version == "UM25C": interface.enable_higher_resolution() try: self.log("Connecting") if ble: self.backed.ble.connect() else: interface.connect() self.emit("connected") self.log("Connected") while self.running: if ble: data = self.backed.ble.read() else: data = interface.read() self.log(json.dumps(data)) if data: data["name"] = self.config.read("name") self.update(data) self.storage.store_measurement(data) time.sleep(self.config.read("rate")) except (KeyboardInterrupt, SystemExit): raise except: logging.exception(sys.exc_info()[0]) self.emit("log", traceback.format_exc()) self.emit("log-error") finally: if ble: self.backed.ble.close() else: interface.close() self.emit("disconnected") self.log("Disconnected") self.thread = None
def setUp(self): env = { "MINIO_SECRET_KEY": "imagerie", "MINIO_ACCESS_KEY": "imagerie" } self.container = client.containers.run('minio/minio', name='preptest', command='server /data', remove=True, environment=env, detach=True, ports={'9000/tcp': 9002}) logger.debug(self.container) self.s3_storage = Storage(prep_job.s3_storage) self.gcs_storage = Storage(prep_job.gcs_storage) pth = os.path.join(PARENT_DIR, "resources/gummies2/") # self.gcs_storage.upload_path(pth,'{}'.format('projects')) self.s3_storage.upload_path(pth, '{}'.format('projects'))
class LFC(commands.Cog, name='Looking for Crew'): def __init__(self, client): self.client = client self.Storage = Storage() async def auto_remove(self, user: discord.User): await asyncio.sleep(7200) for guild in self.Storage.get_lfc_enabled_guilds(self.client): if user in guild.members: role = self.Storage.get_lfc_role(guild) try: await guild.get_member(user.id).remove_roles(role) except discord.Forbidden: continue @Utils.matchLFCChannel() @commands.command( brief='Sets the user into `Looking for Crew` status for 2 hours.', description='This command gives the user the set `Looking for Crew` role. \n\ You can only use this command in the set channels.\n\ Type ?nlfc once you\'re in a crew to avoid getting further notifications.\n\ The role will be automatically removed after 2 hours.', usage='?lfc' ) async def lfc(self, ctx): local_role = self.Storage.get_lfc_role(ctx.guild) if local_role not in ctx.author.roles: user = self.client.get_user(ctx.author.id) for guild in self.Storage.get_lfc_enabled_guilds(self.client): if user in guild.members: role = self.Storage.get_lfc_role(guild) try: await guild.get_member(user.id).add_roles(role) except discord.Forbidden: continue await ctx.send(f'{local_role.mention}, {ctx.author.mention} is now looking for crew.') self.client.loop.create_task(self.auto_remove(user)) @Utils.matchLFCChannel() @commands.command( brief='Removes the `Looking for Crew` status manually.', description='This removes the `Looking for Crew` status.\n\ If you are looking for a crew again use `?lfc`.', usage='?nlfc' ) async def nlfc(self, ctx): local_role = self.Storage.get_lfc_role(ctx.guild) if local_role in ctx.author.roles: user = self.client.get_user(ctx.author.id) for guild in self.Storage.get_lfc_enabled_guilds(self.client): if user in guild.members: role = self.Storage.get_lfc_role(guild) try: await guild.get_member(user.id).remove_roles(role) except discord.Forbidden: continue await ctx.send(f'{ctx.author.mention}, you are no longer looking for a crew.')
def __init__(self, client): self.client = client self.whois_messages = dict() self.prev = '⏪' self.next = '⏩' self.beginning = '⏮' self.end = '⏭' self.stop = '⏹' self.emojis = [ self.beginning, self.prev, self.next, self.end, self.stop ] self.Storage = Storage()
def run(self): self.storage = Storage() self.config = Config() self.interface = Wrapper() try: self.log("Connecting") self.retry(self.interface.connect) self.emit("connected") self.log("Connected") name = self.config.read("name") interval = float(self.config.read("rate")) version = self.config.read("version") session_id = self.storage.create_session(name, version) while self.running: begin = timer() data = self.retry(self.interface.read) if isinstance(data, str): if data in ["disconnected", "connected"]: self.disconnect() return raise Exception(data) else: self.log(json.dumps(data)) if data: data["session_id"] = session_id self.update(data, version) self.storage.store_measurement(data) measurement_runtime = timer() - begin sleep_time = interval - measurement_runtime if sleep_time > 0: sleep(sleep_time) except (KeyboardInterrupt, SystemExit): raise except: logging.exception(sys.exc_info()[0]) self.emit("log", traceback.format_exc()) self.emit("log-error") finally: self.disconnect()
def setUp(self): super(NmapToolTest, self).setUp() self.exploit = Exploit(exploit_id=1, name='test_name', risk_level=RiskLevel.NONE) self.exploit2 = Exploit(exploit_id=2, name='test_name', risk_level=RiskLevel.HIGH) self.exploit_conf_args = Exploit(exploit_id=3) self.exploit_conf_args.name = 'test_name2' self.exploit_conf_args.risk_level = RiskLevel.HIGH self.config = { 'scripts': { 'test_name': { 'args': 'test_args' }, 'test_name2': { 'args': MagicMock() } } } self.cfg = { 'tools': { 'nmap': { 'disable_scripts': [], }, 'common': { 'rate': 1337, 'http': { 'useragent': 'test_useragent' } } }, 'config_filename': '' } self.exploits = [self.exploit, self.exploit2] self.port = Port(number=13, transport_protocol=TransportProtocol.TCP, node=Node(node_id=1, ip=ipaddress.ip_address('127.0.0.1'))) self.port.scan = Scan(start=14, end=13) self.port.protocol = 'test_service' self.aucote = MagicMock(storage=Storage(":memory:")) self.context = ScanContext(aucote=self.aucote, scanner=MagicMock(scan=Scan())) self.nmap_tool = NmapTool(context=self.context, exploits=self.exploits, port=self.port, config=self.config)
def on_open(self, sid, data): self.init() data = json.loads(data) self.config.write("version", data["version"]) if "port" in data: self.config.write("port", data["port"]) if "ble_address" in data: self.config.write("ble_address", data["ble_address"]) else: data["ble_address"] = self.config.read("ble_address") storage = Storage() last = storage.fetch_last_measurement_by_name(data["name"]) if last: if time() - int(last["timestamp"]) > 3600: match = re.match( ".+( [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2})$", data["name"]) if match: data["name"] = data["name"][:-len(match.group(1))] data["name"] += " " + pendulum.now().format("YYYY-MM-DD HH:mm") if not data["name"]: data["name"] = "My measurement" self.config.write("name", data["name"]) try: self.config.write("rate", float(data["rate"])) except ValueError: pass tc_ble = data["version"].startswith( "TC") and "USB" not in data["version"] if tc_ble and ("ble_address" not in data or not data["ble_address"]): self.daemon.log("BLE address is missing. Select address in Setup") return self.emit("connecting") self.daemon.start()
def post(self): """ Executes a prep job to create an image corpus for training. Use this method to start a prep job. """ job_def = request.json job_def['process_json'] = True # Hardcode to process json file from project folder job = Job(job_def['name'],job_def) job.type = 'preprocess' dt = newdt.now() job.start_time = int(dt.timestamp()*1000) job.request = {'full_path': request.full_path,'remote_addr':request.remote_addr,'method':request.method} jb = aug_queue.enqueue( preprocess, job,job_timeout=-1,result_ttl=86400,ttl=-1) jb.meta['job_def'] = job_def dt = newdt.now() jb.meta['job_init_time'] = str(int(dt.timestamp()*1000)) jb.status = 'Running' jb.save_meta() json_str = job.to_json_string() st = { 'BUCKET' : job.bucket, 'USE_GCS' : job.use_gcs, 'ACCESS_KEY' : access_key, 'SECRET_KEY' : secret_key, 'S3_URL' : s3_url } storage = Storage(st) storage.upload_data(json_str,'jobs/running/{}_0_preprocess_r_{}.json'.format(str(job.start_time),jb.id),contentType='application/json') storage.upload_data(json_str,'jobs/all/{}_0_preprocess_r_{}.json'.format(str(job.start_time),jb.id),contentType='application/json') return { "status": jb.status, 'job_id': jb.id, 'meta':jb.meta},201
def __init__(self, client): self.client = client self.Storage = Storage() self.profile_messages = dict() self.profile_status = dict() self.steam_emoji = None self.xbox_emoji = None self.psn_emoji = None self.nintendo_emoji = None self.minecraft_emoji = None self.origin_emoji = None self.blizzard_emoji = None self.bethesda_emoji = None self.twitch_emoji = None self.mixer_emoji = None self.youtube_emoji = None self.twitter_emoji = None self.reddit_emoji = None self.itchio_emoji = None self.social_emoji = None self.sot_emoji = None self.game_emoji = '🎮' self.game_emoji_url = 'https://emojipedia-us.s3.dualstack.us-west-1.amazonaws.com/thumbs/120/google/146/video-game_1f3ae.png' self.stop_emoji = '⏹' self.emojis = list() self.social_platforms = [ 'twitch', 'mixer', 'youtube', 'twitter', 'reddit', 'itchio' ] self.gt_platforms = [ 'steam', 'xbox', 'psn', 'nintendo', 'minecraft', 'origin', 'blizzard', 'bethesda' ]
def __init__(self, exploits, kudu_queue, tools_config): self.exploits = exploits self._kudu_queue = kudu_queue signal.signal(signal.SIGINT, self.signal_handler) signal.signal(signal.SIGTERM, self.signal_handler) signal.signal(signal.SIGHUP, self.graceful_stop) self.load_tools(tools_config) self._storage = Storage(conn_string=cfg['storage.db'], nodes_limit=cfg['storage.max_nodes_query']) self.ioloop = IOLoop.current() self.topdis = Topdis(cfg['topdis.api.host'], cfg['topdis.api.port'], cfg['topdis.api.base']) self.async_task_managers = { TaskManagerType.SCANNER: AsyncTaskManager.instance(name=TaskManagerType.SCANNER.value, parallel_tasks=10), TaskManagerType.REGULAR: AsyncTaskManager.instance(name=TaskManagerType.REGULAR.value, parallel_tasks=cfg['service.scans.parallel_tasks']), TaskManagerType.QUICK: AsyncTaskManager.instance(name=TaskManagerType.QUICK.value, parallel_tasks=30) } for task_manager in self.async_task_managers.values(): throttling_consumer = ThrottlingConsumer(manager=task_manager) self.ioloop.add_callback(partial(cfg.add_rabbit_consumer, throttling_consumer)) self.ioloop.add_callback(throttling_consumer.consume) self.web_server = WebServer(self, cfg['service.api.v1.host'], cfg['service.api.v1.port'], path=cfg['service.api.path']) self._tftp_thread = TFTPThread(cfg['tftp.host'], cfg['tftp.port'], timeout=cfg['tftp.timeout'], min_port=cfg['tftp.min_port'], max_port=cfg['tftp.max_port']) self._storage_thread = StorageThread(storage=self._storage) self.scanners = [] if cfg.toucan: for task_manager in self.async_task_managers.values(): cfg.toucan_monitor.register_toucan_key(key='throttling.rate', add_prefix=False, default=1, callback=task_manager.change_throttling_toucan)
def __init__(self, command_prefix: typing.Union[str, typing.Callable[ [discord.Message], typing.Awaitable]] = None, **options): if not command_prefix: command_prefix = get_prefix super().__init__(command_prefix, **options) self.config = Config() self.logger = FakeLogger() self.db = Storage(self) self.commands_used = collections.Counter() with open("credentials.json", "r") as f: credentials = json.load(f) self.token = credentials["discord_token"] self.uptime = datetime.datetime.utcnow() self.loop.set_debug(True)
def post(self): """ Executes a training. Use this method to start a training. """ job_def = request.json job = Job(job_def['name'], job_def) job.type = 'train' dt = newdt.now() job.start_time = int(dt.timestamp() * 1000) job.request = { 'full_path': request.full_path, 'remote_addr': request.remote_addr, 'method': request.method } if hasattr(job, 'ml_engine') and job.ml_engine: jb = train_queue.enqueue(train_mlengine, job, job_timeout=-1, result_ttl=-1) else: jb = train_queue.enqueue(train_job_method, job, job_timeout=-1) jb.meta['job_init_time'] = str(int(dt.timestamp() * 1000)) jb.meta['job_def'] = job_def jb.save_meta() json_str = job.to_json_string() st = { 'BUCKET': job.bucket, 'USE_GCS': job.use_gcs, 'ACCESS_KEY': access_key, 'SECRET_KEY': secret_key, 'S3_URL': s3_url } storage = Storage(st) storage.upload_data(json_str, 'jobs/running/{}_0_train_r_{}.json'.format( str(job.start_time), jb.id), contentType='application/json') storage.upload_data(json_str, 'jobs/all/{}_0_train_r_{}.json'.format( str(job.start_time), jb.id), contentType='application/json') return { "status": jb.get_status(), 'job_id': jb.id, 'meta': jb.meta }, 201
def main(): env = gym.make(env_name) goal_object = Goal(min_position=env.env.min_position, max_position=env.env.max_position, step=0.2) meta_policy, target_meta_policy, policy, target_policy = get_policies( env, goal_object) optimizer = Optimizer(meta_policy, target_meta_policy, policy, target_policy, mini_batch_size, discount, learning_rate, update_epochs) episode_rewards = deque(maxlen=50) get_meta_epsilon = lambda episode: np.exp(-episode * e_meta_decay) get_epsilon = lambda episode: np.exp(-episode * e_decay) frame = 0 meta_frame = 0 for eps in range(0, n_eps + 1): print('Episode', eps) if eps % 5 == 0: episode_rewards.append( test_env(target_meta_policy, target_policy, goal_object)) print('Avg reward', np.mean(episode_rewards)) storage = Storage(device=device) storage_meta = Storage(device=device) meta_state = env.reset() state = meta_state.copy() state = torch.FloatTensor(state).to(device) done = False for step in range(10): goal = meta_policy.act(meta_state, get_meta_epsilon(meta_frame)) goal_value = torch.FloatTensor( [goal_object.get_goal(int(goal.item()))]).to(device) print('Predicted goal', goal_value) goal_reached = False for i in range(30): joint_state = torch.cat((state, goal_value), axis=0).to(device) with torch.no_grad(): action = policy.act(joint_state, get_epsilon(frame)) next_state, reward, done, _ = env.step(int(action.item())) intrinsic_reward = get_intrinsic_reward(goal_value, next_state) goal_reached = True if intrinsic_reward else False joint_next_state = np.concatenate( [next_state, [goal_value.item]], axis=0) storage.push(joint_state, action, intrinsic_reward, joint_next_state, done) extrinsic_reward += reward state = next_state state = torch.FloatTensor(state).to(device) frame += 1 if done or goal_reached: break storage_meta.push(meta_state, goal, extrinsic_reward, next_state, done) # do i really pass here unnormalized goal? meta_state = state meta_frame += 1 if done: break storage.compute() storage_meta.compute() loss_meta, loss = optimizer.update(storage_meta, storage) if eps % target_policy_update: target_meta_policy.load_state_dict(meta_policy.state_dict()) target_policy.load_state_dict(policy.state_dict()) with open('metrics.csv', 'a') as metrics: metrics.write('{},{}\n'.format(loss_meta, loss))
def setUp(self): self.maxDiff = None self.storage = Storage(conn_string=getenv('AUCOTE_TEST_POSTGRES')) self.scan = Scan(start=1, end=17, protocol=TransportProtocol.UDP, scanner='test_name') self.scan_1 = Scan(rowid=56, protocol=TransportProtocol.UDP, scanner='test_name', start=13, end=19) self.scan_2 = Scan(rowid=79, protocol=TransportProtocol.UDP, scanner='test_name', start=2, end=18) self.scan_3 = Scan(rowid=80, protocol=TransportProtocol.UDP, scanner='test_name_2', start=20, end=45) self.scan_4 = Scan(rowid=78, protocol=TransportProtocol.TCP, scanner='portdetection', start=1, end=2) self.node_1 = Node(node_id=1, ip=ipaddress.ip_address('127.0.0.1')) self.node_1.name = 'test_node_1' self.node_1.scan = Scan(start=15) self.node_2 = Node(node_id=2, ip=ipaddress.ip_address('127.0.0.2')) self.node_2.name = 'test_node_2' self.node_2.scan = Scan(start=56) self.node_3 = Node(node_id=3, ip=ipaddress.ip_address('127.0.0.3')) self.node_3.name = 'test_node_3' self.node_3.scan = Scan(start=98) self.node_4 = Node(node_id=4, ip=ipaddress.ip_address('127.0.0.4')) self.node_4.name = 'test_node_4' self.node_4.scan = Scan(start=3) self.node_scan_1 = NodeScan(node=self.node_1, rowid=13, scan=self.scan_1, timestamp=15) self.node_scan_2 = NodeScan(node=self.node_2, rowid=15, scan=self.scan_1, timestamp=56) self.node_scan_3 = NodeScan(node=self.node_3, rowid=16, scan=self.scan_1, timestamp=98) self.node_scan_3 = NodeScan(node=self.node_3, rowid=17, scan=self.scan_2, timestamp=90) self.port_1 = Port(node=self.node_1, number=45, transport_protocol=TransportProtocol.UDP) self.port_1.scan = self.scan_1 self.port_scan_1 = PortScan(port=self.port_1, scan=self.scan_1, timestamp=176, rowid=124) self.port_2 = Port(node=self.node_2, transport_protocol=TransportProtocol.UDP, number=65) self.port_2.scan = Scan(start=3, end=45) self.port_scan_2 = PortScan(port=self.port_2, scan=self.scan_1, timestamp=987, rowid=15) self.port_3 = Port(node=self.node_3, transport_protocol=TransportProtocol.ICMP, number=99) self.port_3.scan = Scan(start=43, end=180) self.port_scan_3 = PortScan(port=self.port_3, scan=self.scan_1, timestamp=619, rowid=13) self.port_4 = Port(node=self.node_1, number=80, transport_protocol=TransportProtocol.UDP) self.port_4.scan = self.scan_1 self.port_scan_4 = PortScan(port=self.port_4, scan=self.scan_1, timestamp=650, rowid=480) self.port_5 = Port(node=self.node_4, number=22, transport_protocol=TransportProtocol.TCP) self.port_5.scan = self.scan_4 self.exploit_1 = Exploit(exploit_id=14, name='test_name', app='test_app') self.exploit_2 = Exploit(exploit_id=2, name='test_name_2', app='test_app_2') self.exploit_3 = Exploit(exploit_id=56, name='test_name_2', app='test_app') self.exploit_4 = Exploit(exploit_id=0, name='portdetection', app='portdetection') self.security_scan_1 = SecurityScan(exploit=self.exploit_1, port=self.port_1, scan=self.scan_1, scan_start=178, scan_end=851) self.security_scan_2 = SecurityScan(exploit=self.exploit_2, port=self.port_1, scan=self.scan_1, scan_start=109, scan_end=775) self.security_scan_3 = SecurityScan(exploit=self.exploit_3, port=self.port_1, scan=self.scan_2, scan_start=113, scan_end=353) self.security_scan_4 = SecurityScan(exploit=self.exploit_1, port=self.port_1, scan=self.scan_3, scan_start=180, scan_end=222) self.security_scan_5 = SecurityScan(exploit=self.exploit_1, port=self.port_1, scan=self.scan_2, scan_start=14, scan_end=156) self.security_scan_6 = SecurityScan(exploit=self.exploit_2, port=self.port_1, scan=self.scan_2, scan_start=56, scan_end=780) self.security_scan_7 = SecurityScan(exploit=self.exploit_4, port=self.port_5, scan=self.scan_4, scan_start=14, scan_end=890) self.vuln_change_1 = PortDetectionChange( change_time=124445, current_finding=self.port_scan_1, previous_finding=self.port_scan_2) self.vuln_change_2 = PortDetectionChange( change_time=32434, current_finding=self.port_scan_2, previous_finding=self.port_scan_3) self.vulnerability_1 = Vulnerability(port=self.port_1, output='test_output_1', exploit=self.exploit_1, cve='CVE-2017', cvss=6.7, subid=1, vuln_time=13, rowid=134, scan=self.scan_1) self.vulnerability_2 = Vulnerability(port=self.port_1, output='test_output_2', exploit=self.exploit_2, cve='CWE-14', cvss=8.9, subid=2, vuln_time=98, rowid=152, scan=self.scan_1) self.vulnerability_3 = Vulnerability(port=self.port_1, output='test_output_3', exploit=self.exploit_1, cve='CVE-2016', cvss=3.7, subid=2, vuln_time=15, rowid=153, scan=self.scan_2) self.vulnerability_4 = Vulnerability(port=self.port_1, output='test_output_4', exploit=self.exploit_2, cve='CWE-15', cvss=2.9, subid=1, vuln_time=124, rowid=169, scan=self.scan_2) self.vulnerability_5 = Vulnerability(port=self.port_5, output='', exploit=self.exploit_4, cve=None, cvss=None, subid=0, vuln_time=124, rowid=200, scan=self.scan_4, expiration_time=400) self.vulnerability_6 = Vulnerability(port=self.port_5, output='tftp', exploit=self.exploit_4, cve=None, cvss=None, subid=1, vuln_time=124, rowid=201, scan=self.scan_4, expiration_time=400) self.vulnerability_7 = Vulnerability(port=self.port_5, output='tftp server name', exploit=self.exploit_4, cve=None, cvss=None, subid=2, vuln_time=124, rowid=202, scan=self.scan_4, expiration_time=400) self.vulnerability_8 = Vulnerability(port=self.port_5, output='6.7.8', exploit=self.exploit_4, cve=None, cvss=None, subid=3, vuln_time=124, rowid=203, scan=self.scan_4, expiration_time=400) self.vulnerability_9 = Vulnerability(port=self.port_5, output='HERE IS TFTP\n\n\n > ', exploit=self.exploit_4, cve=None, cvss=None, subid=4, vuln_time=124, rowid=204, scan=self.scan_4, expiration_time=400) self.vulnerability_10 = Vulnerability(port=self.port_5, output='test:cpe', exploit=self.exploit_4, cve=None, cvss=None, subid=5, vuln_time=124, rowid=205, scan=self.scan_4, expiration_time=400) self.vulnerability_11 = Vulnerability(port=self.port_5, output='os name', exploit=self.exploit_4, cve=None, cvss=None, subid=6, vuln_time=124, rowid=206, scan=self.scan_4, expiration_time=400) self.vulnerability_12 = Vulnerability(port=self.port_5, output='os version', exploit=self.exploit_4, cve=None, cvss=None, subid=7, vuln_time=124, rowid=207, scan=self.scan_4, expiration_time=400) self.vulnerability_13 = Vulnerability(port=self.port_5, output='test:os:cpe', exploit=self.exploit_4, cve=None, cvss=None, subid=8, vuln_time=124, rowid=208, scan=self.scan_4, expiration_time=400)
class StorageTest(TestCase): SEL_SEC_SCAN = ( "SELECT scan_id, exploit_id, exploit_app, exploit_name, node_id, node_ip, port_protocol, " "port_number, sec_scan_start, sec_scan_end FROM security_scans", ) SEL_NOD_SCAN = ( "SELECT scan_id, node_id, node_ip, time from nodes_scans", ) SEL_POR_SCAN = ( "SELECT scan_id, node_id, node_ip, port, port_protocol, time from ports_scans", ) SEL_CHANGE = ( "SELECT type, vulnerability_id, vulnerability_subid, previous_id, current_id, time FROM changes", ) SEL_SCANS = ( "SELECT ROWID, scan_start, scan_end, protocol, scanner_name FROM scans", ) SEL_VULNS = ( "SELECT scan_id, node_id, node_ip, port_protocol, port, vulnerability_id, vulnerability_subid, cve, cvss, output, time FROM vulnerabilities", ) def setUp(self): self.maxDiff = None self.storage = Storage(conn_string=getenv('AUCOTE_TEST_POSTGRES')) self.scan = Scan(start=1, end=17, protocol=TransportProtocol.UDP, scanner='test_name') self.scan_1 = Scan(rowid=56, protocol=TransportProtocol.UDP, scanner='test_name', start=13, end=19) self.scan_2 = Scan(rowid=79, protocol=TransportProtocol.UDP, scanner='test_name', start=2, end=18) self.scan_3 = Scan(rowid=80, protocol=TransportProtocol.UDP, scanner='test_name_2', start=20, end=45) self.scan_4 = Scan(rowid=78, protocol=TransportProtocol.TCP, scanner='portdetection', start=1, end=2) self.node_1 = Node(node_id=1, ip=ipaddress.ip_address('127.0.0.1')) self.node_1.name = 'test_node_1' self.node_1.scan = Scan(start=15) self.node_2 = Node(node_id=2, ip=ipaddress.ip_address('127.0.0.2')) self.node_2.name = 'test_node_2' self.node_2.scan = Scan(start=56) self.node_3 = Node(node_id=3, ip=ipaddress.ip_address('127.0.0.3')) self.node_3.name = 'test_node_3' self.node_3.scan = Scan(start=98) self.node_4 = Node(node_id=4, ip=ipaddress.ip_address('127.0.0.4')) self.node_4.name = 'test_node_4' self.node_4.scan = Scan(start=3) self.node_scan_1 = NodeScan(node=self.node_1, rowid=13, scan=self.scan_1, timestamp=15) self.node_scan_2 = NodeScan(node=self.node_2, rowid=15, scan=self.scan_1, timestamp=56) self.node_scan_3 = NodeScan(node=self.node_3, rowid=16, scan=self.scan_1, timestamp=98) self.node_scan_3 = NodeScan(node=self.node_3, rowid=17, scan=self.scan_2, timestamp=90) self.port_1 = Port(node=self.node_1, number=45, transport_protocol=TransportProtocol.UDP) self.port_1.scan = self.scan_1 self.port_scan_1 = PortScan(port=self.port_1, scan=self.scan_1, timestamp=176, rowid=124) self.port_2 = Port(node=self.node_2, transport_protocol=TransportProtocol.UDP, number=65) self.port_2.scan = Scan(start=3, end=45) self.port_scan_2 = PortScan(port=self.port_2, scan=self.scan_1, timestamp=987, rowid=15) self.port_3 = Port(node=self.node_3, transport_protocol=TransportProtocol.ICMP, number=99) self.port_3.scan = Scan(start=43, end=180) self.port_scan_3 = PortScan(port=self.port_3, scan=self.scan_1, timestamp=619, rowid=13) self.port_4 = Port(node=self.node_1, number=80, transport_protocol=TransportProtocol.UDP) self.port_4.scan = self.scan_1 self.port_scan_4 = PortScan(port=self.port_4, scan=self.scan_1, timestamp=650, rowid=480) self.port_5 = Port(node=self.node_4, number=22, transport_protocol=TransportProtocol.TCP) self.port_5.scan = self.scan_4 self.exploit_1 = Exploit(exploit_id=14, name='test_name', app='test_app') self.exploit_2 = Exploit(exploit_id=2, name='test_name_2', app='test_app_2') self.exploit_3 = Exploit(exploit_id=56, name='test_name_2', app='test_app') self.exploit_4 = Exploit(exploit_id=0, name='portdetection', app='portdetection') self.security_scan_1 = SecurityScan(exploit=self.exploit_1, port=self.port_1, scan=self.scan_1, scan_start=178, scan_end=851) self.security_scan_2 = SecurityScan(exploit=self.exploit_2, port=self.port_1, scan=self.scan_1, scan_start=109, scan_end=775) self.security_scan_3 = SecurityScan(exploit=self.exploit_3, port=self.port_1, scan=self.scan_2, scan_start=113, scan_end=353) self.security_scan_4 = SecurityScan(exploit=self.exploit_1, port=self.port_1, scan=self.scan_3, scan_start=180, scan_end=222) self.security_scan_5 = SecurityScan(exploit=self.exploit_1, port=self.port_1, scan=self.scan_2, scan_start=14, scan_end=156) self.security_scan_6 = SecurityScan(exploit=self.exploit_2, port=self.port_1, scan=self.scan_2, scan_start=56, scan_end=780) self.security_scan_7 = SecurityScan(exploit=self.exploit_4, port=self.port_5, scan=self.scan_4, scan_start=14, scan_end=890) self.vuln_change_1 = PortDetectionChange( change_time=124445, current_finding=self.port_scan_1, previous_finding=self.port_scan_2) self.vuln_change_2 = PortDetectionChange( change_time=32434, current_finding=self.port_scan_2, previous_finding=self.port_scan_3) self.vulnerability_1 = Vulnerability(port=self.port_1, output='test_output_1', exploit=self.exploit_1, cve='CVE-2017', cvss=6.7, subid=1, vuln_time=13, rowid=134, scan=self.scan_1) self.vulnerability_2 = Vulnerability(port=self.port_1, output='test_output_2', exploit=self.exploit_2, cve='CWE-14', cvss=8.9, subid=2, vuln_time=98, rowid=152, scan=self.scan_1) self.vulnerability_3 = Vulnerability(port=self.port_1, output='test_output_3', exploit=self.exploit_1, cve='CVE-2016', cvss=3.7, subid=2, vuln_time=15, rowid=153, scan=self.scan_2) self.vulnerability_4 = Vulnerability(port=self.port_1, output='test_output_4', exploit=self.exploit_2, cve='CWE-15', cvss=2.9, subid=1, vuln_time=124, rowid=169, scan=self.scan_2) self.vulnerability_5 = Vulnerability(port=self.port_5, output='', exploit=self.exploit_4, cve=None, cvss=None, subid=0, vuln_time=124, rowid=200, scan=self.scan_4, expiration_time=400) self.vulnerability_6 = Vulnerability(port=self.port_5, output='tftp', exploit=self.exploit_4, cve=None, cvss=None, subid=1, vuln_time=124, rowid=201, scan=self.scan_4, expiration_time=400) self.vulnerability_7 = Vulnerability(port=self.port_5, output='tftp server name', exploit=self.exploit_4, cve=None, cvss=None, subid=2, vuln_time=124, rowid=202, scan=self.scan_4, expiration_time=400) self.vulnerability_8 = Vulnerability(port=self.port_5, output='6.7.8', exploit=self.exploit_4, cve=None, cvss=None, subid=3, vuln_time=124, rowid=203, scan=self.scan_4, expiration_time=400) self.vulnerability_9 = Vulnerability(port=self.port_5, output='HERE IS TFTP\n\n\n > ', exploit=self.exploit_4, cve=None, cvss=None, subid=4, vuln_time=124, rowid=204, scan=self.scan_4, expiration_time=400) self.vulnerability_10 = Vulnerability(port=self.port_5, output='test:cpe', exploit=self.exploit_4, cve=None, cvss=None, subid=5, vuln_time=124, rowid=205, scan=self.scan_4, expiration_time=400) self.vulnerability_11 = Vulnerability(port=self.port_5, output='os name', exploit=self.exploit_4, cve=None, cvss=None, subid=6, vuln_time=124, rowid=206, scan=self.scan_4, expiration_time=400) self.vulnerability_12 = Vulnerability(port=self.port_5, output='os version', exploit=self.exploit_4, cve=None, cvss=None, subid=7, vuln_time=124, rowid=207, scan=self.scan_4, expiration_time=400) self.vulnerability_13 = Vulnerability(port=self.port_5, output='test:os:cpe', exploit=self.exploit_4, cve=None, cvss=None, subid=8, vuln_time=124, rowid=208, scan=self.scan_4, expiration_time=400) def prepare_tables(self): self.storage.connect() self.storage.remove_all() self.storage.init_schema() self.prepare_scans() self.prepare_nodes_scans() self.prepare_ports_scans() self.prepare_vulnerabilities() self.prepare_security_scans() def prepare_scans(self): self.storage.execute(( "INSERT INTO scans(ROWID, protocol, scanner_name, scan_start, scan_end) " "VALUES " "(56, 17, 'test_name', 13, 19), " "(80, 17, 'test_name_2', 20, 45), " "(79, 17, 'test_name', 2, 18)," "(78, 6, 'portdetection', 1, 2)", )) def prepare_nodes_scans(self): self.storage.execute(( "INSERT INTO nodes_scans (ROWID, scan_id, node_id, node_ip, time) VALUES " "(13, 56, 1, '127.0.0.1', 15), " "(15, 56, 2, '127.0.0.2', 56), " "(16, 56, 3, '127.0.0.3', 98), " "(17, 79, 3, '127.0.0.3', 90) " "", )) def prepare_ports_scans(self): self.storage.execute(( 'INSERT INTO ports_scans(ROWID, scan_id, node_id, node_ip, port, port_protocol, time) ' 'VALUES ' "(124, 56, 1, '127.0.0.1', 45, 17, 176), " "(480, 56, 1, '127.0.0.1', 80, 17, 650), " "(13, 56, 3, '127.0.0.3', 99, 1, 619), " "(15, 56, 2, '127.0.0.2', 65, 17, 987)", )) def prepare_security_scans(self): self.storage.execute(( "INSERT INTO security_scans(exploit_id, exploit_name, exploit_app, scan_id, node_id, " "node_ip, port_number, port_protocol, sec_scan_start, sec_scan_end) VALUES " "(14, 'test_name', 'test_app', 56, 1, '127.0.0.1', 45, 17, 178, 851), " "(56, 'test_name_2', 'test_app', 79, 1, '127.0.0.1', 45, 17, 113, 353), " "(14, 'test_name', 'test_app', 80, 1, '127.0.0.1', 45, 17, 180, 222), " "(2, 'test_name_2', 'test_app_2', 56, 1, '127.0.0.1', 45, 17, 109, 775), " "(14, 'test_name', 'test_app', 79, 1, '127.0.0.1', 45, 17, 14, 156), " "(2, 'test_name_2', 'test_app_2', 79, 1, '127.0.0.1', 45, 17, 56, 780), " "(0, 'portdetection', 'portdetection', 78, 4, '127.0.0.4', 22, 6, 14, 890)", )) def prepare_vulnerabilities(self): self.storage.execute(( "INSERT INTO vulnerabilities (ROWID, scan_id, node_id, node_ip, port_protocol, port, " "vulnerability_id, vulnerability_subid, cve, cvss, output, time, expiration_time) VALUES " "(134, 56, 1, '127.0.0.1', 17, 45, 14, 1, 'CVE-2017', 6.7, 'test_output_1', 13, 600)," "(152, 56, 1, '127.0.0.1', 17, 45, 2, 2, 'CWE-14', 8.9, 'test_output_2', 98, 600)," "(153, 79, 1, '127.0.0.1', 17, 45, 14, 2, 'CVE-2016', 3.7, 'test_output_3', 15, NULL)," "(169, 79, 1, '127.0.0.1', 17, 45, 2, 1, 'CWE-15', 2.9, 'test_output_4', 124, 600), " "(200, 78, 4, '127.0.0.4', 6, 22, 0, 0, NULL, NULL, '', 124, 400), " "(201, 78, 4, '127.0.0.4', 6, 22, 0, 1, NULL, NULL, 'tftp', 124, 400), " "(202, 78, 4, '127.0.0.4', 6, 22, 0, 2, NULL, NULL, 'tftp server name', 124, 400), " "(203, 78, 4, '127.0.0.4', 6, 22, 0, 3, NULL, NULL, '6.7.8', 124, 400), " "(204, 78, 4, '127.0.0.4', 6, 22, 0, 4, NULL, NULL, 'HERE IS TFTP\n\n\n > ', 124, 400), " "(205, 78, 4, '127.0.0.4', 6, 22, 0, 5, NULL, NULL, 'test:cpe', 124, 400), " "(206, 78, 4, '127.0.0.4', 6, 22, 0, 6, NULL, NULL, 'os name', 124, 400), " "(207, 78, 4, '127.0.0.4', 6, 22, 0, 7, NULL, NULL, 'os version', 124, 400), " "(208, 78, 4, '127.0.0.4', 6, 22, 0, 8, NULL, NULL, 'test:os:cpe', 124, 400)" "", )) def test_context_manager(self): with self.storage: self.assertTrue(True) def test_connect(self): self.storage.connect() self.assertIsInstance(self.storage.conn, connection) def test_close(self): self.storage.conn = connect(":memory:") self.storage.close() self.assertEqual(self.storage.conn, None) def test_cursor_property(self): self.assertEqual(self.storage.cursor, self.storage._cursor) @patch('utils.storage.time.time', MagicMock(return_value=134)) def test_save_nodes(self): self.storage.connect() self.storage.remove_all() self.storage.init_schema() expected = [(56, 1, '127.0.0.1', 134), (56, 2, '127.0.0.2', 134), (56, 3, '127.0.0.3', 134)] self.storage.save_nodes(nodes=[self.node_1, self.node_2, self.node_3], scan=self.scan_1) result = self.storage.execute(self.SEL_NOD_SCAN) self.assertEqual(result, expected) @patch('utils.storage.time.time', MagicMock(return_value=60)) def test_get_nodes(self): self.prepare_tables() expected = [self.node_2, self.node_3, self.node_3] result = self.storage.get_nodes(pasttime=10, scan=self.scan_1) self.assertCountEqual(result, expected) def test_get_vulnerabilities(self): self.prepare_tables() expected = [self.vulnerability_1] result = self.storage.get_vulnerabilities(port=self.port_1, scan=self.scan_1, exploit=self.exploit_1) self.assertCountEqual(result, expected) def test_get_nodes_by_scan(self): self.prepare_tables() expected = [self.node_1, self.node_2, self.node_3] result = self.storage.get_nodes_by_scan(scan=self.scan_1) self.assertCountEqual(result, expected) @patch('utils.storage.time.time', MagicMock(return_value=167)) def test_save_port(self): self.storage.connect() self.storage.remove_all() self.storage.init_schema() expected = [(56, 1, '127.0.0.1', 45, 17, 167)] self.storage.save_port(port=self.port_1, scan=self.scan_1) result = self.storage.execute(self.SEL_POR_SCAN) self.assertEqual(result, expected) @patch('utils.storage.time.time', MagicMock(return_value=167)) def test_save_ports(self): self.storage.connect() self.storage.remove_all() self.storage.init_schema() expected = [(56, 1, '127.0.0.1', 45, 17, 167), (56, 2, '127.0.0.2', 65, 17, 167), (56, 3, '127.0.0.3', 99, 1, 167)] self.storage.save_ports(ports=[self.port_1, self.port_2, self.port_3], scan=self.scan_1) result = self.storage.execute(self.SEL_POR_SCAN) self.assertEqual(result, expected) @patch('utils.storage.time.time', MagicMock(return_value=1000)) def test_get_ports(self): self.prepare_tables() expected = [ self.port_scan_2.port, self.port_scan_3.port, self.port_scan_4.port ] result = self.storage.get_ports(pasttime=400, scan=self.scan_1) self.assertCountEqual(result, expected) def test_get_ports_by_scan_and_node(self): self.prepare_tables() expected = [self.port_scan_1, self.port_scan_4] result = self.storage.get_ports_by_scan_and_node(node=self.node_1, scan=self.scan_1) self.assertCountEqual(result, expected) def test_save_security_scan(self): self.storage.connect() self.storage.remove_all() self.storage.init_schema() expected = [(56, 14, 'test_app', 'test_name', 1, '127.0.0.1', 17, 45, 13.0, 19.0)] self.storage.save_security_scan(exploit=self.exploit_1, port=self.port_1, scan=self.scan_1) result = self.storage.execute(self.SEL_SEC_SCAN) self.assertEqual(result, expected) def test_save_security_scan_twice(self): self.storage.connect() self.storage.remove_all() self.storage.init_schema() self.port_1.scan.start = None expected = [(56, 14, 'test_app', 'test_name', 1, '127.0.0.1', 17, 45, None, 19.0)] self.storage.save_security_scan(exploit=self.exploit_1, port=self.port_1, scan=self.scan_1) result = self.storage.execute(self.SEL_SEC_SCAN) self.assertEqual(result, expected) self.port_1.scan.start = 13 self.port_1.scan.end = None expected = [(56, 14, 'test_app', 'test_name', 1, '127.0.0.1', 17, 45, 13.0, 19.0)] self.storage.save_security_scan(exploit=self.exploit_1, port=self.port_1, scan=self.scan_1) result = self.storage.execute(self.SEL_SEC_SCAN) self.assertEqual(result, expected) def test_save_security_scans(self): self.storage.connect() self.storage.remove_all() self.storage.init_schema() expected = [(56, 14, 'test_app', 'test_name', 1, '127.0.0.1', 17, 45, 13.0, 19.0), (56, 2, 'test_app_2', 'test_name_2', 1, '127.0.0.1', 17, 45, 13.0, 19.0), (56, 56, 'test_app', 'test_name_2', 1, '127.0.0.1', 17, 45, 13.0, 19.0)] self.storage.save_security_scans( exploits=[self.exploit_1, self.exploit_2, self.exploit_3], port=self.port_1, scan=self.scan_1) result = self.storage.execute(self.SEL_SEC_SCAN) self.assertEqual(result, expected) def test_save_changes(self): self.storage.connect() self.storage.remove_all() self.storage.init_schema() expected = [(1, 0, 0, 15, 124, 124445), (1, 0, 0, 13, 15, 32434)] self.storage.save_changes( changes=[self.vuln_change_1, self.vuln_change_2]) result = self.storage.execute(self.SEL_CHANGE) self.assertEqual(result, expected) def test_get_ports_by_nodes_without_nodes(self): expected = [] result = self.storage.get_ports_by_nodes([]) self.assertEqual(result, expected) @patch('utils.storage.time.time', MagicMock(return_value=1000)) def test_get_ports_by_nodes(self): self.storage.nodes_limit = 2 self.prepare_tables() expected = [self.port_scan_2.port, self.port_scan_4.port] result = self.storage.get_ports_by_nodes( nodes=[self.node_1, self.node_3, self.node_2], pasttime=400, protocol=TransportProtocol.UDP) self.assertCountEqual(result, expected) def test_execute_query(self): query = "part_1", "arg_1", "arg_2" self.storage._cursor = MagicMock() self.storage._cursor.rowcount = 2 self.storage.conn = MagicMock() result = self.storage.execute(query) self.storage.cursor.execute.assert_called_once_with( "part_1", "arg_1", "arg_2") self.storage.cursor.fetchall.assert_called_once_with() self.assertEqual(result, self.storage.cursor.fetchall()) def test_execute_query_list(self): queries = [("part_1", "arg_1", "arg_2"), ("part_2", "arg_3")] self.storage._cursor = MagicMock() self.storage._cursor.rowcount = 2 self.storage.conn = MagicMock() self.storage.execute(queries) self.storage.cursor.execute.assert_has_calls( (call("part_1", "arg_1", "arg_2"), call("part_2", "arg_3"))) def test_transport_protocol_none(self): result = self.storage._transport_protocol(None) self.assertIsNone(result) @patch('utils.storage.TransportProtocol') def test_transport_protocol(self, protocol): result = self.storage._transport_protocol(6) protocol.from_iana.assert_called_once_with(6) self.assertEqual(result, protocol.from_iana()) def test_protocol_to_iana_none(self): protocol = None result = self.storage._protocol_to_iana(protocol) self.assertIsNone(result) def test_save_scan(self): self.storage.connect() self.storage.remove_all() self.storage.init_schema() expected = [(1, 13, 19, 17, 'test_name')] self.storage.save_scan(scan=self.scan_1) result = self.storage.execute(self.SEL_SCANS) self.assertEqual(result, expected) def test_update_scan(self): self.prepare_tables() self.scan_1.end = 456 self.storage.update_scan(self.scan_1) expected = [(456, )] result = self.storage.execute( ("SELECT scan_end FROM scans WHERE ROWID=56", )) self.assertEqual(result, expected) def test_get_scan_id(self): expected = 56 result = self.storage.get_scan_id(self.scan_1) self.assertEqual(result, expected) def test_get_scan_id(self): self.prepare_tables() expected = self.scan_1.rowid self.scan_1.rowid = None result = self.storage.get_scan_id(self.scan_1) self.assertEqual(result, expected) def test_get_scan_id_without_results(self): self.prepare_tables() expected = None self.scan_1.rowid = None self.scan_1._scanner = None result = self.storage.get_scan_id(self.scan_1) self.assertEqual(result, expected) def test_get_scans(self): self.prepare_tables() result = self.storage.get_scans(scanner_name='test_name', protocol=TransportProtocol.UDP, amount=2) self.assertEqual([self.scan_1, self.scan_2], result) for obj in self.scan_1, self.scan_2: self.assertEqual(result[result.index(obj)].rowid, obj.rowid) def test_get_scans_by_node(self): self.prepare_tables() expected = [self.scan_1, self.scan_2] result = self.storage.get_scans_by_node(node=self.node_3, scan=self.scan_1) self.assertEqual(result, expected) def test_get_scans_by_sec_scan(self): self.prepare_tables() expected = [self.scan_3, self.scan_1, self.scan_2] result = self.storage.get_scans_by_security_scan( port=self.port_1, exploit=self.exploit_1) self.assertEqual(result, expected) def test_get_scans_by_sec_scan_second(self): self.prepare_tables() expected = [self.scan_2] result = self.storage.get_scans_by_security_scan( port=self.port_1, exploit=self.exploit_3) self.assertEqual(result, expected) def test_get_scan_by_id(self): self.prepare_tables() expected = self.scan_1 result = self.storage.get_scan_by_id(56) self.assertEqual(result, expected) def test_get_scan_by_id_no_results(self): self.storage.connect() self.storage.init_schema() result = self.storage.get_scan_by_id(15) self.assertEqual(result, None) def test_save_vulnerabilities(self): self.storage.connect() self.storage.remove_all() self.storage.init_schema() expected = [(56, 1, '127.0.0.1', 17, 45, 14, 1, 'CVE-2017', '6.7', 'test_output_1', 13), (56, 1, '127.0.0.1', 17, 45, 14, 2, 'CVE-2016', '3.7', 'test_output_3', 15)] self.storage.save_vulnerabilities( vulnerabilities=[self.vulnerability_1, self.vulnerability_3], scan=self.scan_1) result = self.storage.execute(self.SEL_VULNS) self.assertCountEqual(result, expected) def test_scans(self): self.prepare_tables() self.assertEqual(self.storage.scans(2, 0), [self.scan_3, self.scan_1]) self.assertEqual(self.storage.scans(2, 1), [self.scan_2, self.scan_4]) def ports_scans_by_scan(self): self.prepare_tables() expected = [ self.port_scan_1, self.port_scan_2, self.port_scan_3, self.port_scan_4 ] result = self.storage.ports_scans_by_scan(self.scan_1) def test_expire_vulnerability(self): self.prepare_tables() result = self.storage.expire_vulnerability(self.vulnerability_3) self.assertEqual(result.expiration_time, 178) def test_active_vulnerabilities(self): self.prepare_tables() expected = [self.vulnerability_3] result = self.storage.active_vulnerabilities() self.assertEqual(result, expected) def test_expire_vulnerabilities(self): self.prepare_tables() expected = [] self.storage.expire_vulnerabilities() result = self.storage.active_vulnerabilities(timestamp=400) self.assertEqual(result, expected) def test_active_vulnerabilities_different_timestamps(self): self.prepare_tables() expected = [self.vulnerability_3] result = self.storage.active_vulnerabilities(timestamp=0) self.assertEqual(result, expected) expected = [] result = self.storage.active_vulnerabilities(timestamp=40) self.assertEqual(result, expected) def test_portdetection_vulns(self): self.prepare_tables() expected = { 'name': 'tftp server name', 'version': '6.7.8', 'banner': 'HERE IS TFTP\n\n\n > ', 'cpe': 'test:cpe', 'protocol': 'tftp', 'os_name': 'os name', 'os_version': 'os version', 'os_cpe': 'test:os:cpe' } result = self.storage.portdetection_vulns(self.vulnerability_5) self.assertEqual(result, expected)
def __init__(self, backend): self.backed = backend self.storage = Storage() if self.storage.fetch_status() != "disconnected": self.storage.update_status("disconnected")
class Daemon: running = None thread = None storage = None config = None def __init__(self, backend): self.backed = backend self.storage = Storage() if self.storage.fetch_status() != "disconnected": self.storage.update_status("disconnected") def start(self): self.running = True if self.thread is None: self.thread = Thread(target=self.run) if not self.thread.is_alive(): self.thread.start() def stop(self): self.log("Disconnecting") self.running = False while self.thread and self.thread.is_alive(): time.sleep(0.1) self.emit("disconnected") def run(self): self.storage = Storage() self.config = Config() interface = Interface(port=self.config.read("port")) try: self.log("Connecting") interface.connect() self.emit("connected") self.log("Connected") while self.running: data = interface.read() self.log(json.dumps(data)) if data: data["name"] = self.config.read("name") self.update(data) self.storage.store_measurement(data) time.sleep(self.config.read("rate")) except (KeyboardInterrupt, SystemExit): raise except: logging.exception(sys.exc_info()[0]) self.emit("log", traceback.format_exc()) finally: interface.close() self.emit("disconnected") self.log("Disconnected") self.thread = None def update(self, data): format = Format() table = [] for name in format.table_fields: callback = getattr(format, name) table.append(callback(data)) graph = {} for name in format.graph_fields: if name == "timestamp": callback = getattr(format, name) value = callback(data) else: value = data[name] graph[name] = value self.emit("update", json.dumps({ "table": table, "graph": graph, })) def emit(self, event, data=None): if event == "log": self.storage.log(data) elif event in [ "connecting", "connected", "disconnecting", "disconnected" ]: self.storage.update_status(event) self.backed.emit(event, data) def log(self, message): prefix = arrow.now().format("YYYY-MM-DD HH:mm:ss") + " - " self.emit("log", prefix + message + "\n")
def __init__(self, name): self.name = name self._event_storage = Storage()
def __init__(self, client): self.client = client self.Storage = Storage() self.created_channels = []
class Index: config = None storage = None def __init__(self): pass def register(self): blueprint = Blueprint("index", __name__, template_folder="templates") blueprint.add_url_rule("/", "default", self.render_default) blueprint.add_url_rule("/data", "data", self.render_data) blueprint.add_url_rule("/graph", "graph", self.render_graph) blueprint.add_url_rule("/graph.json", "graph_data", self.render_graph_data) blueprint.add_url_rule("/ble", "ble", self.render_ble) blueprint.add_url_rule("/serial", "serial", self.render_serial) blueprint.context_processor(self.fill) return blueprint def init(self): self.config = Config() self.storage = Storage() def fill(self): variables = { "rd_user_version": version, "format": Format(), "url_for": self.url_for, "version": self.config.read("version", "UM34C"), "port": self.config.read("port", ""), "rate": str(self.config.read("rate", 1.0)), "name": self.config.read("name", arrow.now().format("YYYY-MM-DD")), "ble_address": self.config.read("ble_address"), } status = self.storage.fetch_status() variables["status"] = status.title() variables["connect_disabled"] = status != "disconnected" variables[ "connect_button"] = "Connect" if status == "disconnected" else "Disconnect" return variables def render_default(self): self.init() self.storage.clear_log() log = self.storage.fetch_log() return render_template("default.html", log=log, page="default") def render_data(self): self.init() names, selected = self.prepare_selection() name = self.storage.translate_selected_name(selected) if request.args.get("export") == "": string = io.StringIO() writer = csv.writer(string) format = Format() names = [] for field in format.export_fields: names.append(format.field_name(field)) writer.writerow(names) for item in self.storage.fetch_measurements(name): values = [] for field in format.export_fields: if field == "time": values.append(format.time(item)) else: values.append(item[field]) writer.writerow(values) output = make_response(string.getvalue()) output.headers[ "Content-Disposition"] = "attachment; filename=" + name + ".csv" output.headers["Content-type"] = "text/csv" return output elif request.args.get("destroy") == "": self.storage.destroy_measurements(name) flash("Measurements with session name '" + name + "' were deleted", "danger") return redirect(request.path) page = request.args.get("page", 1, int) limit = 100 offset = limit * (page - 1) count = self.storage.fetch_measurements_count(name) pages = self.prepare_pages(name, page, limit, count) measurements = self.storage.fetch_measurements(name, limit, offset) return render_template( "data.html", names=names, selected=selected, measurements=measurements, page="data", pages=pages, ) def prepare_pages(self, name, page, limit, count, blocks=10): first_page = 1 related = 3 last_page = int(ceil(count / limit)) steps = set( range(max((first_page, page - related)), min((last_page, page + related)) + 1)) quotient = (last_page - 1) / blocks if len(steps) > 1: for index in range(0, blocks): steps.add(round(quotient * index) + first_page) steps.add(last_page) steps = sorted(steps) pages = [] for number in steps: pages.append({ "number": number, "link": url_for("index.data", page=number, name=name), "current": number == page, }) return pages def render_graph(self): self.init() names, selected = self.prepare_selection() last_measurement = None if selected == "": last_measurement = self.storage.fetch_last_measurement() return render_template("graph.html", names=names, selected=selected, item=last_measurement, left_axis="voltage", right_axis="current", page="graph") def render_graph_data(self): self.init() selected = request.args.get("name") name = self.storage.translate_selected_name(selected) left_axis = request.args.get("left_axis") right_axis = request.args.get("right_axis") format = Format() data = [] for item in self.storage.fetch_measurements(name): if left_axis in item: data.append({ "date": format.timestamp(item), "left": item[left_axis], "right": item[right_axis], }) return jsonify(data) def prepare_selection(self): names = self.storage.fetch_measurement_names() selected = request.args.get("name") if not selected: selected = "" return names, selected def fill_config_from_parameters(self): value = request.args.get("version") if value is not None: self.config.write("version", value) value = request.args.get("name") if value is not None: self.config.write("name", value) value = request.args.get("rate") if value is not None: self.config.write("rate", float(value)) def render_ble(self): self.init() self.fill_config_from_parameters() return render_template("ble.html") def render_serial(self): self.init() self.fill_config_from_parameters() return render_template("serial.html") def url_for(self, endpoint, **values): if endpoint == "static": filename = values.get("filename", None) if filename: file_path = static_path + "/" + filename values["v"] = int(os.stat(file_path).st_mtime) return url_for(endpoint, **values)
def init(self): self.config = Config() self.storage = Storage()
def test_eval_image(self): s = Storage(cfg.config) evts = tflogs.get_events(cfg.folder,'/tmp',s,full=True) print(len(evts)) print(evts)