class Utils:
def __init__(self):
self.Storage = Storage()
def matchLFCChannel(self):
def does_channel_match(ctx):
lfc_settings = self.Storage.get_lfc_settings(ctx.guild)
if lfc_settings['status']:
if ctx.channel in lfc_settings['channels'] or len(
lfc_settings['channels']) == 0:
return True
return False
return commands.check(does_channel_match)
def matchProfileChannel(self):
def does_channel_match(ctx):
profile_settings = self.Storage.get_profile_settings(ctx.guild)
if profile_settings['status']:
if ctx.channel in profile_settings['channels'] or len(
profile_settings['channels']) == 0:
return True
return False
return commands.check(does_channel_match)
def activtyLoggingEnabled(self):
def has_guild_enabled(ctx):
return self.Storage.get_activity_logging_status(ctx.guild)
return commands.check(has_guild_enabled)
def sample(self):
env = self.env
state = np.array(env.reset())
self.storage = Storage(self.config.rollout_length)
cumm_return = 0
for _ in range(self.config.rollout_length):
self.env.render()
preditions = self.network(state)
next_state, rewards, done, _ = env.step(preditions['actions'])
#reward = reward_normlizer(reward)
self.storage.add(preditions)
cumm_return += rewards
self.storage.add({'rewards': torch.from_numpy(np.asarray(rewards)).unsqueeze(-1).float(),
'mask': torch.from_numpy(np.asarray(1 - done)).unsqueeze(-1).float(),
'states': torch.from_numpy(state).float().unsqueeze(0)})
# if done :
# state = np.array(env.reset())
# else :
state = np.array(next_state)
print('cummulative return', cumm_return)
print('avg return', cumm_return / self.config.rollout_length)
prediction = self.network(state)
self.storage.add(prediction)
self.storage.placeholder()
advantages = torch.Tensor(np.zeros((1)))
returns = preditions['values'].detach()
return compute_gae(advantages, returns, self.storage, self.config)
class AutoVoice(commands.Cog, name='Auto-Voice'):
def __init__(self, client):
self.client = client
self.Storage = Storage()
self.created_channels = []
@commands.Cog.listener()
async def on_voice_state_update(self, user: discord.Member, before, after):
if after.channel:
auto_voice_channel = self.Storage.get_auto_voice_channel(
user.guild)
if after.channel == auto_voice_channel: #Joined Get Ship Voice
category = self.client.get_channel(after.channel.category_id)
names = self.Storage.get_auto_voice_names(user.guild)
names = channel_names if len(names) == 0 else names
name = choice(names)
pos = auto_voice_channel.position + 1
voice_channel = await category.create_voice_channel(
name, reason='Created ship channel.', position=pos)
await voice_channel.edit(position=pos,
reason='Moved Ship Channel.')
self.created_channels.append(voice_channel)
await user.move_to(voice_channel,
reason='Moved to created channel.')
if before.channel in self.created_channels and len(
before.channel.members
) == 0: #Left any of the created Voice Channels
try:
await before.channel.delete(reason='Ship unmanned.')
except discord.errors.NotFound:
pass
self.created_channels.remove(before.channel)
def on_open(self, sid, data):
self.init()
data = json.loads(data)
self.config.write("version", data["version"])
self.config.write("port", data["port"])
storage = Storage()
last = storage.fetch_last_measurement_by_name(data["name"])
if last:
now = arrow.now()
if now.timestamp - int(last["timestamp"]) > 3600:
match = re.match(
".+( [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2})$",
data["name"])
if match:
data["name"] = data["name"][:-len(match.group(1))]
data["name"] += " " + arrow.now().format("YYYY-MM-DD HH:mm")
self.config.write("name", data["name"])
try:
self.config.write("rate", float(data["rate"]))
except ValueError:
pass
self.emit("connecting")
self.daemon.start()
def run(self):
self.storage = Storage()
self.config = Config()
interface = Interface(port=self.config.read("port"))
try:
self.log("Connecting")
interface.connect()
self.emit("connected")
self.log("Connected")
while self.running:
data = interface.read()
self.log(json.dumps(data))
if data:
data["name"] = self.config.read("name")
self.update(data)
self.storage.store_measurement(data)
time.sleep(self.config.read("rate"))
except (KeyboardInterrupt, SystemExit):
raise
except:
logging.exception(sys.exc_info()[0])
self.emit("log", traceback.format_exc())
finally:
interface.close()
self.emit("disconnected")
self.log("Disconnected")
self.thread = None
def run(self):
self.storage = Storage()
self.config = Config()
self.interface = Wrapper()
try:
self.log("Connecting")
self.retry(self.interface.connect)
self.emit("connected")
self.log("Connected")
while self.running:
data = self.retry(self.interface.read)
if isinstance(data, str):
if data in ["disconnected", "connected"]:
self.disconnect()
return
raise Exception(data)
else:
self.log(json.dumps(data))
if data:
data["name"] = self.config.read("name")
self.update(data)
self.storage.store_measurement(data)
sleep(self.config.read("rate"))
except (KeyboardInterrupt, SystemExit):
raise
except:
logging.exception(sys.exc_info()[0])
self.emit("log", traceback.format_exc())
self.emit("log-error")
finally:
self.disconnect()
def __init__(self, backend, on_receive, on_receive_interval):
self.backed = backend
self.on_receive = on_receive
self.on_receive_interval = on_receive_interval
self.storage = Storage()
if self.storage.fetch_status() != "disconnected":
self.storage.update_status("disconnected")
self.loop = asyncio.new_event_loop()
def __init__(self, stop_crit, stop_value=None, group_by_keys=None, window_id=None, storage_mode = None):
self.stop_crit = stop_crit
self.window_id = cuid.cuid() if not window_id else window_id
self.group_by_keys = group_by_keys if group_by_keys else []
self.stop_value = stop_value
self.cache = Storage(storage_mode)
self.existing_keys = set()
self._count = None
self.reset_window()
def run(self):
self.storage = Storage()
self.config = Config()
interface = None
version = self.config.read("version")
if version.startswith("TC"):
ble = True
self.interface = self.backed.ble
else:
ble = False
self.interface = interface = Interface(
port=self.config.read("port"))
if version == "UM25C":
interface.enable_higher_resolution()
try:
self.log("Connecting")
if ble:
self.backed.ble.connect()
else:
interface.connect()
self.emit("connected")
self.log("Connected")
while self.running:
if ble:
data = self.backed.ble.read()
else:
data = interface.read()
self.log(json.dumps(data))
if data:
data["name"] = self.config.read("name")
self.update(data)
self.storage.store_measurement(data)
time.sleep(self.config.read("rate"))
except (KeyboardInterrupt, SystemExit):
raise
except:
logging.exception(sys.exc_info()[0])
self.emit("log", traceback.format_exc())
self.emit("log-error")
finally:
if ble:
self.backed.ble.close()
else:
interface.close()
self.emit("disconnected")
self.log("Disconnected")
self.thread = None
def setUp(self):
env = {
"MINIO_SECRET_KEY": "imagerie",
"MINIO_ACCESS_KEY": "imagerie"
}
self.container = client.containers.run('minio/minio',
name='preptest', command='server /data', remove=True,
environment=env, detach=True, ports={'9000/tcp': 9002})
logger.debug(self.container)
self.s3_storage = Storage(prep_job.s3_storage)
self.gcs_storage = Storage(prep_job.gcs_storage)
pth = os.path.join(PARENT_DIR, "resources/gummies2/")
# self.gcs_storage.upload_path(pth,'{}'.format('projects'))
self.s3_storage.upload_path(pth, '{}'.format('projects'))
class LFC(commands.Cog, name='Looking for Crew'):
def __init__(self, client):
self.client = client
self.Storage = Storage()
async def auto_remove(self, user: discord.User):
await asyncio.sleep(7200)
for guild in self.Storage.get_lfc_enabled_guilds(self.client):
if user in guild.members:
role = self.Storage.get_lfc_role(guild)
try:
await guild.get_member(user.id).remove_roles(role)
except discord.Forbidden:
continue
@Utils.matchLFCChannel()
@commands.command(
brief='Sets the user into `Looking for Crew` status for 2 hours.',
description='This command gives the user the set `Looking for Crew` role. \n\
You can only use this command in the set channels.\n\
Type ?nlfc once you\'re in a crew to avoid getting further notifications.\n\
The role will be automatically removed after 2 hours.',
usage='?lfc'
)
async def lfc(self, ctx):
local_role = self.Storage.get_lfc_role(ctx.guild)
if local_role not in ctx.author.roles:
user = self.client.get_user(ctx.author.id)
for guild in self.Storage.get_lfc_enabled_guilds(self.client):
if user in guild.members:
role = self.Storage.get_lfc_role(guild)
try:
await guild.get_member(user.id).add_roles(role)
except discord.Forbidden:
continue
await ctx.send(f'{local_role.mention}, {ctx.author.mention} is now looking for crew.')
self.client.loop.create_task(self.auto_remove(user))
@Utils.matchLFCChannel()
@commands.command(
brief='Removes the `Looking for Crew` status manually.',
description='This removes the `Looking for Crew` status.\n\
If you are looking for a crew again use `?lfc`.',
usage='?nlfc'
)
async def nlfc(self, ctx):
local_role = self.Storage.get_lfc_role(ctx.guild)
if local_role in ctx.author.roles:
user = self.client.get_user(ctx.author.id)
for guild in self.Storage.get_lfc_enabled_guilds(self.client):
if user in guild.members:
role = self.Storage.get_lfc_role(guild)
try:
await guild.get_member(user.id).remove_roles(role)
except discord.Forbidden:
continue
await ctx.send(f'{ctx.author.mention}, you are no longer looking for a crew.')
def __init__(self, client):
self.client = client
self.whois_messages = dict()
self.prev = '⏪'
self.next = '⏩'
self.beginning = '⏮'
self.end = '⏭'
self.stop = '⏹'
self.emojis = [
self.beginning, self.prev, self.next, self.end, self.stop
]
self.Storage = Storage()
def run(self):
self.storage = Storage()
self.config = Config()
self.interface = Wrapper()
try:
self.log("Connecting")
self.retry(self.interface.connect)
self.emit("connected")
self.log("Connected")
name = self.config.read("name")
interval = float(self.config.read("rate"))
version = self.config.read("version")
session_id = self.storage.create_session(name, version)
while self.running:
begin = timer()
data = self.retry(self.interface.read)
if isinstance(data, str):
if data in ["disconnected", "connected"]:
self.disconnect()
return
raise Exception(data)
else:
self.log(json.dumps(data))
if data:
data["session_id"] = session_id
self.update(data, version)
self.storage.store_measurement(data)
measurement_runtime = timer() - begin
sleep_time = interval - measurement_runtime
if sleep_time > 0:
sleep(sleep_time)
except (KeyboardInterrupt, SystemExit):
raise
except:
logging.exception(sys.exc_info()[0])
self.emit("log", traceback.format_exc())
self.emit("log-error")
finally:
self.disconnect()
def setUp(self):
super(NmapToolTest, self).setUp()
self.exploit = Exploit(exploit_id=1,
name='test_name',
risk_level=RiskLevel.NONE)
self.exploit2 = Exploit(exploit_id=2,
name='test_name',
risk_level=RiskLevel.HIGH)
self.exploit_conf_args = Exploit(exploit_id=3)
self.exploit_conf_args.name = 'test_name2'
self.exploit_conf_args.risk_level = RiskLevel.HIGH
self.config = {
'scripts': {
'test_name': {
'args': 'test_args'
},
'test_name2': {
'args': MagicMock()
}
}
}
self.cfg = {
'tools': {
'nmap': {
'disable_scripts': [],
},
'common': {
'rate': 1337,
'http': {
'useragent': 'test_useragent'
}
}
},
'config_filename': ''
}
self.exploits = [self.exploit, self.exploit2]
self.port = Port(number=13,
transport_protocol=TransportProtocol.TCP,
node=Node(node_id=1,
ip=ipaddress.ip_address('127.0.0.1')))
self.port.scan = Scan(start=14, end=13)
self.port.protocol = 'test_service'
self.aucote = MagicMock(storage=Storage(":memory:"))
self.context = ScanContext(aucote=self.aucote,
scanner=MagicMock(scan=Scan()))
self.nmap_tool = NmapTool(context=self.context,
exploits=self.exploits,
port=self.port,
config=self.config)
def on_open(self, sid, data):
self.init()
data = json.loads(data)
self.config.write("version", data["version"])
if "port" in data:
self.config.write("port", data["port"])
if "ble_address" in data:
self.config.write("ble_address", data["ble_address"])
else:
data["ble_address"] = self.config.read("ble_address")
storage = Storage()
last = storage.fetch_last_measurement_by_name(data["name"])
if last:
if time() - int(last["timestamp"]) > 3600:
match = re.match(
".+( [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2})$",
data["name"])
if match:
data["name"] = data["name"][:-len(match.group(1))]
data["name"] += " " + pendulum.now().format("YYYY-MM-DD HH:mm")
if not data["name"]:
data["name"] = "My measurement"
self.config.write("name", data["name"])
try:
self.config.write("rate", float(data["rate"]))
except ValueError:
pass
tc_ble = data["version"].startswith(
"TC") and "USB" not in data["version"]
if tc_ble and ("ble_address" not in data or not data["ble_address"]):
self.daemon.log("BLE address is missing. Select address in Setup")
return
self.emit("connecting")
self.daemon.start()
def post(self):
"""
Executes a prep job to create an image corpus for training.
Use this method to start a prep job.
"""
job_def = request.json
job_def['process_json'] = True # Hardcode to process json file from project folder
job = Job(job_def['name'],job_def)
job.type = 'preprocess'
dt = newdt.now()
job.start_time = int(dt.timestamp()*1000)
job.request = {'full_path': request.full_path,'remote_addr':request.remote_addr,'method':request.method}
jb = aug_queue.enqueue(
preprocess, job,job_timeout=-1,result_ttl=86400,ttl=-1)
jb.meta['job_def'] = job_def
dt = newdt.now()
jb.meta['job_init_time'] = str(int(dt.timestamp()*1000))
jb.status = 'Running'
jb.save_meta()
json_str = job.to_json_string()
st = {
'BUCKET' : job.bucket,
'USE_GCS' : job.use_gcs,
'ACCESS_KEY' : access_key,
'SECRET_KEY' : secret_key,
'S3_URL' : s3_url
}
storage = Storage(st)
storage.upload_data(json_str,'jobs/running/{}_0_preprocess_r_{}.json'.format(str(job.start_time),jb.id),contentType='application/json')
storage.upload_data(json_str,'jobs/all/{}_0_preprocess_r_{}.json'.format(str(job.start_time),jb.id),contentType='application/json')
return {
"status": jb.status,
'job_id': jb.id,
'meta':jb.meta},201
def __init__(self, client):
self.client = client
self.Storage = Storage()
self.profile_messages = dict()
self.profile_status = dict()
self.steam_emoji = None
self.xbox_emoji = None
self.psn_emoji = None
self.nintendo_emoji = None
self.minecraft_emoji = None
self.origin_emoji = None
self.blizzard_emoji = None
self.bethesda_emoji = None
self.twitch_emoji = None
self.mixer_emoji = None
self.youtube_emoji = None
self.twitter_emoji = None
self.reddit_emoji = None
self.itchio_emoji = None
self.social_emoji = None
self.sot_emoji = None
self.game_emoji = '🎮'
self.game_emoji_url = 'https://emojipedia-us.s3.dualstack.us-west-1.amazonaws.com/thumbs/120/google/146/video-game_1f3ae.png'
self.stop_emoji = '⏹'
self.emojis = list()
self.social_platforms = [
'twitch', 'mixer', 'youtube', 'twitter', 'reddit', 'itchio'
]
self.gt_platforms = [
'steam', 'xbox', 'psn', 'nintendo', 'minecraft', 'origin',
'blizzard', 'bethesda'
]
def __init__(self, exploits, kudu_queue, tools_config):
self.exploits = exploits
self._kudu_queue = kudu_queue
signal.signal(signal.SIGINT, self.signal_handler)
signal.signal(signal.SIGTERM, self.signal_handler)
signal.signal(signal.SIGHUP, self.graceful_stop)
self.load_tools(tools_config)
self._storage = Storage(conn_string=cfg['storage.db'], nodes_limit=cfg['storage.max_nodes_query'])
self.ioloop = IOLoop.current()
self.topdis = Topdis(cfg['topdis.api.host'], cfg['topdis.api.port'], cfg['topdis.api.base'])
self.async_task_managers = {
TaskManagerType.SCANNER: AsyncTaskManager.instance(name=TaskManagerType.SCANNER.value,
parallel_tasks=10),
TaskManagerType.REGULAR: AsyncTaskManager.instance(name=TaskManagerType.REGULAR.value,
parallel_tasks=cfg['service.scans.parallel_tasks']),
TaskManagerType.QUICK: AsyncTaskManager.instance(name=TaskManagerType.QUICK.value, parallel_tasks=30)
}
for task_manager in self.async_task_managers.values():
throttling_consumer = ThrottlingConsumer(manager=task_manager)
self.ioloop.add_callback(partial(cfg.add_rabbit_consumer, throttling_consumer))
self.ioloop.add_callback(throttling_consumer.consume)
self.web_server = WebServer(self, cfg['service.api.v1.host'], cfg['service.api.v1.port'],
path=cfg['service.api.path'])
self._tftp_thread = TFTPThread(cfg['tftp.host'], cfg['tftp.port'], timeout=cfg['tftp.timeout'],
min_port=cfg['tftp.min_port'], max_port=cfg['tftp.max_port'])
self._storage_thread = StorageThread(storage=self._storage)
self.scanners = []
if cfg.toucan:
for task_manager in self.async_task_managers.values():
cfg.toucan_monitor.register_toucan_key(key='throttling.rate', add_prefix=False, default=1,
callback=task_manager.change_throttling_toucan)
def __init__(self,
command_prefix: typing.Union[str, typing.Callable[
[discord.Message], typing.Awaitable]] = None,
**options):
if not command_prefix:
command_prefix = get_prefix
super().__init__(command_prefix, **options)
self.config = Config()
self.logger = FakeLogger()
self.db = Storage(self)
self.commands_used = collections.Counter()
with open("credentials.json", "r") as f:
credentials = json.load(f)
self.token = credentials["discord_token"]
self.uptime = datetime.datetime.utcnow()
self.loop.set_debug(True)
def post(self):
"""
Executes a training.
Use this method to start a training.
"""
job_def = request.json
job = Job(job_def['name'], job_def)
job.type = 'train'
dt = newdt.now()
job.start_time = int(dt.timestamp() * 1000)
job.request = {
'full_path': request.full_path,
'remote_addr': request.remote_addr,
'method': request.method
}
if hasattr(job, 'ml_engine') and job.ml_engine:
jb = train_queue.enqueue(train_mlengine,
job,
job_timeout=-1,
result_ttl=-1)
else:
jb = train_queue.enqueue(train_job_method, job, job_timeout=-1)
jb.meta['job_init_time'] = str(int(dt.timestamp() * 1000))
jb.meta['job_def'] = job_def
jb.save_meta()
json_str = job.to_json_string()
st = {
'BUCKET': job.bucket,
'USE_GCS': job.use_gcs,
'ACCESS_KEY': access_key,
'SECRET_KEY': secret_key,
'S3_URL': s3_url
}
storage = Storage(st)
storage.upload_data(json_str,
'jobs/running/{}_0_train_r_{}.json'.format(
str(job.start_time), jb.id),
contentType='application/json')
storage.upload_data(json_str,
'jobs/all/{}_0_train_r_{}.json'.format(
str(job.start_time), jb.id),
contentType='application/json')
return {
"status": jb.get_status(),
'job_id': jb.id,
'meta': jb.meta
}, 201
def main():
env = gym.make(env_name)
goal_object = Goal(min_position=env.env.min_position,
max_position=env.env.max_position,
step=0.2)
meta_policy, target_meta_policy, policy, target_policy = get_policies(
env, goal_object)
optimizer = Optimizer(meta_policy, target_meta_policy, policy,
target_policy, mini_batch_size, discount,
learning_rate, update_epochs)
episode_rewards = deque(maxlen=50)
get_meta_epsilon = lambda episode: np.exp(-episode * e_meta_decay)
get_epsilon = lambda episode: np.exp(-episode * e_decay)
frame = 0
meta_frame = 0
for eps in range(0, n_eps + 1):
print('Episode', eps)
if eps % 5 == 0:
episode_rewards.append(
test_env(target_meta_policy, target_policy, goal_object))
print('Avg reward', np.mean(episode_rewards))
storage = Storage(device=device)
storage_meta = Storage(device=device)
meta_state = env.reset()
state = meta_state.copy()
state = torch.FloatTensor(state).to(device)
done = False
for step in range(10):
goal = meta_policy.act(meta_state, get_meta_epsilon(meta_frame))
goal_value = torch.FloatTensor(
[goal_object.get_goal(int(goal.item()))]).to(device)
print('Predicted goal', goal_value)
goal_reached = False
for i in range(30):
joint_state = torch.cat((state, goal_value), axis=0).to(device)
with torch.no_grad():
action = policy.act(joint_state, get_epsilon(frame))
next_state, reward, done, _ = env.step(int(action.item()))
intrinsic_reward = get_intrinsic_reward(goal_value, next_state)
goal_reached = True if intrinsic_reward else False
joint_next_state = np.concatenate(
[next_state, [goal_value.item]], axis=0)
storage.push(joint_state, action, intrinsic_reward,
joint_next_state, done)
extrinsic_reward += reward
state = next_state
state = torch.FloatTensor(state).to(device)
frame += 1
if done or goal_reached:
break
storage_meta.push(meta_state, goal, extrinsic_reward, next_state,
done) # do i really pass here unnormalized goal?
meta_state = state
meta_frame += 1
if done:
break
storage.compute()
storage_meta.compute()
loss_meta, loss = optimizer.update(storage_meta, storage)
if eps % target_policy_update:
target_meta_policy.load_state_dict(meta_policy.state_dict())
target_policy.load_state_dict(policy.state_dict())
with open('metrics.csv', 'a') as metrics:
metrics.write('{},{}\n'.format(loss_meta, loss))
def setUp(self):
self.maxDiff = None
self.storage = Storage(conn_string=getenv('AUCOTE_TEST_POSTGRES'))
self.scan = Scan(start=1,
end=17,
protocol=TransportProtocol.UDP,
scanner='test_name')
self.scan_1 = Scan(rowid=56,
protocol=TransportProtocol.UDP,
scanner='test_name',
start=13,
end=19)
self.scan_2 = Scan(rowid=79,
protocol=TransportProtocol.UDP,
scanner='test_name',
start=2,
end=18)
self.scan_3 = Scan(rowid=80,
protocol=TransportProtocol.UDP,
scanner='test_name_2',
start=20,
end=45)
self.scan_4 = Scan(rowid=78,
protocol=TransportProtocol.TCP,
scanner='portdetection',
start=1,
end=2)
self.node_1 = Node(node_id=1, ip=ipaddress.ip_address('127.0.0.1'))
self.node_1.name = 'test_node_1'
self.node_1.scan = Scan(start=15)
self.node_2 = Node(node_id=2, ip=ipaddress.ip_address('127.0.0.2'))
self.node_2.name = 'test_node_2'
self.node_2.scan = Scan(start=56)
self.node_3 = Node(node_id=3, ip=ipaddress.ip_address('127.0.0.3'))
self.node_3.name = 'test_node_3'
self.node_3.scan = Scan(start=98)
self.node_4 = Node(node_id=4, ip=ipaddress.ip_address('127.0.0.4'))
self.node_4.name = 'test_node_4'
self.node_4.scan = Scan(start=3)
self.node_scan_1 = NodeScan(node=self.node_1,
rowid=13,
scan=self.scan_1,
timestamp=15)
self.node_scan_2 = NodeScan(node=self.node_2,
rowid=15,
scan=self.scan_1,
timestamp=56)
self.node_scan_3 = NodeScan(node=self.node_3,
rowid=16,
scan=self.scan_1,
timestamp=98)
self.node_scan_3 = NodeScan(node=self.node_3,
rowid=17,
scan=self.scan_2,
timestamp=90)
self.port_1 = Port(node=self.node_1,
number=45,
transport_protocol=TransportProtocol.UDP)
self.port_1.scan = self.scan_1
self.port_scan_1 = PortScan(port=self.port_1,
scan=self.scan_1,
timestamp=176,
rowid=124)
self.port_2 = Port(node=self.node_2,
transport_protocol=TransportProtocol.UDP,
number=65)
self.port_2.scan = Scan(start=3, end=45)
self.port_scan_2 = PortScan(port=self.port_2,
scan=self.scan_1,
timestamp=987,
rowid=15)
self.port_3 = Port(node=self.node_3,
transport_protocol=TransportProtocol.ICMP,
number=99)
self.port_3.scan = Scan(start=43, end=180)
self.port_scan_3 = PortScan(port=self.port_3,
scan=self.scan_1,
timestamp=619,
rowid=13)
self.port_4 = Port(node=self.node_1,
number=80,
transport_protocol=TransportProtocol.UDP)
self.port_4.scan = self.scan_1
self.port_scan_4 = PortScan(port=self.port_4,
scan=self.scan_1,
timestamp=650,
rowid=480)
self.port_5 = Port(node=self.node_4,
number=22,
transport_protocol=TransportProtocol.TCP)
self.port_5.scan = self.scan_4
self.exploit_1 = Exploit(exploit_id=14,
name='test_name',
app='test_app')
self.exploit_2 = Exploit(exploit_id=2,
name='test_name_2',
app='test_app_2')
self.exploit_3 = Exploit(exploit_id=56,
name='test_name_2',
app='test_app')
self.exploit_4 = Exploit(exploit_id=0,
name='portdetection',
app='portdetection')
self.security_scan_1 = SecurityScan(exploit=self.exploit_1,
port=self.port_1,
scan=self.scan_1,
scan_start=178,
scan_end=851)
self.security_scan_2 = SecurityScan(exploit=self.exploit_2,
port=self.port_1,
scan=self.scan_1,
scan_start=109,
scan_end=775)
self.security_scan_3 = SecurityScan(exploit=self.exploit_3,
port=self.port_1,
scan=self.scan_2,
scan_start=113,
scan_end=353)
self.security_scan_4 = SecurityScan(exploit=self.exploit_1,
port=self.port_1,
scan=self.scan_3,
scan_start=180,
scan_end=222)
self.security_scan_5 = SecurityScan(exploit=self.exploit_1,
port=self.port_1,
scan=self.scan_2,
scan_start=14,
scan_end=156)
self.security_scan_6 = SecurityScan(exploit=self.exploit_2,
port=self.port_1,
scan=self.scan_2,
scan_start=56,
scan_end=780)
self.security_scan_7 = SecurityScan(exploit=self.exploit_4,
port=self.port_5,
scan=self.scan_4,
scan_start=14,
scan_end=890)
self.vuln_change_1 = PortDetectionChange(
change_time=124445,
current_finding=self.port_scan_1,
previous_finding=self.port_scan_2)
self.vuln_change_2 = PortDetectionChange(
change_time=32434,
current_finding=self.port_scan_2,
previous_finding=self.port_scan_3)
self.vulnerability_1 = Vulnerability(port=self.port_1,
output='test_output_1',
exploit=self.exploit_1,
cve='CVE-2017',
cvss=6.7,
subid=1,
vuln_time=13,
rowid=134,
scan=self.scan_1)
self.vulnerability_2 = Vulnerability(port=self.port_1,
output='test_output_2',
exploit=self.exploit_2,
cve='CWE-14',
cvss=8.9,
subid=2,
vuln_time=98,
rowid=152,
scan=self.scan_1)
self.vulnerability_3 = Vulnerability(port=self.port_1,
output='test_output_3',
exploit=self.exploit_1,
cve='CVE-2016',
cvss=3.7,
subid=2,
vuln_time=15,
rowid=153,
scan=self.scan_2)
self.vulnerability_4 = Vulnerability(port=self.port_1,
output='test_output_4',
exploit=self.exploit_2,
cve='CWE-15',
cvss=2.9,
subid=1,
vuln_time=124,
rowid=169,
scan=self.scan_2)
self.vulnerability_5 = Vulnerability(port=self.port_5,
output='',
exploit=self.exploit_4,
cve=None,
cvss=None,
subid=0,
vuln_time=124,
rowid=200,
scan=self.scan_4,
expiration_time=400)
self.vulnerability_6 = Vulnerability(port=self.port_5,
output='tftp',
exploit=self.exploit_4,
cve=None,
cvss=None,
subid=1,
vuln_time=124,
rowid=201,
scan=self.scan_4,
expiration_time=400)
self.vulnerability_7 = Vulnerability(port=self.port_5,
output='tftp server name',
exploit=self.exploit_4,
cve=None,
cvss=None,
subid=2,
vuln_time=124,
rowid=202,
scan=self.scan_4,
expiration_time=400)
self.vulnerability_8 = Vulnerability(port=self.port_5,
output='6.7.8',
exploit=self.exploit_4,
cve=None,
cvss=None,
subid=3,
vuln_time=124,
rowid=203,
scan=self.scan_4,
expiration_time=400)
self.vulnerability_9 = Vulnerability(port=self.port_5,
output='HERE IS TFTP\n\n\n > ',
exploit=self.exploit_4,
cve=None,
cvss=None,
subid=4,
vuln_time=124,
rowid=204,
scan=self.scan_4,
expiration_time=400)
self.vulnerability_10 = Vulnerability(port=self.port_5,
output='test:cpe',
exploit=self.exploit_4,
cve=None,
cvss=None,
subid=5,
vuln_time=124,
rowid=205,
scan=self.scan_4,
expiration_time=400)
self.vulnerability_11 = Vulnerability(port=self.port_5,
output='os name',
exploit=self.exploit_4,
cve=None,
cvss=None,
subid=6,
vuln_time=124,
rowid=206,
scan=self.scan_4,
expiration_time=400)
self.vulnerability_12 = Vulnerability(port=self.port_5,
output='os version',
exploit=self.exploit_4,
cve=None,
cvss=None,
subid=7,
vuln_time=124,
rowid=207,
scan=self.scan_4,
expiration_time=400)
self.vulnerability_13 = Vulnerability(port=self.port_5,
output='test:os:cpe',
exploit=self.exploit_4,
cve=None,
cvss=None,
subid=8,
vuln_time=124,
rowid=208,
scan=self.scan_4,
expiration_time=400)
class StorageTest(TestCase):
SEL_SEC_SCAN = (
"SELECT scan_id, exploit_id, exploit_app, exploit_name, node_id, node_ip, port_protocol, "
"port_number, sec_scan_start, sec_scan_end FROM security_scans", )
SEL_NOD_SCAN = (
"SELECT scan_id, node_id, node_ip, time from nodes_scans", )
SEL_POR_SCAN = (
"SELECT scan_id, node_id, node_ip, port, port_protocol, time from ports_scans",
)
SEL_CHANGE = (
"SELECT type, vulnerability_id, vulnerability_subid, previous_id, current_id, time FROM changes",
)
SEL_SCANS = (
"SELECT ROWID, scan_start, scan_end, protocol, scanner_name FROM scans",
)
SEL_VULNS = (
"SELECT scan_id, node_id, node_ip, port_protocol, port, vulnerability_id, vulnerability_subid, cve, cvss, output, time FROM vulnerabilities",
)
def setUp(self):
self.maxDiff = None
self.storage = Storage(conn_string=getenv('AUCOTE_TEST_POSTGRES'))
self.scan = Scan(start=1,
end=17,
protocol=TransportProtocol.UDP,
scanner='test_name')
self.scan_1 = Scan(rowid=56,
protocol=TransportProtocol.UDP,
scanner='test_name',
start=13,
end=19)
self.scan_2 = Scan(rowid=79,
protocol=TransportProtocol.UDP,
scanner='test_name',
start=2,
end=18)
self.scan_3 = Scan(rowid=80,
protocol=TransportProtocol.UDP,
scanner='test_name_2',
start=20,
end=45)
self.scan_4 = Scan(rowid=78,
protocol=TransportProtocol.TCP,
scanner='portdetection',
start=1,
end=2)
self.node_1 = Node(node_id=1, ip=ipaddress.ip_address('127.0.0.1'))
self.node_1.name = 'test_node_1'
self.node_1.scan = Scan(start=15)
self.node_2 = Node(node_id=2, ip=ipaddress.ip_address('127.0.0.2'))
self.node_2.name = 'test_node_2'
self.node_2.scan = Scan(start=56)
self.node_3 = Node(node_id=3, ip=ipaddress.ip_address('127.0.0.3'))
self.node_3.name = 'test_node_3'
self.node_3.scan = Scan(start=98)
self.node_4 = Node(node_id=4, ip=ipaddress.ip_address('127.0.0.4'))
self.node_4.name = 'test_node_4'
self.node_4.scan = Scan(start=3)
self.node_scan_1 = NodeScan(node=self.node_1,
rowid=13,
scan=self.scan_1,
timestamp=15)
self.node_scan_2 = NodeScan(node=self.node_2,
rowid=15,
scan=self.scan_1,
timestamp=56)
self.node_scan_3 = NodeScan(node=self.node_3,
rowid=16,
scan=self.scan_1,
timestamp=98)
self.node_scan_3 = NodeScan(node=self.node_3,
rowid=17,
scan=self.scan_2,
timestamp=90)
self.port_1 = Port(node=self.node_1,
number=45,
transport_protocol=TransportProtocol.UDP)
self.port_1.scan = self.scan_1
self.port_scan_1 = PortScan(port=self.port_1,
scan=self.scan_1,
timestamp=176,
rowid=124)
self.port_2 = Port(node=self.node_2,
transport_protocol=TransportProtocol.UDP,
number=65)
self.port_2.scan = Scan(start=3, end=45)
self.port_scan_2 = PortScan(port=self.port_2,
scan=self.scan_1,
timestamp=987,
rowid=15)
self.port_3 = Port(node=self.node_3,
transport_protocol=TransportProtocol.ICMP,
number=99)
self.port_3.scan = Scan(start=43, end=180)
self.port_scan_3 = PortScan(port=self.port_3,
scan=self.scan_1,
timestamp=619,
rowid=13)
self.port_4 = Port(node=self.node_1,
number=80,
transport_protocol=TransportProtocol.UDP)
self.port_4.scan = self.scan_1
self.port_scan_4 = PortScan(port=self.port_4,
scan=self.scan_1,
timestamp=650,
rowid=480)
self.port_5 = Port(node=self.node_4,
number=22,
transport_protocol=TransportProtocol.TCP)
self.port_5.scan = self.scan_4
self.exploit_1 = Exploit(exploit_id=14,
name='test_name',
app='test_app')
self.exploit_2 = Exploit(exploit_id=2,
name='test_name_2',
app='test_app_2')
self.exploit_3 = Exploit(exploit_id=56,
name='test_name_2',
app='test_app')
self.exploit_4 = Exploit(exploit_id=0,
name='portdetection',
app='portdetection')
self.security_scan_1 = SecurityScan(exploit=self.exploit_1,
port=self.port_1,
scan=self.scan_1,
scan_start=178,
scan_end=851)
self.security_scan_2 = SecurityScan(exploit=self.exploit_2,
port=self.port_1,
scan=self.scan_1,
scan_start=109,
scan_end=775)
self.security_scan_3 = SecurityScan(exploit=self.exploit_3,
port=self.port_1,
scan=self.scan_2,
scan_start=113,
scan_end=353)
self.security_scan_4 = SecurityScan(exploit=self.exploit_1,
port=self.port_1,
scan=self.scan_3,
scan_start=180,
scan_end=222)
self.security_scan_5 = SecurityScan(exploit=self.exploit_1,
port=self.port_1,
scan=self.scan_2,
scan_start=14,
scan_end=156)
self.security_scan_6 = SecurityScan(exploit=self.exploit_2,
port=self.port_1,
scan=self.scan_2,
scan_start=56,
scan_end=780)
self.security_scan_7 = SecurityScan(exploit=self.exploit_4,
port=self.port_5,
scan=self.scan_4,
scan_start=14,
scan_end=890)
self.vuln_change_1 = PortDetectionChange(
change_time=124445,
current_finding=self.port_scan_1,
previous_finding=self.port_scan_2)
self.vuln_change_2 = PortDetectionChange(
change_time=32434,
current_finding=self.port_scan_2,
previous_finding=self.port_scan_3)
self.vulnerability_1 = Vulnerability(port=self.port_1,
output='test_output_1',
exploit=self.exploit_1,
cve='CVE-2017',
cvss=6.7,
subid=1,
vuln_time=13,
rowid=134,
scan=self.scan_1)
self.vulnerability_2 = Vulnerability(port=self.port_1,
output='test_output_2',
exploit=self.exploit_2,
cve='CWE-14',
cvss=8.9,
subid=2,
vuln_time=98,
rowid=152,
scan=self.scan_1)
self.vulnerability_3 = Vulnerability(port=self.port_1,
output='test_output_3',
exploit=self.exploit_1,
cve='CVE-2016',
cvss=3.7,
subid=2,
vuln_time=15,
rowid=153,
scan=self.scan_2)
self.vulnerability_4 = Vulnerability(port=self.port_1,
output='test_output_4',
exploit=self.exploit_2,
cve='CWE-15',
cvss=2.9,
subid=1,
vuln_time=124,
rowid=169,
scan=self.scan_2)
self.vulnerability_5 = Vulnerability(port=self.port_5,
output='',
exploit=self.exploit_4,
cve=None,
cvss=None,
subid=0,
vuln_time=124,
rowid=200,
scan=self.scan_4,
expiration_time=400)
self.vulnerability_6 = Vulnerability(port=self.port_5,
output='tftp',
exploit=self.exploit_4,
cve=None,
cvss=None,
subid=1,
vuln_time=124,
rowid=201,
scan=self.scan_4,
expiration_time=400)
self.vulnerability_7 = Vulnerability(port=self.port_5,
output='tftp server name',
exploit=self.exploit_4,
cve=None,
cvss=None,
subid=2,
vuln_time=124,
rowid=202,
scan=self.scan_4,
expiration_time=400)
self.vulnerability_8 = Vulnerability(port=self.port_5,
output='6.7.8',
exploit=self.exploit_4,
cve=None,
cvss=None,
subid=3,
vuln_time=124,
rowid=203,
scan=self.scan_4,
expiration_time=400)
self.vulnerability_9 = Vulnerability(port=self.port_5,
output='HERE IS TFTP\n\n\n > ',
exploit=self.exploit_4,
cve=None,
cvss=None,
subid=4,
vuln_time=124,
rowid=204,
scan=self.scan_4,
expiration_time=400)
self.vulnerability_10 = Vulnerability(port=self.port_5,
output='test:cpe',
exploit=self.exploit_4,
cve=None,
cvss=None,
subid=5,
vuln_time=124,
rowid=205,
scan=self.scan_4,
expiration_time=400)
self.vulnerability_11 = Vulnerability(port=self.port_5,
output='os name',
exploit=self.exploit_4,
cve=None,
cvss=None,
subid=6,
vuln_time=124,
rowid=206,
scan=self.scan_4,
expiration_time=400)
self.vulnerability_12 = Vulnerability(port=self.port_5,
output='os version',
exploit=self.exploit_4,
cve=None,
cvss=None,
subid=7,
vuln_time=124,
rowid=207,
scan=self.scan_4,
expiration_time=400)
self.vulnerability_13 = Vulnerability(port=self.port_5,
output='test:os:cpe',
exploit=self.exploit_4,
cve=None,
cvss=None,
subid=8,
vuln_time=124,
rowid=208,
scan=self.scan_4,
expiration_time=400)
def prepare_tables(self):
self.storage.connect()
self.storage.remove_all()
self.storage.init_schema()
self.prepare_scans()
self.prepare_nodes_scans()
self.prepare_ports_scans()
self.prepare_vulnerabilities()
self.prepare_security_scans()
def prepare_scans(self):
self.storage.execute((
"INSERT INTO scans(ROWID, protocol, scanner_name, scan_start, scan_end) "
"VALUES "
"(56, 17, 'test_name', 13, 19), "
"(80, 17, 'test_name_2', 20, 45), "
"(79, 17, 'test_name', 2, 18),"
"(78, 6, 'portdetection', 1, 2)", ))
def prepare_nodes_scans(self):
self.storage.execute((
"INSERT INTO nodes_scans (ROWID, scan_id, node_id, node_ip, time) VALUES "
"(13, 56, 1, '127.0.0.1', 15), "
"(15, 56, 2, '127.0.0.2', 56), "
"(16, 56, 3, '127.0.0.3', 98), "
"(17, 79, 3, '127.0.0.3', 90) "
"", ))
def prepare_ports_scans(self):
self.storage.execute((
'INSERT INTO ports_scans(ROWID, scan_id, node_id, node_ip, port, port_protocol, time) '
'VALUES '
"(124, 56, 1, '127.0.0.1', 45, 17, 176), "
"(480, 56, 1, '127.0.0.1', 80, 17, 650), "
"(13, 56, 3, '127.0.0.3', 99, 1, 619), "
"(15, 56, 2, '127.0.0.2', 65, 17, 987)", ))
def prepare_security_scans(self):
self.storage.execute((
"INSERT INTO security_scans(exploit_id, exploit_name, exploit_app, scan_id, node_id, "
"node_ip, port_number, port_protocol, sec_scan_start, sec_scan_end) VALUES "
"(14, 'test_name', 'test_app', 56, 1, '127.0.0.1', 45, 17, 178, 851), "
"(56, 'test_name_2', 'test_app', 79, 1, '127.0.0.1', 45, 17, 113, 353), "
"(14, 'test_name', 'test_app', 80, 1, '127.0.0.1', 45, 17, 180, 222), "
"(2, 'test_name_2', 'test_app_2', 56, 1, '127.0.0.1', 45, 17, 109, 775), "
"(14, 'test_name', 'test_app', 79, 1, '127.0.0.1', 45, 17, 14, 156), "
"(2, 'test_name_2', 'test_app_2', 79, 1, '127.0.0.1', 45, 17, 56, 780), "
"(0, 'portdetection', 'portdetection', 78, 4, '127.0.0.4', 22, 6, 14, 890)",
))
def prepare_vulnerabilities(self):
self.storage.execute((
"INSERT INTO vulnerabilities (ROWID, scan_id, node_id, node_ip, port_protocol, port, "
"vulnerability_id, vulnerability_subid, cve, cvss, output, time, expiration_time) VALUES "
"(134, 56, 1, '127.0.0.1', 17, 45, 14, 1, 'CVE-2017', 6.7, 'test_output_1', 13, 600),"
"(152, 56, 1, '127.0.0.1', 17, 45, 2, 2, 'CWE-14', 8.9, 'test_output_2', 98, 600),"
"(153, 79, 1, '127.0.0.1', 17, 45, 14, 2, 'CVE-2016', 3.7, 'test_output_3', 15, NULL),"
"(169, 79, 1, '127.0.0.1', 17, 45, 2, 1, 'CWE-15', 2.9, 'test_output_4', 124, 600), "
"(200, 78, 4, '127.0.0.4', 6, 22, 0, 0, NULL, NULL, '', 124, 400), "
"(201, 78, 4, '127.0.0.4', 6, 22, 0, 1, NULL, NULL, 'tftp', 124, 400), "
"(202, 78, 4, '127.0.0.4', 6, 22, 0, 2, NULL, NULL, 'tftp server name', 124, 400), "
"(203, 78, 4, '127.0.0.4', 6, 22, 0, 3, NULL, NULL, '6.7.8', 124, 400), "
"(204, 78, 4, '127.0.0.4', 6, 22, 0, 4, NULL, NULL, 'HERE IS TFTP\n\n\n > ', 124, 400), "
"(205, 78, 4, '127.0.0.4', 6, 22, 0, 5, NULL, NULL, 'test:cpe', 124, 400), "
"(206, 78, 4, '127.0.0.4', 6, 22, 0, 6, NULL, NULL, 'os name', 124, 400), "
"(207, 78, 4, '127.0.0.4', 6, 22, 0, 7, NULL, NULL, 'os version', 124, 400), "
"(208, 78, 4, '127.0.0.4', 6, 22, 0, 8, NULL, NULL, 'test:os:cpe', 124, 400)"
"", ))
def test_context_manager(self):
with self.storage:
self.assertTrue(True)
def test_connect(self):
self.storage.connect()
self.assertIsInstance(self.storage.conn, connection)
def test_close(self):
self.storage.conn = connect(":memory:")
self.storage.close()
self.assertEqual(self.storage.conn, None)
def test_cursor_property(self):
self.assertEqual(self.storage.cursor, self.storage._cursor)
@patch('utils.storage.time.time', MagicMock(return_value=134))
def test_save_nodes(self):
self.storage.connect()
self.storage.remove_all()
self.storage.init_schema()
expected = [(56, 1, '127.0.0.1', 134), (56, 2, '127.0.0.2', 134),
(56, 3, '127.0.0.3', 134)]
self.storage.save_nodes(nodes=[self.node_1, self.node_2, self.node_3],
scan=self.scan_1)
result = self.storage.execute(self.SEL_NOD_SCAN)
self.assertEqual(result, expected)
@patch('utils.storage.time.time', MagicMock(return_value=60))
def test_get_nodes(self):
self.prepare_tables()
expected = [self.node_2, self.node_3, self.node_3]
result = self.storage.get_nodes(pasttime=10, scan=self.scan_1)
self.assertCountEqual(result, expected)
def test_get_vulnerabilities(self):
self.prepare_tables()
expected = [self.vulnerability_1]
result = self.storage.get_vulnerabilities(port=self.port_1,
scan=self.scan_1,
exploit=self.exploit_1)
self.assertCountEqual(result, expected)
def test_get_nodes_by_scan(self):
self.prepare_tables()
expected = [self.node_1, self.node_2, self.node_3]
result = self.storage.get_nodes_by_scan(scan=self.scan_1)
self.assertCountEqual(result, expected)
@patch('utils.storage.time.time', MagicMock(return_value=167))
def test_save_port(self):
self.storage.connect()
self.storage.remove_all()
self.storage.init_schema()
expected = [(56, 1, '127.0.0.1', 45, 17, 167)]
self.storage.save_port(port=self.port_1, scan=self.scan_1)
result = self.storage.execute(self.SEL_POR_SCAN)
self.assertEqual(result, expected)
@patch('utils.storage.time.time', MagicMock(return_value=167))
def test_save_ports(self):
self.storage.connect()
self.storage.remove_all()
self.storage.init_schema()
expected = [(56, 1, '127.0.0.1', 45, 17, 167),
(56, 2, '127.0.0.2', 65, 17, 167),
(56, 3, '127.0.0.3', 99, 1, 167)]
self.storage.save_ports(ports=[self.port_1, self.port_2, self.port_3],
scan=self.scan_1)
result = self.storage.execute(self.SEL_POR_SCAN)
self.assertEqual(result, expected)
@patch('utils.storage.time.time', MagicMock(return_value=1000))
def test_get_ports(self):
self.prepare_tables()
expected = [
self.port_scan_2.port, self.port_scan_3.port, self.port_scan_4.port
]
result = self.storage.get_ports(pasttime=400, scan=self.scan_1)
self.assertCountEqual(result, expected)
def test_get_ports_by_scan_and_node(self):
self.prepare_tables()
expected = [self.port_scan_1, self.port_scan_4]
result = self.storage.get_ports_by_scan_and_node(node=self.node_1,
scan=self.scan_1)
self.assertCountEqual(result, expected)
def test_save_security_scan(self):
self.storage.connect()
self.storage.remove_all()
self.storage.init_schema()
expected = [(56, 14, 'test_app', 'test_name', 1, '127.0.0.1', 17, 45,
13.0, 19.0)]
self.storage.save_security_scan(exploit=self.exploit_1,
port=self.port_1,
scan=self.scan_1)
result = self.storage.execute(self.SEL_SEC_SCAN)
self.assertEqual(result, expected)
def test_save_security_scan_twice(self):
self.storage.connect()
self.storage.remove_all()
self.storage.init_schema()
self.port_1.scan.start = None
expected = [(56, 14, 'test_app', 'test_name', 1, '127.0.0.1', 17, 45,
None, 19.0)]
self.storage.save_security_scan(exploit=self.exploit_1,
port=self.port_1,
scan=self.scan_1)
result = self.storage.execute(self.SEL_SEC_SCAN)
self.assertEqual(result, expected)
self.port_1.scan.start = 13
self.port_1.scan.end = None
expected = [(56, 14, 'test_app', 'test_name', 1, '127.0.0.1', 17, 45,
13.0, 19.0)]
self.storage.save_security_scan(exploit=self.exploit_1,
port=self.port_1,
scan=self.scan_1)
result = self.storage.execute(self.SEL_SEC_SCAN)
self.assertEqual(result, expected)
def test_save_security_scans(self):
self.storage.connect()
self.storage.remove_all()
self.storage.init_schema()
expected = [(56, 14, 'test_app', 'test_name', 1, '127.0.0.1', 17, 45,
13.0, 19.0),
(56, 2, 'test_app_2', 'test_name_2', 1, '127.0.0.1', 17,
45, 13.0, 19.0),
(56, 56, 'test_app', 'test_name_2', 1, '127.0.0.1', 17, 45,
13.0, 19.0)]
self.storage.save_security_scans(
exploits=[self.exploit_1, self.exploit_2, self.exploit_3],
port=self.port_1,
scan=self.scan_1)
result = self.storage.execute(self.SEL_SEC_SCAN)
self.assertEqual(result, expected)
def test_save_changes(self):
self.storage.connect()
self.storage.remove_all()
self.storage.init_schema()
expected = [(1, 0, 0, 15, 124, 124445), (1, 0, 0, 13, 15, 32434)]
self.storage.save_changes(
changes=[self.vuln_change_1, self.vuln_change_2])
result = self.storage.execute(self.SEL_CHANGE)
self.assertEqual(result, expected)
def test_get_ports_by_nodes_without_nodes(self):
expected = []
result = self.storage.get_ports_by_nodes([])
self.assertEqual(result, expected)
@patch('utils.storage.time.time', MagicMock(return_value=1000))
def test_get_ports_by_nodes(self):
self.storage.nodes_limit = 2
self.prepare_tables()
expected = [self.port_scan_2.port, self.port_scan_4.port]
result = self.storage.get_ports_by_nodes(
nodes=[self.node_1, self.node_3, self.node_2],
pasttime=400,
protocol=TransportProtocol.UDP)
self.assertCountEqual(result, expected)
def test_execute_query(self):
query = "part_1", "arg_1", "arg_2"
self.storage._cursor = MagicMock()
self.storage._cursor.rowcount = 2
self.storage.conn = MagicMock()
result = self.storage.execute(query)
self.storage.cursor.execute.assert_called_once_with(
"part_1", "arg_1", "arg_2")
self.storage.cursor.fetchall.assert_called_once_with()
self.assertEqual(result, self.storage.cursor.fetchall())
def test_execute_query_list(self):
queries = [("part_1", "arg_1", "arg_2"), ("part_2", "arg_3")]
self.storage._cursor = MagicMock()
self.storage._cursor.rowcount = 2
self.storage.conn = MagicMock()
self.storage.execute(queries)
self.storage.cursor.execute.assert_has_calls(
(call("part_1", "arg_1", "arg_2"), call("part_2", "arg_3")))
def test_transport_protocol_none(self):
result = self.storage._transport_protocol(None)
self.assertIsNone(result)
@patch('utils.storage.TransportProtocol')
def test_transport_protocol(self, protocol):
result = self.storage._transport_protocol(6)
protocol.from_iana.assert_called_once_with(6)
self.assertEqual(result, protocol.from_iana())
def test_protocol_to_iana_none(self):
protocol = None
result = self.storage._protocol_to_iana(protocol)
self.assertIsNone(result)
def test_save_scan(self):
self.storage.connect()
self.storage.remove_all()
self.storage.init_schema()
expected = [(1, 13, 19, 17, 'test_name')]
self.storage.save_scan(scan=self.scan_1)
result = self.storage.execute(self.SEL_SCANS)
self.assertEqual(result, expected)
def test_update_scan(self):
self.prepare_tables()
self.scan_1.end = 456
self.storage.update_scan(self.scan_1)
expected = [(456, )]
result = self.storage.execute(
("SELECT scan_end FROM scans WHERE ROWID=56", ))
self.assertEqual(result, expected)
def test_get_scan_id(self):
expected = 56
result = self.storage.get_scan_id(self.scan_1)
self.assertEqual(result, expected)
def test_get_scan_id(self):
self.prepare_tables()
expected = self.scan_1.rowid
self.scan_1.rowid = None
result = self.storage.get_scan_id(self.scan_1)
self.assertEqual(result, expected)
def test_get_scan_id_without_results(self):
self.prepare_tables()
expected = None
self.scan_1.rowid = None
self.scan_1._scanner = None
result = self.storage.get_scan_id(self.scan_1)
self.assertEqual(result, expected)
def test_get_scans(self):
self.prepare_tables()
result = self.storage.get_scans(scanner_name='test_name',
protocol=TransportProtocol.UDP,
amount=2)
self.assertEqual([self.scan_1, self.scan_2], result)
for obj in self.scan_1, self.scan_2:
self.assertEqual(result[result.index(obj)].rowid, obj.rowid)
def test_get_scans_by_node(self):
self.prepare_tables()
expected = [self.scan_1, self.scan_2]
result = self.storage.get_scans_by_node(node=self.node_3,
scan=self.scan_1)
self.assertEqual(result, expected)
def test_get_scans_by_sec_scan(self):
self.prepare_tables()
expected = [self.scan_3, self.scan_1, self.scan_2]
result = self.storage.get_scans_by_security_scan(
port=self.port_1, exploit=self.exploit_1)
self.assertEqual(result, expected)
def test_get_scans_by_sec_scan_second(self):
self.prepare_tables()
expected = [self.scan_2]
result = self.storage.get_scans_by_security_scan(
port=self.port_1, exploit=self.exploit_3)
self.assertEqual(result, expected)
def test_get_scan_by_id(self):
self.prepare_tables()
expected = self.scan_1
result = self.storage.get_scan_by_id(56)
self.assertEqual(result, expected)
def test_get_scan_by_id_no_results(self):
self.storage.connect()
self.storage.init_schema()
result = self.storage.get_scan_by_id(15)
self.assertEqual(result, None)
def test_save_vulnerabilities(self):
self.storage.connect()
self.storage.remove_all()
self.storage.init_schema()
expected = [(56, 1, '127.0.0.1', 17, 45, 14, 1, 'CVE-2017', '6.7',
'test_output_1', 13),
(56, 1, '127.0.0.1', 17, 45, 14, 2, 'CVE-2016', '3.7',
'test_output_3', 15)]
self.storage.save_vulnerabilities(
vulnerabilities=[self.vulnerability_1, self.vulnerability_3],
scan=self.scan_1)
result = self.storage.execute(self.SEL_VULNS)
self.assertCountEqual(result, expected)
def test_scans(self):
self.prepare_tables()
self.assertEqual(self.storage.scans(2, 0), [self.scan_3, self.scan_1])
self.assertEqual(self.storage.scans(2, 1), [self.scan_2, self.scan_4])
def ports_scans_by_scan(self):
self.prepare_tables()
expected = [
self.port_scan_1, self.port_scan_2, self.port_scan_3,
self.port_scan_4
]
result = self.storage.ports_scans_by_scan(self.scan_1)
def test_expire_vulnerability(self):
self.prepare_tables()
result = self.storage.expire_vulnerability(self.vulnerability_3)
self.assertEqual(result.expiration_time, 178)
def test_active_vulnerabilities(self):
self.prepare_tables()
expected = [self.vulnerability_3]
result = self.storage.active_vulnerabilities()
self.assertEqual(result, expected)
def test_expire_vulnerabilities(self):
self.prepare_tables()
expected = []
self.storage.expire_vulnerabilities()
result = self.storage.active_vulnerabilities(timestamp=400)
self.assertEqual(result, expected)
def test_active_vulnerabilities_different_timestamps(self):
self.prepare_tables()
expected = [self.vulnerability_3]
result = self.storage.active_vulnerabilities(timestamp=0)
self.assertEqual(result, expected)
expected = []
result = self.storage.active_vulnerabilities(timestamp=40)
self.assertEqual(result, expected)
def test_portdetection_vulns(self):
self.prepare_tables()
expected = {
'name': 'tftp server name',
'version': '6.7.8',
'banner': 'HERE IS TFTP\n\n\n > ',
'cpe': 'test:cpe',
'protocol': 'tftp',
'os_name': 'os name',
'os_version': 'os version',
'os_cpe': 'test:os:cpe'
}
result = self.storage.portdetection_vulns(self.vulnerability_5)
self.assertEqual(result, expected)
def __init__(self, backend):
self.backed = backend
self.storage = Storage()
if self.storage.fetch_status() != "disconnected":
self.storage.update_status("disconnected")
class Daemon:
running = None
thread = None
storage = None
config = None
def __init__(self, backend):
self.backed = backend
self.storage = Storage()
if self.storage.fetch_status() != "disconnected":
self.storage.update_status("disconnected")
def start(self):
self.running = True
if self.thread is None:
self.thread = Thread(target=self.run)
if not self.thread.is_alive():
self.thread.start()
def stop(self):
self.log("Disconnecting")
self.running = False
while self.thread and self.thread.is_alive():
time.sleep(0.1)
self.emit("disconnected")
def run(self):
self.storage = Storage()
self.config = Config()
interface = Interface(port=self.config.read("port"))
try:
self.log("Connecting")
interface.connect()
self.emit("connected")
self.log("Connected")
while self.running:
data = interface.read()
self.log(json.dumps(data))
if data:
data["name"] = self.config.read("name")
self.update(data)
self.storage.store_measurement(data)
time.sleep(self.config.read("rate"))
except (KeyboardInterrupt, SystemExit):
raise
except:
logging.exception(sys.exc_info()[0])
self.emit("log", traceback.format_exc())
finally:
interface.close()
self.emit("disconnected")
self.log("Disconnected")
self.thread = None
def update(self, data):
format = Format()
table = []
for name in format.table_fields:
callback = getattr(format, name)
table.append(callback(data))
graph = {}
for name in format.graph_fields:
if name == "timestamp":
callback = getattr(format, name)
value = callback(data)
else:
value = data[name]
graph[name] = value
self.emit("update", json.dumps({
"table": table,
"graph": graph,
}))
def emit(self, event, data=None):
if event == "log":
self.storage.log(data)
elif event in [
"connecting", "connected", "disconnecting", "disconnected"
]:
self.storage.update_status(event)
self.backed.emit(event, data)
def log(self, message):
prefix = arrow.now().format("YYYY-MM-DD HH:mm:ss") + " - "
self.emit("log", prefix + message + "\n")
def __init__(self, name):
self.name = name
self._event_storage = Storage()
def __init__(self, client):
self.client = client
self.Storage = Storage()
self.created_channels = []
class Index:
config = None
storage = None
def __init__(self):
pass
def register(self):
blueprint = Blueprint("index", __name__, template_folder="templates")
blueprint.add_url_rule("/", "default", self.render_default)
blueprint.add_url_rule("/data", "data", self.render_data)
blueprint.add_url_rule("/graph", "graph", self.render_graph)
blueprint.add_url_rule("/graph.json", "graph_data",
self.render_graph_data)
blueprint.add_url_rule("/ble", "ble", self.render_ble)
blueprint.add_url_rule("/serial", "serial", self.render_serial)
blueprint.context_processor(self.fill)
return blueprint
def init(self):
self.config = Config()
self.storage = Storage()
def fill(self):
variables = {
"rd_user_version": version,
"format": Format(),
"url_for": self.url_for,
"version": self.config.read("version", "UM34C"),
"port": self.config.read("port", ""),
"rate": str(self.config.read("rate", 1.0)),
"name": self.config.read("name",
arrow.now().format("YYYY-MM-DD")),
"ble_address": self.config.read("ble_address"),
}
status = self.storage.fetch_status()
variables["status"] = status.title()
variables["connect_disabled"] = status != "disconnected"
variables[
"connect_button"] = "Connect" if status == "disconnected" else "Disconnect"
return variables
def render_default(self):
self.init()
self.storage.clear_log()
log = self.storage.fetch_log()
return render_template("default.html", log=log, page="default")
def render_data(self):
self.init()
names, selected = self.prepare_selection()
name = self.storage.translate_selected_name(selected)
if request.args.get("export") == "":
string = io.StringIO()
writer = csv.writer(string)
format = Format()
names = []
for field in format.export_fields:
names.append(format.field_name(field))
writer.writerow(names)
for item in self.storage.fetch_measurements(name):
values = []
for field in format.export_fields:
if field == "time":
values.append(format.time(item))
else:
values.append(item[field])
writer.writerow(values)
output = make_response(string.getvalue())
output.headers[
"Content-Disposition"] = "attachment; filename=" + name + ".csv"
output.headers["Content-type"] = "text/csv"
return output
elif request.args.get("destroy") == "":
self.storage.destroy_measurements(name)
flash("Measurements with session name '" + name + "' were deleted",
"danger")
return redirect(request.path)
page = request.args.get("page", 1, int)
limit = 100
offset = limit * (page - 1)
count = self.storage.fetch_measurements_count(name)
pages = self.prepare_pages(name, page, limit, count)
measurements = self.storage.fetch_measurements(name, limit, offset)
return render_template(
"data.html",
names=names,
selected=selected,
measurements=measurements,
page="data",
pages=pages,
)
def prepare_pages(self, name, page, limit, count, blocks=10):
first_page = 1
related = 3
last_page = int(ceil(count / limit))
steps = set(
range(max((first_page, page - related)),
min((last_page, page + related)) + 1))
quotient = (last_page - 1) / blocks
if len(steps) > 1:
for index in range(0, blocks):
steps.add(round(quotient * index) + first_page)
steps.add(last_page)
steps = sorted(steps)
pages = []
for number in steps:
pages.append({
"number": number,
"link": url_for("index.data", page=number, name=name),
"current": number == page,
})
return pages
def render_graph(self):
self.init()
names, selected = self.prepare_selection()
last_measurement = None
if selected == "":
last_measurement = self.storage.fetch_last_measurement()
return render_template("graph.html",
names=names,
selected=selected,
item=last_measurement,
left_axis="voltage",
right_axis="current",
page="graph")
def render_graph_data(self):
self.init()
selected = request.args.get("name")
name = self.storage.translate_selected_name(selected)
left_axis = request.args.get("left_axis")
right_axis = request.args.get("right_axis")
format = Format()
data = []
for item in self.storage.fetch_measurements(name):
if left_axis in item:
data.append({
"date": format.timestamp(item),
"left": item[left_axis],
"right": item[right_axis],
})
return jsonify(data)
def prepare_selection(self):
names = self.storage.fetch_measurement_names()
selected = request.args.get("name")
if not selected:
selected = ""
return names, selected
def fill_config_from_parameters(self):
value = request.args.get("version")
if value is not None:
self.config.write("version", value)
value = request.args.get("name")
if value is not None:
self.config.write("name", value)
value = request.args.get("rate")
if value is not None:
self.config.write("rate", float(value))
def render_ble(self):
self.init()
self.fill_config_from_parameters()
return render_template("ble.html")
def render_serial(self):
self.init()
self.fill_config_from_parameters()
return render_template("serial.html")
def url_for(self, endpoint, **values):
if endpoint == "static":
filename = values.get("filename", None)
if filename:
file_path = static_path + "/" + filename
values["v"] = int(os.stat(file_path).st_mtime)
return url_for(endpoint, **values)
def init(self):
self.config = Config()
self.storage = Storage()
def test_eval_image(self):
s = Storage(cfg.config)
evts = tflogs.get_events(cfg.folder,'/tmp',s,full=True)
print(len(evts))
print(evts)
