def setUp(self):
env = {
"MINIO_SECRET_KEY": "imagerie",
"MINIO_ACCESS_KEY": "imagerie"
}
self.container = client.containers.run('minio/minio',
name='preptest', command='server /data', remove=True,
environment=env, detach=True, ports={'9000/tcp': 9002})
logger.debug(self.container)
self.s3_storage = Storage(prep_job.s3_storage)
self.gcs_storage = Storage(prep_job.gcs_storage)
pth = os.path.join(PARENT_DIR, "resources/gummies2/")
# self.gcs_storage.upload_path(pth,'{}'.format('projects'))
self.s3_storage.upload_path(pth, '{}'.format('projects'))
def post(self):
"""
Executes a prep job to create an image corpus for training.
Use this method to start a prep job.
"""
job_def = request.json
job_def['process_json'] = True # Hardcode to process json file from project folder
job = Job(job_def['name'],job_def)
job.type = 'preprocess'
dt = newdt.now()
job.start_time = int(dt.timestamp()*1000)
job.request = {'full_path': request.full_path,'remote_addr':request.remote_addr,'method':request.method}
jb = aug_queue.enqueue(
preprocess, job,job_timeout=-1,result_ttl=86400,ttl=-1)
jb.meta['job_def'] = job_def
dt = newdt.now()
jb.meta['job_init_time'] = str(int(dt.timestamp()*1000))
jb.status = 'Running'
jb.save_meta()
json_str = job.to_json_string()
st = {
'BUCKET' : job.bucket,
'USE_GCS' : job.use_gcs,
'ACCESS_KEY' : access_key,
'SECRET_KEY' : secret_key,
'S3_URL' : s3_url
}
storage = Storage(st)
storage.upload_data(json_str,'jobs/running/{}_0_preprocess_r_{}.json'.format(str(job.start_time),jb.id),contentType='application/json')
storage.upload_data(json_str,'jobs/all/{}_0_preprocess_r_{}.json'.format(str(job.start_time),jb.id),contentType='application/json')
return {
"status": jb.status,
'job_id': jb.id,
'meta':jb.meta},201
def on_open(self, sid, data):
self.init()
data = json.loads(data)
self.config.write("version", data["version"])
self.config.write("port", data["port"])
storage = Storage()
last = storage.fetch_last_measurement_by_name(data["name"])
if last:
now = arrow.now()
if now.timestamp - int(last["timestamp"]) > 3600:
match = re.match(
".+( [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2})$",
data["name"])
if match:
data["name"] = data["name"][:-len(match.group(1))]
data["name"] += " " + arrow.now().format("YYYY-MM-DD HH:mm")
self.config.write("name", data["name"])
try:
self.config.write("rate", float(data["rate"]))
except ValueError:
pass
self.emit("connecting")
self.daemon.start()
def sample(self):
env = self.env
state = np.array(env.reset())
self.storage = Storage(self.config.rollout_length)
cumm_return = 0
for _ in range(self.config.rollout_length):
self.env.render()
preditions = self.network(state)
next_state, rewards, done, _ = env.step(preditions['actions'])
#reward = reward_normlizer(reward)
self.storage.add(preditions)
cumm_return += rewards
self.storage.add({'rewards': torch.from_numpy(np.asarray(rewards)).unsqueeze(-1).float(),
'mask': torch.from_numpy(np.asarray(1 - done)).unsqueeze(-1).float(),
'states': torch.from_numpy(state).float().unsqueeze(0)})
# if done :
# state = np.array(env.reset())
# else :
state = np.array(next_state)
print('cummulative return', cumm_return)
print('avg return', cumm_return / self.config.rollout_length)
prediction = self.network(state)
self.storage.add(prediction)
self.storage.placeholder()
advantages = torch.Tensor(np.zeros((1)))
returns = preditions['values'].detach()
return compute_gae(advantages, returns, self.storage, self.config)
def run(self):
self.storage = Storage()
self.config = Config()
self.interface = Wrapper()
try:
self.log("Connecting")
self.retry(self.interface.connect)
self.emit("connected")
self.log("Connected")
while self.running:
data = self.retry(self.interface.read)
if isinstance(data, str):
if data in ["disconnected", "connected"]:
self.disconnect()
return
raise Exception(data)
else:
self.log(json.dumps(data))
if data:
data["name"] = self.config.read("name")
self.update(data)
self.storage.store_measurement(data)
sleep(self.config.read("rate"))
except (KeyboardInterrupt, SystemExit):
raise
except:
logging.exception(sys.exc_info()[0])
self.emit("log", traceback.format_exc())
self.emit("log-error")
finally:
self.disconnect()
def run(self):
self.storage = Storage()
self.config = Config()
interface = Interface(port=self.config.read("port"))
try:
self.log("Connecting")
interface.connect()
self.emit("connected")
self.log("Connected")
while self.running:
data = interface.read()
self.log(json.dumps(data))
if data:
data["name"] = self.config.read("name")
self.update(data)
self.storage.store_measurement(data)
time.sleep(self.config.read("rate"))
except (KeyboardInterrupt, SystemExit):
raise
except:
logging.exception(sys.exc_info()[0])
self.emit("log", traceback.format_exc())
finally:
interface.close()
self.emit("disconnected")
self.log("Disconnected")
self.thread = None
def __init__(self, backend, on_receive, on_receive_interval):
self.backed = backend
self.on_receive = on_receive
self.on_receive_interval = on_receive_interval
self.storage = Storage()
if self.storage.fetch_status() != "disconnected":
self.storage.update_status("disconnected")
self.loop = asyncio.new_event_loop()
def setUp(self):
super(NmapToolTest, self).setUp()
self.exploit = Exploit(exploit_id=1,
name='test_name',
risk_level=RiskLevel.NONE)
self.exploit2 = Exploit(exploit_id=2,
name='test_name',
risk_level=RiskLevel.HIGH)
self.exploit_conf_args = Exploit(exploit_id=3)
self.exploit_conf_args.name = 'test_name2'
self.exploit_conf_args.risk_level = RiskLevel.HIGH
self.config = {
'scripts': {
'test_name': {
'args': 'test_args'
},
'test_name2': {
'args': MagicMock()
}
}
}
self.cfg = {
'tools': {
'nmap': {
'disable_scripts': [],
},
'common': {
'rate': 1337,
'http': {
'useragent': 'test_useragent'
}
}
},
'config_filename': ''
}
self.exploits = [self.exploit, self.exploit2]
self.port = Port(number=13,
transport_protocol=TransportProtocol.TCP,
node=Node(node_id=1,
ip=ipaddress.ip_address('127.0.0.1')))
self.port.scan = Scan(start=14, end=13)
self.port.protocol = 'test_service'
self.aucote = MagicMock(storage=Storage(":memory:"))
self.context = ScanContext(aucote=self.aucote,
scanner=MagicMock(scan=Scan()))
self.nmap_tool = NmapTool(context=self.context,
exploits=self.exploits,
port=self.port,
config=self.config)
def __init__(self, stop_crit, stop_value=None, group_by_keys=None, window_id=None, storage_mode = None):
self.stop_crit = stop_crit
self.window_id = cuid.cuid() if not window_id else window_id
self.group_by_keys = group_by_keys if group_by_keys else []
self.stop_value = stop_value
self.cache = Storage(storage_mode)
self.existing_keys = set()
self._count = None
self.reset_window()
def run(self):
self.storage = Storage()
self.config = Config()
interface = None
version = self.config.read("version")
if version.startswith("TC"):
ble = True
self.interface = self.backed.ble
else:
ble = False
self.interface = interface = Interface(
port=self.config.read("port"))
if version == "UM25C":
interface.enable_higher_resolution()
try:
self.log("Connecting")
if ble:
self.backed.ble.connect()
else:
interface.connect()
self.emit("connected")
self.log("Connected")
while self.running:
if ble:
data = self.backed.ble.read()
else:
data = interface.read()
self.log(json.dumps(data))
if data:
data["name"] = self.config.read("name")
self.update(data)
self.storage.store_measurement(data)
time.sleep(self.config.read("rate"))
except (KeyboardInterrupt, SystemExit):
raise
except:
logging.exception(sys.exc_info()[0])
self.emit("log", traceback.format_exc())
self.emit("log-error")
finally:
if ble:
self.backed.ble.close()
else:
interface.close()
self.emit("disconnected")
self.log("Disconnected")
self.thread = None
def __init__(self, client):
self.client = client
self.whois_messages = dict()
self.prev = '⏪'
self.next = '⏩'
self.beginning = '⏮'
self.end = '⏭'
self.stop = '⏹'
self.emojis = [
self.beginning, self.prev, self.next, self.end, self.stop
]
self.Storage = Storage()
def post(self):
"""
Executes a training.
Use this method to start a training.
"""
job_def = request.json
job = Job(job_def['name'], job_def)
job.type = 'train'
dt = newdt.now()
job.start_time = int(dt.timestamp() * 1000)
job.request = {
'full_path': request.full_path,
'remote_addr': request.remote_addr,
'method': request.method
}
if hasattr(job, 'ml_engine') and job.ml_engine:
jb = train_queue.enqueue(train_mlengine,
job,
job_timeout=-1,
result_ttl=-1)
else:
jb = train_queue.enqueue(train_job_method, job, job_timeout=-1)
jb.meta['job_init_time'] = str(int(dt.timestamp() * 1000))
jb.meta['job_def'] = job_def
jb.save_meta()
json_str = job.to_json_string()
st = {
'BUCKET': job.bucket,
'USE_GCS': job.use_gcs,
'ACCESS_KEY': access_key,
'SECRET_KEY': secret_key,
'S3_URL': s3_url
}
storage = Storage(st)
storage.upload_data(json_str,
'jobs/running/{}_0_train_r_{}.json'.format(
str(job.start_time), jb.id),
contentType='application/json')
storage.upload_data(json_str,
'jobs/all/{}_0_train_r_{}.json'.format(
str(job.start_time), jb.id),
contentType='application/json')
return {
"status": jb.get_status(),
'job_id': jb.id,
'meta': jb.meta
}, 201
def run(self):
self.storage = Storage()
self.config = Config()
self.interface = Wrapper()
try:
self.log("Connecting")
self.retry(self.interface.connect)
self.emit("connected")
self.log("Connected")
name = self.config.read("name")
interval = float(self.config.read("rate"))
version = self.config.read("version")
session_id = self.storage.create_session(name, version)
while self.running:
begin = timer()
data = self.retry(self.interface.read)
if isinstance(data, str):
if data in ["disconnected", "connected"]:
self.disconnect()
return
raise Exception(data)
else:
self.log(json.dumps(data))
if data:
data["session_id"] = session_id
self.update(data, version)
self.storage.store_measurement(data)
measurement_runtime = timer() - begin
sleep_time = interval - measurement_runtime
if sleep_time > 0:
sleep(sleep_time)
except (KeyboardInterrupt, SystemExit):
raise
except:
logging.exception(sys.exc_info()[0])
self.emit("log", traceback.format_exc())
self.emit("log-error")
finally:
self.disconnect()
def __init__(self,
command_prefix: typing.Union[str, typing.Callable[
[discord.Message], typing.Awaitable]] = None,
**options):
if not command_prefix:
command_prefix = get_prefix
super().__init__(command_prefix, **options)
self.config = Config()
self.logger = FakeLogger()
self.db = Storage(self)
self.commands_used = collections.Counter()
with open("credentials.json", "r") as f:
credentials = json.load(f)
self.token = credentials["discord_token"]
self.uptime = datetime.datetime.utcnow()
self.loop.set_debug(True)
def on_open(self, sid, data):
self.init()
data = json.loads(data)
self.config.write("version", data["version"])
if "port" in data:
self.config.write("port", data["port"])
if "ble_address" in data:
self.config.write("ble_address", data["ble_address"])
else:
data["ble_address"] = self.config.read("ble_address")
storage = Storage()
last = storage.fetch_last_measurement_by_name(data["name"])
if last:
if time() - int(last["timestamp"]) > 3600:
match = re.match(
".+( [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2})$",
data["name"])
if match:
data["name"] = data["name"][:-len(match.group(1))]
data["name"] += " " + pendulum.now().format("YYYY-MM-DD HH:mm")
if not data["name"]:
data["name"] = "My measurement"
self.config.write("name", data["name"])
try:
self.config.write("rate", float(data["rate"]))
except ValueError:
pass
tc_ble = data["version"].startswith(
"TC") and "USB" not in data["version"]
if tc_ble and ("ble_address" not in data or not data["ble_address"]):
self.daemon.log("BLE address is missing. Select address in Setup")
return
self.emit("connecting")
self.daemon.start()
def __init__(self, exploits, kudu_queue, tools_config):
self.exploits = exploits
self._kudu_queue = kudu_queue
signal.signal(signal.SIGINT, self.signal_handler)
signal.signal(signal.SIGTERM, self.signal_handler)
signal.signal(signal.SIGHUP, self.graceful_stop)
self.load_tools(tools_config)
self._storage = Storage(conn_string=cfg['storage.db'], nodes_limit=cfg['storage.max_nodes_query'])
self.ioloop = IOLoop.current()
self.topdis = Topdis(cfg['topdis.api.host'], cfg['topdis.api.port'], cfg['topdis.api.base'])
self.async_task_managers = {
TaskManagerType.SCANNER: AsyncTaskManager.instance(name=TaskManagerType.SCANNER.value,
parallel_tasks=10),
TaskManagerType.REGULAR: AsyncTaskManager.instance(name=TaskManagerType.REGULAR.value,
parallel_tasks=cfg['service.scans.parallel_tasks']),
TaskManagerType.QUICK: AsyncTaskManager.instance(name=TaskManagerType.QUICK.value, parallel_tasks=30)
}
for task_manager in self.async_task_managers.values():
throttling_consumer = ThrottlingConsumer(manager=task_manager)
self.ioloop.add_callback(partial(cfg.add_rabbit_consumer, throttling_consumer))
self.ioloop.add_callback(throttling_consumer.consume)
self.web_server = WebServer(self, cfg['service.api.v1.host'], cfg['service.api.v1.port'],
path=cfg['service.api.path'])
self._tftp_thread = TFTPThread(cfg['tftp.host'], cfg['tftp.port'], timeout=cfg['tftp.timeout'],
min_port=cfg['tftp.min_port'], max_port=cfg['tftp.max_port'])
self._storage_thread = StorageThread(storage=self._storage)
self.scanners = []
if cfg.toucan:
for task_manager in self.async_task_managers.values():
cfg.toucan_monitor.register_toucan_key(key='throttling.rate', add_prefix=False, default=1,
callback=task_manager.change_throttling_toucan)
def __init__(self, client):
self.client = client
self.Storage = Storage()
self.profile_messages = dict()
self.profile_status = dict()
self.steam_emoji = None
self.xbox_emoji = None
self.psn_emoji = None
self.nintendo_emoji = None
self.minecraft_emoji = None
self.origin_emoji = None
self.blizzard_emoji = None
self.bethesda_emoji = None
self.twitch_emoji = None
self.mixer_emoji = None
self.youtube_emoji = None
self.twitter_emoji = None
self.reddit_emoji = None
self.itchio_emoji = None
self.social_emoji = None
self.sot_emoji = None
self.game_emoji = '🎮'
self.game_emoji_url = 'https://emojipedia-us.s3.dualstack.us-west-1.amazonaws.com/thumbs/120/google/146/video-game_1f3ae.png'
self.stop_emoji = '⏹'
self.emojis = list()
self.social_platforms = [
'twitch', 'mixer', 'youtube', 'twitter', 'reddit', 'itchio'
]
self.gt_platforms = [
'steam', 'xbox', 'psn', 'nintendo', 'minecraft', 'origin',
'blizzard', 'bethesda'
]
import wx
from utils.storage import Storage
from windows.mainWindow import MainWindow
if __name__ == "__main__":
app = wx.App()
with Storage() as s:
MainWindow(s)
app.MainLoop()
def test_eval_image(self):
s = Storage(cfg.config)
evts = tflogs.get_events(cfg.folder,'/tmp',s,full=True)
print(len(evts))
print(evts)
def main():
env = gym.make(env_name)
goal_object = Goal(min_position=env.env.min_position,
max_position=env.env.max_position,
step=0.2)
meta_policy, target_meta_policy, policy, target_policy = get_policies(
env, goal_object)
optimizer = Optimizer(meta_policy, target_meta_policy, policy,
target_policy, mini_batch_size, discount,
learning_rate, update_epochs)
episode_rewards = deque(maxlen=50)
get_meta_epsilon = lambda episode: np.exp(-episode * e_meta_decay)
get_epsilon = lambda episode: np.exp(-episode * e_decay)
frame = 0
meta_frame = 0
for eps in range(0, n_eps + 1):
print('Episode', eps)
if eps % 5 == 0:
episode_rewards.append(
test_env(target_meta_policy, target_policy, goal_object))
print('Avg reward', np.mean(episode_rewards))
storage = Storage(device=device)
storage_meta = Storage(device=device)
meta_state = env.reset()
state = meta_state.copy()
state = torch.FloatTensor(state).to(device)
done = False
for step in range(10):
goal = meta_policy.act(meta_state, get_meta_epsilon(meta_frame))
goal_value = torch.FloatTensor(
[goal_object.get_goal(int(goal.item()))]).to(device)
print('Predicted goal', goal_value)
goal_reached = False
for i in range(30):
joint_state = torch.cat((state, goal_value), axis=0).to(device)
with torch.no_grad():
action = policy.act(joint_state, get_epsilon(frame))
next_state, reward, done, _ = env.step(int(action.item()))
intrinsic_reward = get_intrinsic_reward(goal_value, next_state)
goal_reached = True if intrinsic_reward else False
joint_next_state = np.concatenate(
[next_state, [goal_value.item]], axis=0)
storage.push(joint_state, action, intrinsic_reward,
joint_next_state, done)
extrinsic_reward += reward
state = next_state
state = torch.FloatTensor(state).to(device)
frame += 1
if done or goal_reached:
break
storage_meta.push(meta_state, goal, extrinsic_reward, next_state,
done) # do i really pass here unnormalized goal?
meta_state = state
meta_frame += 1
if done:
break
storage.compute()
storage_meta.compute()
loss_meta, loss = optimizer.update(storage_meta, storage)
if eps % target_policy_update:
target_meta_policy.load_state_dict(meta_policy.state_dict())
target_policy.load_state_dict(policy.state_dict())
with open('metrics.csv', 'a') as metrics:
metrics.write('{},{}\n'.format(loss_meta, loss))
def __init__(self, client):
self.client = client
self.Storage = Storage()
self.created_channels = []
def test_list_metadata(self):
s = Storage(cfg.config)
self._test_list_metadata(s)
s = Storage(cfg.gsconfig)
self._test_list_metadata(s)
def __init__(self, name):
self.name = name
self._event_storage = Storage()
def test_write_string(self):
s = Storage(cfg.config)
self._test_write_string(s)
s = Storage(cfg.gsconfig)
self._test_write_string(s)
def __init__(self, backend):
self.backed = backend
self.storage = Storage()
if self.storage.fetch_status() != "disconnected":
self.storage.update_status("disconnected")
def run(args=None, embedded=False):
if not args:
args = parse_cli()
port = args.port
daemon = "daemon" in args and args.daemon
app = Flask(__name__, static_folder=static_path)
app.config["embedded"] = embedded
app.register_blueprint(Index().register())
logger = logging.getLogger()
logger.setLevel(logging.INFO)
formatter = logging.Formatter("%(asctime)s - %(levelname)s - %(message)s")
console = StreamHandler()
console.setLevel(logging.DEBUG)
console.setFormatter(formatter)
logger.addHandler(console)
if not app.debug:
file = TimedRotatingFileHandler(data_path + "/error.log",
when="w0",
backupCount=14)
file.setLevel(logging.ERROR)
file.setFormatter(formatter)
logger.addHandler(file)
try:
config = Config()
secret_key = config.read("secret_key")
if not secret_key:
secret_key = "".join(
random.choice(string.ascii_uppercase + string.digits)
for _ in range(16))
config.write("secret_key", secret_key)
app.secret_key = secret_key
Storage().init()
sockets = socketio.Server(async_mode="threading")
app.wsgi_app = socketio.Middleware(sockets, app.wsgi_app)
sockets.register_namespace(
Backend(args.on_receive, args.on_receive_interval))
if not embedded:
def open_in_browser():
logging.info("Application is starting...")
url = "http://127.0.0.1:%s" % port
while not url_ok(url):
sleep(0.5)
logging.info("Application is available at " + url)
if not app.debug and not daemon:
webbrowser.open(url)
Thread(target=open_in_browser, daemon=True).start()
app.run(host="0.0.0.0", port=port, threaded=True, use_reloader=False)
except (KeyboardInterrupt, SystemExit):
raise
except:
logging.exception(sys.exc_info()[0])
def setUp(self):
self.maxDiff = None
self.storage = Storage(conn_string=getenv('AUCOTE_TEST_POSTGRES'))
self.scan = Scan(start=1,
end=17,
protocol=TransportProtocol.UDP,
scanner='test_name')
self.scan_1 = Scan(rowid=56,
protocol=TransportProtocol.UDP,
scanner='test_name',
start=13,
end=19)
self.scan_2 = Scan(rowid=79,
protocol=TransportProtocol.UDP,
scanner='test_name',
start=2,
end=18)
self.scan_3 = Scan(rowid=80,
protocol=TransportProtocol.UDP,
scanner='test_name_2',
start=20,
end=45)
self.scan_4 = Scan(rowid=78,
protocol=TransportProtocol.TCP,
scanner='portdetection',
start=1,
end=2)
self.node_1 = Node(node_id=1, ip=ipaddress.ip_address('127.0.0.1'))
self.node_1.name = 'test_node_1'
self.node_1.scan = Scan(start=15)
self.node_2 = Node(node_id=2, ip=ipaddress.ip_address('127.0.0.2'))
self.node_2.name = 'test_node_2'
self.node_2.scan = Scan(start=56)
self.node_3 = Node(node_id=3, ip=ipaddress.ip_address('127.0.0.3'))
self.node_3.name = 'test_node_3'
self.node_3.scan = Scan(start=98)
self.node_4 = Node(node_id=4, ip=ipaddress.ip_address('127.0.0.4'))
self.node_4.name = 'test_node_4'
self.node_4.scan = Scan(start=3)
self.node_scan_1 = NodeScan(node=self.node_1,
rowid=13,
scan=self.scan_1,
timestamp=15)
self.node_scan_2 = NodeScan(node=self.node_2,
rowid=15,
scan=self.scan_1,
timestamp=56)
self.node_scan_3 = NodeScan(node=self.node_3,
rowid=16,
scan=self.scan_1,
timestamp=98)
self.node_scan_3 = NodeScan(node=self.node_3,
rowid=17,
scan=self.scan_2,
timestamp=90)
self.port_1 = Port(node=self.node_1,
number=45,
transport_protocol=TransportProtocol.UDP)
self.port_1.scan = self.scan_1
self.port_scan_1 = PortScan(port=self.port_1,
scan=self.scan_1,
timestamp=176,
rowid=124)
self.port_2 = Port(node=self.node_2,
transport_protocol=TransportProtocol.UDP,
number=65)
self.port_2.scan = Scan(start=3, end=45)
self.port_scan_2 = PortScan(port=self.port_2,
scan=self.scan_1,
timestamp=987,
rowid=15)
self.port_3 = Port(node=self.node_3,
transport_protocol=TransportProtocol.ICMP,
number=99)
self.port_3.scan = Scan(start=43, end=180)
self.port_scan_3 = PortScan(port=self.port_3,
scan=self.scan_1,
timestamp=619,
rowid=13)
self.port_4 = Port(node=self.node_1,
number=80,
transport_protocol=TransportProtocol.UDP)
self.port_4.scan = self.scan_1
self.port_scan_4 = PortScan(port=self.port_4,
scan=self.scan_1,
timestamp=650,
rowid=480)
self.port_5 = Port(node=self.node_4,
number=22,
transport_protocol=TransportProtocol.TCP)
self.port_5.scan = self.scan_4
self.exploit_1 = Exploit(exploit_id=14,
name='test_name',
app='test_app')
self.exploit_2 = Exploit(exploit_id=2,
name='test_name_2',
app='test_app_2')
self.exploit_3 = Exploit(exploit_id=56,
name='test_name_2',
app='test_app')
self.exploit_4 = Exploit(exploit_id=0,
name='portdetection',
app='portdetection')
self.security_scan_1 = SecurityScan(exploit=self.exploit_1,
port=self.port_1,
scan=self.scan_1,
scan_start=178,
scan_end=851)
self.security_scan_2 = SecurityScan(exploit=self.exploit_2,
port=self.port_1,
scan=self.scan_1,
scan_start=109,
scan_end=775)
self.security_scan_3 = SecurityScan(exploit=self.exploit_3,
port=self.port_1,
scan=self.scan_2,
scan_start=113,
scan_end=353)
self.security_scan_4 = SecurityScan(exploit=self.exploit_1,
port=self.port_1,
scan=self.scan_3,
scan_start=180,
scan_end=222)
self.security_scan_5 = SecurityScan(exploit=self.exploit_1,
port=self.port_1,
scan=self.scan_2,
scan_start=14,
scan_end=156)
self.security_scan_6 = SecurityScan(exploit=self.exploit_2,
port=self.port_1,
scan=self.scan_2,
scan_start=56,
scan_end=780)
self.security_scan_7 = SecurityScan(exploit=self.exploit_4,
port=self.port_5,
scan=self.scan_4,
scan_start=14,
scan_end=890)
self.vuln_change_1 = PortDetectionChange(
change_time=124445,
current_finding=self.port_scan_1,
previous_finding=self.port_scan_2)
self.vuln_change_2 = PortDetectionChange(
change_time=32434,
current_finding=self.port_scan_2,
previous_finding=self.port_scan_3)
self.vulnerability_1 = Vulnerability(port=self.port_1,
output='test_output_1',
exploit=self.exploit_1,
cve='CVE-2017',
cvss=6.7,
subid=1,
vuln_time=13,
rowid=134,
scan=self.scan_1)
self.vulnerability_2 = Vulnerability(port=self.port_1,
output='test_output_2',
exploit=self.exploit_2,
cve='CWE-14',
cvss=8.9,
subid=2,
vuln_time=98,
rowid=152,
scan=self.scan_1)
self.vulnerability_3 = Vulnerability(port=self.port_1,
output='test_output_3',
exploit=self.exploit_1,
cve='CVE-2016',
cvss=3.7,
subid=2,
vuln_time=15,
rowid=153,
scan=self.scan_2)
self.vulnerability_4 = Vulnerability(port=self.port_1,
output='test_output_4',
exploit=self.exploit_2,
cve='CWE-15',
cvss=2.9,
subid=1,
vuln_time=124,
rowid=169,
scan=self.scan_2)
self.vulnerability_5 = Vulnerability(port=self.port_5,
output='',
exploit=self.exploit_4,
cve=None,
cvss=None,
subid=0,
vuln_time=124,
rowid=200,
scan=self.scan_4,
expiration_time=400)
self.vulnerability_6 = Vulnerability(port=self.port_5,
output='tftp',
exploit=self.exploit_4,
cve=None,
cvss=None,
subid=1,
vuln_time=124,
rowid=201,
scan=self.scan_4,
expiration_time=400)
self.vulnerability_7 = Vulnerability(port=self.port_5,
output='tftp server name',
exploit=self.exploit_4,
cve=None,
cvss=None,
subid=2,
vuln_time=124,
rowid=202,
scan=self.scan_4,
expiration_time=400)
self.vulnerability_8 = Vulnerability(port=self.port_5,
output='6.7.8',
exploit=self.exploit_4,
cve=None,
cvss=None,
subid=3,
vuln_time=124,
rowid=203,
scan=self.scan_4,
expiration_time=400)
self.vulnerability_9 = Vulnerability(port=self.port_5,
output='HERE IS TFTP\n\n\n > ',
exploit=self.exploit_4,
cve=None,
cvss=None,
subid=4,
vuln_time=124,
rowid=204,
scan=self.scan_4,
expiration_time=400)
self.vulnerability_10 = Vulnerability(port=self.port_5,
output='test:cpe',
exploit=self.exploit_4,
cve=None,
cvss=None,
subid=5,
vuln_time=124,
rowid=205,
scan=self.scan_4,
expiration_time=400)
self.vulnerability_11 = Vulnerability(port=self.port_5,
output='os name',
exploit=self.exploit_4,
cve=None,
cvss=None,
subid=6,
vuln_time=124,
rowid=206,
scan=self.scan_4,
expiration_time=400)
self.vulnerability_12 = Vulnerability(port=self.port_5,
output='os version',
exploit=self.exploit_4,
cve=None,
cvss=None,
subid=7,
vuln_time=124,
rowid=207,
scan=self.scan_4,
expiration_time=400)
self.vulnerability_13 = Vulnerability(port=self.port_5,
output='test:os:cpe',
exploit=self.exploit_4,
cve=None,
cvss=None,
subid=8,
vuln_time=124,
rowid=208,
scan=self.scan_4,
expiration_time=400)
def __init__(self, client):
self.client = client
self.Storage = Storage()
def init(self):
self.config = Config()
self.storage = Storage()
def create_app(test_config=None):
logging_conf_path = os.path.normpath(
os.path.join(os.path.dirname(__file__), '../logging.config'))
print(logging_conf_path)
logging.config.fileConfig(logging_conf_path)
log = logging.getLogger(__name__)
app = Flask(__name__, instance_relative_config=True)
CORS(app, resources={r"/api/*": {
"origins": "*"
}})
app.config[
'SWAGGER_UI_DOC_EXPANSION'] = settings.RESTPLUS_SWAGGER_UI_DOC_EXPANSION
app.config['RESTPLUS_VALIDATE'] = settings.RESTPLUS_VALIDATE
app.config['RESTPLUS_MASK_SWAGGER'] = settings.RESTPLUS_MASK_SWAGGER
app.config['ERROR_404_HELP'] = settings.RESTPLUS_ERROR_404_HELP
if 'JWT_SECRET_KEY' in os.environ:
app.config['JWT_SECRET_KEY'] = os.environ['JWT_SECRET_KEY']
else:
app.config['JWT_SECRET_KEY'] = 'imagerie'
app.config['PROPAGATE_EXCEPTIONS'] = True
app.config['JWT_TOKEN_LOCATION'] = ['headers', 'query_string']
app.config['JWT_BLACKLIST_ENABLED'] = True
app.config['JWT_ACCESS_TOKEN_EXPIRES'] = False
app.config['JWT_BLACKLIST_TOKEN_CHECKS'] = ['access', 'refresh']
settings.jwt = JWTManager(app)
blueprint = Blueprint('api', __name__, url_prefix='/api/capture')
authorizations = {
'Bearer Auth': {
'type': 'apiKey',
'in': 'header',
'name': 'Authorization'
},
}
settings.api = Api(blueprint,
version='1.0',
title='Flexible Vision Capture and Detection API',
description='An Image capture and object detection api',
security='Bearer Auth',
authorizations=authorizations)
settings.jwt._set_error_handler_callbacks(settings.api)
settings.api.add_namespace(auth_ns)
settings.api.add_namespace(project_ns)
settings.api.add_namespace(train_ns)
settings.api.add_namespace(predict_ns)
settings.api.add_namespace(cameras_ns)
settings.api.add_namespace(models_ns)
settings.api.add_namespace(corpus_ns)
settings.api.add_namespace(jobs_ns)
app.register_blueprint(blueprint)
@settings.jwt.token_in_blacklist_loader
def check_if_token_in_blacklist(decrypted_token):
jti = decrypted_token['jti']
return jti in settings.blacklist
config_file = Path('config.json')
if config_file.is_file():
with open('config.json') as f:
settings.data = json.load(f)
else:
if 'XRES' in os.environ:
settings.data['xres'] = int(os.environ["XRES"])
else:
settings.data['xres'] = 640
if 'WORKSTATION_NAME' in os.environ:
settings.data['workstation_name'] = os.environ['WORKSTATION_NAME']
else:
settings.data['workstation_name'] = socket.gethostname()
if 'YRES' in os.environ:
settings.data['yres'] = int(os.environ["YRES"])
else:
settings.data['yres'] = 480
if 'BW' in os.environ:
settings.data['bw'] = True
else:
settings.data['bw'] = False
if 'ACCESS_KEY' in os.environ:
settings.data['ACCESS_KEY'] = os.environ['ACCESS_KEY']
if 'SECRET_KEY' in os.environ:
settings.data['SECRET_KEY'] = os.environ['SECRET_KEY']
if 'S3_URL' in os.environ:
settings.data['S3_URL'] = os.environ['S3_URL']
if 'BUCKET' in os.environ:
settings.data['BUCKET'] = os.environ['BUCKET']
if 'USE_GCS' in os.environ and os.environ['USE_GCS'].lower() == 'true':
settings.data['USE_GCS'] = True
if 'SPINNAKER' in os.environ and os.environ['SPINNAKER'].lower(
) == 'true':
settings.data['SPINNAKER'] = True
settings.pgsystem = PySpin.System.GetInstance()
else:
settings.data['SPINNAKER'] = False
settings.data['image_type'] = '.jpg'
settings.storage = Storage(settings.data)
settings.camera = Cameras(settings.data)
#Load existing projects if you can
if 'BUCKET' in settings.data and 'URL' in settings.data:
statefile = 'projects/projects.json'
try:
jsonbody = settings.storge.download_to_string(statefile)
state = json.loads(jsonbody)
for key, value in state.items():
projects[key] = Project(key, value)
print(projects)
except ClientError:
pass
try:
os.makedirs(app.instance_path)
except OSError:
pass
@app.route('/')
def root():
return render_template('index.html')
@app.after_request
def add_header(r):
"""
Add headers to both force latest IE rendering engine or Chrome Frame,
and also to cache the rendered page for 10 minutes.
"""
r.headers["Cache-Control"] = "no-cache, no-store, must-revalidate"
r.headers["Pragma"] = "no-cache"
r.headers["Expires"] = "0"
r.headers['Cache-Control'] = 'public, max-age=0'
return r
return app