def setUp(self):
super(ApplicationResourceTest, self).setUp()
self.api_list_url = '/api/v1/applications/'
self.container_list_url = '/api/v1/containers/'
self.username = '******'
self.password = '******'
self.user = User.objects.create_user(self.username,
'*****@*****.**', self.password)
self.api_key = self.user.api_key.key
self.app_data = {
'name': 'test-app',
'description': 'test app',
'domain_name': 'test.example.com',
'backend_port': 1234,
'protocol': 'http'
}
host = Host()
host.name = 'local'
host.hostname = os.getenv('DOCKER_TEST_HOST', '127.0.0.1')
host.save()
self.host = host
self.container_data = {
'image': 'base',
'command': '/bin/bash',
'description': 'test app',
'ports': ['1234'],
'hosts': ['/api/v1/hosts/1/']
}
resp = self.api_client.post(self.container_list_url, format='json',
data=self.container_data, authentication=self.get_credentials())
self.app = Application(**self.app_data)
self.app.save()
def _update_v1(request, name, os, payload):
"""Update API v1."""
running_kernel = payload['running_kernel']['version']
start_time = timezone.now()
try:
host = Host.objects.get(name=name)
host.os = os
host.running_kernel = running_kernel
host.save() # Always update at least the modification time
host_packages = {
host_pkg.package.name: host_pkg
for host_pkg in HostPackage.objects.filter(host=host)
}
except Host.DoesNotExist:
host = Host(name=name, os=os, running_kernel=running_kernel)
host.save()
host_packages = {}
logger.info("Created Host '%s'", name)
existing_not_updated = []
installed = payload.get('installed', [])
for item in installed:
_process_installed(host, os, host_packages, existing_not_updated, item)
logger.info("Tracked %d installed packages for host '%s'", len(installed),
name)
uninstalled = payload.get('uninstalled', [])
for item in uninstalled:
existing = host_packages.get(item['name'], None)
if existing is not None:
existing.delete()
logger.info("Untracked %d uninstalled packages for host '%s'",
len(uninstalled), name)
upgradable = payload.get('upgradable', [])
for item in upgradable:
_process_upgradable(host, os, host_packages, existing_not_updated,
item)
logger.info("Tracked %d upgradable packages for host '%s'",
len(upgradable), name)
if payload['update_type'] == 'full':
# Delete orphaned entries based on the modification datetime and the list of already up-to-date IDs
res = HostPackage.objects.filter(
host=host, modified__lt=start_time).exclude(
pk__in=existing_not_updated).delete()
logger.info("Deleted %d HostPackage orphaned entries for host '%s'",
res[0], name)
def _update_v1(request, name, os, payload):
"""Update API v1."""
start_time = timezone.now()
kernel, _ = KernelVersion.objects.get_or_create(
name=payload['running_kernel']['version'], os=os)
try:
host = Host.objects.get(name=name)
host.os = os
host.kernel = kernel
host.save() # Always update at least the modification time
host_packages = {
host_pkg.package.name: host_pkg
for host_pkg in HostPackage.objects.filter(host=host)
}
except Host.DoesNotExist:
host = Host(name=name, os=os, kernel=kernel)
host.save()
host_packages = {}
logger.info("Created Host '%s'", name)
existing_not_updated = []
existing_upgradable_not_updated = []
installed = payload.get('installed', [])
for item in installed:
_process_installed(host, os, host_packages, existing_not_updated, item)
logger.info("Tracked %d installed packages for host '%s'", len(installed),
name)
uninstalled = payload.get('uninstalled', [])
for item in uninstalled:
existing = host_packages.get(item['name'], None)
if existing is not None:
existing.delete()
logger.info("Untracked %d uninstalled packages for host '%s'",
len(uninstalled), name)
upgradable = payload.get('upgradable', [])
for item in upgradable:
_process_upgradable(host, os, host_packages,
existing_upgradable_not_updated, item)
logger.info("Tracked %d upgradable packages for host '%s'",
len(upgradable), name)
if payload['update_type'] == 'full':
_garbage_collection(host, name, start_time, existing_not_updated,
existing_upgradable_not_updated)
def setUp(self):
super(ContainerResourceTest, self).setUp()
self.api_list_url = '/api/v1/containers/'
self.username = '******'
self.password = '******'
self.user = User.objects.create_user(self.username,
'*****@*****.**',
self.password)
self.api_key = self.user.api_key.key
host = Host()
host.name = 'local'
host.hostname = os.getenv('DOCKER_TEST_HOST', '127.0.0.1')
host.save()
self.host = host
self.data = {
'image': 'base',
'command': '/bin/bash',
'description': 'test app',
'ports': [],
'hosts': ['/api/v1/hosts/1/']
}
resp = self.api_client.post(self.api_list_url,
format='json',
data=self.data,
authentication=self.get_credentials())
def setUp(self):
super(ApplicationResourceTest, self).setUp()
self.api_list_url = '/api/v1/applications/'
self.container_list_url = '/api/v1/containers/'
self.username = '******'
self.password = '******'
self.user = User.objects.create_user(self.username,
'*****@*****.**',
self.password)
self.api_key = self.user.api_key.key
self.app_data = {
'name': 'test-app',
'description': 'test app',
'domain_name': 'test.example.com',
'backend_port': 1234,
'protocol': 'http'
}
host = Host()
host.name = 'local'
host.hostname = os.getenv('DOCKER_TEST_HOST', '127.0.0.1')
host.save()
self.host = host
self.container_data = {
'image': 'base',
'command': '/bin/bash',
'description': 'test app',
'ports': ['1234'],
'hosts': ['/api/v1/hosts/1/']
}
resp = self.api_client.post(self.container_list_url,
format='json',
data=self.container_data,
authentication=self.get_credentials())
self.app = Application(**self.app_data)
self.app.save()
def setUp(self):
super(ContainerResourceTest, self).setUp()
self.api_list_url = '/api/v1/containers/'
self.username = '******'
self.password = '******'
self.user = User.objects.create_user(self.username,
'*****@*****.**', self.password)
self.api_key = self.user.api_key.key
host = Host()
host.name = 'local'
host.hostname = os.getenv('DOCKER_TEST_HOST', '127.0.0.1')
host.save()
self.host = host
self.data = {
'image': 'base',
'command': '/bin/bash',
'description': 'test app',
'ports': [],
'hosts': ['/api/v1/hosts/1/']
}
resp = self.api_client.post(self.api_list_url, format='json',
data=self.data, authentication=self.get_credentials())
def create(project, creator):
# If the project argument is an integer then get
# project by `id`, otherwise by `address`
try:
project_id = int(project)
except ValueError:
project_id = None
if project_id:
project = Project.objects.get(id=project_id)
else:
project = Project.objects.get(address=project)
# Check that there are no open checkouts for the
# project (for some projects/editors/permissions this may be
# able to be relaxed in the future)
checkouts_open = Checkout.objects.filter(
Q(project=project) & (Q(status=OPEN) | Q(status=LAUNCHING)))
if checkouts_open:
# TODO Terminate the open checkout if they have not been `saved`
# for more than a certain period of time
checkout = checkouts_open[0]
raise CheckoutCreateError(
type='exists',
message='You already have a checkout open for this project',
data={'url': checkout.editor.url})
# Currently, create a native editor
# In the future, this the editor class might be chosen
# by the user
editor = Editor.create('native')
# Currently, create a native execution host
# In the future, this the editor class might be chosen
# by the user
host = Host.create('native')
return Checkout.objects.create(project=project,
editor=editor,
host=host,
creator=creator)
def populate_hosts():
if not host_file_exists:
print '[!] Did not find hosts.xml'
return
print '[*] Importing Hosts...'
# Allow changes to be made to db after nested blocks have been
# completed.
with transaction.atomic():
for line in host_file:
# Remove cruft from the end of the line:
l = line.rstrip()
# If host has a hostname:
if l[-1] == ')':
# Parse info from nmap scan.
lsplit = l.split(' ')
ipv4 = lsplit[5].strip('()')
hostname = lsplit[4].strip()
# Check if host is already in database.
# If it's not, it'd throw an error if we didn't have this check.
# If an error is thrown, then transaction.atomic() breaks,
# meaning it won't save any Hosts to the database.
if Host.objects.filter(ipv4_address=ipv4,
host_name=hostname).exists():
# Warn user.
print '[!] Host already in database: %s' % ipv4
else:
# Host doesn't exist in our db, so create a new one.
h = Host(ipv4_address=ipv4, host_name=hostname)
# Create an alert for new host.
a = Alert(ipv4_address=ipv4,
message=MSG_HOST_ADD,
date=DATE)
# Save Host to db (won't actually happen until
# 'with transaction.atomic()' is completed):
try:
h.save()
a.save()
except Exception as e:
# This shouldn't happen, unless the user screwed up
# the nmap scan.
# If we get an exception here, then the database
# will not save any of the hosts.
print '[!] %s' % e
elif l[0] != '#':
# Host has no hostname, aka it has an NXDOMAIN.
# '#' indicates last line of Nmap scan.
ipv4 = l.split(' ')[4]
# Check if host is already in database.
# If it's not, it'd throw an error if we didn't have this check.
# If an error is thrown, then transaction.atomic() breaks,
# meaning it won't save any Hosts to the database.
if Host.objects.filter(ipv4_address=ipv4).exists():
# Warn user.
print '[!] Host already in database: %s' % ipv4
else:
# Host doesn't exist in our db, so create a new one.
h = Host(ipv4_address=ipv4, host_name='NXDOMAIN')
# Create an alert for new host.
a = Alert(ipv4_address=ipv4,
message=MSG_HOST_ADD,
date=DATE)
# Save Host to db (won't actually happen until
# 'with transaction.atomic()' is completed):
try:
h.save()
a.save()
except Exception as e:
# This shouldn't happen, unless the user screwed up
# the nmap scan.
# If we get an exception here, then the database
# will not save any of the hosts.
print '[!] %s' % e
print '[*] Hosts: Done!'
def add_data():
"""Load data"""
data = pd.read_csv(r'asset_vulnerability.csv')
interation = 0
"""Get Values"""
for _ in range(data.shape[0]):
asset_hostname = data.loc[interation, 'ASSET - HOSTNAME']
asset_ip_adress = data.loc[interation, 'ASSET - IP_ADDRESS']
vulnerability_title = data.loc[interation, 'VULNERABILITY - TITLE']
vulnerability_severity = data.loc[interation,
'VULNERABILITY - SEVERITY']
vulnerability_cvss = data.loc[interation, 'VULNERABILITY - CVSS']
vulnerability_publication_date = data.loc[
interation, 'VULNERABILITY - PUBLICATION_DATE']
vulnerability_fixed = 'N'
"""Assigning Values to Vulnerability"""
if (str(vulnerability_publication_date) == 'nan'):
vulnerability = Vulnerability(
vulnerability_title=str(vulnerability_title),
vulnerability_severity=str(vulnerability_severity),
vulnerability_cvss=vulnerability_cvss,
vulnerability_publication_date=datetime.today().strftime(
'%Y-%m-%d'),
vulnerability_fixed=str(vulnerability_fixed))
else:
vulnerability = Vulnerability(
vulnerability_title=str(vulnerability_title),
vulnerability_severity=str(vulnerability_severity),
vulnerability_cvss=vulnerability_cvss,
vulnerability_publication_date=vulnerability_publication_date,
vulnerability_fixed=str(vulnerability_fixed))
"""ADD Values to database"""
try:
Compare = Vulnerability.objects.filter(
vulnerability_title=str(vulnerability_title)).get(
vulnerability_title=str(vulnerability_title))
if (str(Compare) == str(vulnerability_title)):
hosts = Host(
asset_hostname=asset_hostname,
asset_ip_adress=asset_ip_adress,
vulnerability_id=Vulnerability.objects.latest('pk').pk)
hosts.save()
pk_host = Host.objects.latest('pk').pk
lasthost = Host.objects.filter(pk=pk_host).get(pk=pk_host)
lasthost.host.add(Vulnerability.objects.latest('pk').pk)
except ObjectDoesNotExist:
vulnerability.save()
hosts = Host(
asset_hostname=asset_hostname,
asset_ip_adress=asset_ip_adress,
vulnerability_id=Vulnerability.objects.latest('pk').pk)
hosts.save()
pk_host = Host.objects.latest('pk').pk
lasthost = Host.objects.filter(pk=pk_host).get(pk=pk_host)
lasthost.host.add(vulnerability)
interation += 1
print("Successfully added !!!!")
def sync_remote_server(request, method):
tgt = sapi.minions_status()['up']
arg = [
'osfinger', 'ipv4', 'cpu_model', 'num_cpus', 'memory_info', 'disk_info'
]
data = sapi.sync_remote_server(tgt=tgt, arg=arg)
count = len(data)
update_list = []
no_update_list = []
for k, v in data[0].items():
host_info = {
'hostname': k,
'os': v['osfinger'],
'cpu': '{} * {}'.format(v['cpu_model'], v['num_cpus']),
'memory': v['memory_info'],
'disk': '|'.join(v['disk_info']),
'ip': '|'.join(v['ipv4'])
}
if method == 'create':
try:
obj = Host.objects.get(hostname=k)
except Host.DoesNotExist:
obj = Host(**host_info)
obj.save()
# records
Record.objects.create(name='hosts',
asset=k,
type=1,
method='create',
before='{}',
after=host_info,
create_user='******')
else:
try:
obj = Host.objects.filter(hostname=k)
obj_info = {
'hostname': k,
'os': obj[0].os,
'cpu': obj[0].cpu,
'memory': obj[0].memory,
'disk': obj[0].disk,
'ip': obj[0].ip
}
diff = removeNone(json_tools.diff(obj_info, host_info))
if diff:
obj.update(**host_info)
# records
Record.objects.create(name='hosts',
asset=k,
type=1,
method='update',
before=obj_info,
after=host_info,
diff=diff,
create_user='******')
update_list.append(k)
else:
no_update_list.append(k)
except Host.DoesNotExist:
print("%s is not exist" % k)
print("update_list: %s" % update_list)
print("no_update_list: %s" % no_update_list)
return Response({"results": data, "count": count})
