def test_less_standard_hosts(self):
endpoint = Endpoint.from_uri('http://123_server/')
endpoint.clean()
endpoint = Endpoint(host='456_desktop')
endpoint.clean()
endpoint = Endpoint(host='_invalid._host.com')
endpoint.clean()
def clean_hosts_run(apps, change):
def err_log(message, html_log, endpoint_html_log, endpoint):
error_suffix = 'It is not possible to migrate it. Delete or edit this endpoint.'
html_log.append({**endpoint_html_log, **{'message': message}})
logger.error('Endpoint (id={}) {}. {}'.format(endpoint.pk, message,
error_suffix))
broken_endpoints.add(endpoint.pk)
html_log = []
broken_endpoints = set()
Endpoint_model = apps.get_model('dojo', 'Endpoint')
Endpoint_Status_model = apps.get_model('dojo', 'Endpoint_Status')
Product_model = apps.get_model('dojo', 'Product')
for endpoint in Endpoint_model.objects.order_by('id'):
endpoint_html_log = {
'view': reverse('view_endpoint', args=[endpoint.pk]),
'edit': reverse('edit_endpoint', args=[endpoint.pk]),
'delete': reverse('delete_endpoint', args=[endpoint.pk]),
}
if endpoint.host:
if not re.match(r'^[A-Za-z][A-Za-z0-9\.\-\+]+$',
endpoint.host): # is old host valid FQDN?
try:
validate_ipv46_address(
endpoint.host) # is old host valid IPv4/6?
except ValidationError:
try:
if '://' in endpoint.host: # is the old host full uri?
parts = Endpoint.from_uri(endpoint.host)
# can raise exception if the old host is not valid URL
else:
parts = Endpoint.from_uri('//' + endpoint.host)
# can raise exception if there is no way to parse the old host
if parts.protocol:
if endpoint.protocol and (endpoint.protocol !=
parts.protocol):
message = 'has defined protocol ({}) and it is not the same as protocol in host ' \
'({})'.format(endpoint.protocol, parts.protocol)
err_log(message, html_log, endpoint_html_log,
endpoint)
else:
if change:
endpoint.protocol = parts.protocol
if parts.userinfo:
if change:
endpoint.userinfo = parts.userinfo
if parts.host:
if change:
endpoint.host = parts.host
else:
message = '"{}" use invalid format of host'.format(
endpoint.host)
err_log(message, html_log, endpoint_html_log,
endpoint)
if parts.port:
try:
if (endpoint.port is not None) and (int(
endpoint.port) != parts.port):
message = 'has defined port number ({}) and it is not the same as port number in ' \
'host ({})'.format(endpoint.port, parts.port)
err_log(message, html_log,
endpoint_html_log, endpoint)
else:
if change:
endpoint.port = parts.port
except ValueError:
message = 'uses non-numeric port: {}'.format(
endpoint.port)
err_log(message, html_log, endpoint_html_log,
endpoint)
if parts.path:
if endpoint.path and (endpoint.path != parts.path):
message = 'has defined path ({}) and it is not the same as path in host ' \
'({})'.format(endpoint.path, parts.path)
err_log(message, html_log, endpoint_html_log,
endpoint)
else:
if change:
endpoint.path = parts.path
if parts.query:
if endpoint.query and (endpoint.query !=
parts.query):
message = 'has defined query ({}) and it is not the same as query in host ' \
'({})'.format(endpoint.query, parts.query)
err_log(message, html_log, endpoint_html_log,
endpoint)
else:
if change:
endpoint.query = parts.query
if parts.fragment:
if endpoint.fragment and (endpoint.fragment !=
parts.fragment):
message = 'has defined fragment ({}) and it is not the same as fragment in host ' \
'({})'.format(endpoint.fragment, parts.fragment)
err_log(message, html_log, endpoint_html_log,
endpoint)
else:
if change:
endpoint.fragment = parts.fragment
if change and (endpoint.pk not in broken_endpoints
): # do not save broken endpoints
endpoint.save()
except ValidationError:
message = '"{}" uses invalid format of host'.format(
endpoint.host)
err_log(message, html_log, endpoint_html_log, endpoint)
try:
Endpoint.clean(
endpoint
) # still don't understand why 'endpoint.clean()' doesn't work
if change:
endpoint.save()
except ValidationError as ves:
for ve in ves:
err_log(ve, html_log, endpoint_html_log, endpoint)
if not endpoint.product:
err_log('Missing product', html_log, endpoint_html_log, endpoint)
if broken_endpoints:
logger.error(
'It is not possible to migrate database because there is/are {} broken endpoint(s). '
'Please check logs.'.format(len(broken_endpoints)))
else:
logger.info('There is not broken endpoint.')
to_be_deleted = set()
for product in Product_model.objects.all().distinct():
for endpoint in Endpoint_model.objects.filter(
product=product).distinct():
if endpoint.id not in to_be_deleted:
ep = endpoint_filter(
protocol=endpoint.protocol,
userinfo=endpoint.userinfo,
host=endpoint.host,
port=endpoint.port,
path=endpoint.path,
query=endpoint.query,
fragment=endpoint.fragment,
product_id=product.pk if product else None).order_by('id')
if ep.count() > 1:
ep_ids = [x.id for x in ep]
to_be_deleted.update(ep_ids[1:])
if change:
message = "Merging Endpoints {} into '{}'".format([
"{} (id={})".format(str(x), x.pk) for x in ep[1:]
], "{} (id={})".format(str(ep[0]), ep[0].pk))
html_log.append(message)
logger.info(message)
Endpoint_Status_model.objects\
.filter(endpoint__in=ep_ids[1:])\
.update(endpoint=ep_ids[0])
epss = Endpoint_Status_model.objects\
.filter(endpoint=ep_ids[0])\
.values('finding')\
.annotate(total=Count('id'))\
.filter(total__gt=1)
for eps in epss:
esm = Endpoint_Status_model.objects\
.filter(finding=eps['finding'])\
.order_by('-last_modified')
message = "Endpoint Statuses {} will be replaced by '{}'".format(
[
"last_modified: {} (id={})".format(
x.last_modified, x.pk) for x in esm[1:]
], "last_modified: {} (id={})".format(
esm[0].last_modified, esm[0].pk))
html_log.append(message)
logger.info(message)
esm.exclude(id=esm[0].pk).delete()
if to_be_deleted:
if change:
message = "Removing endpoints: {}".format(list(to_be_deleted))
Endpoint_model.objects.filter(id__in=to_be_deleted).delete()
else:
message = "Redundant endpoints: {}, migration is required.".format(
list(to_be_deleted))
html_log.append(message)
logger.info(message)
return html_log
