def test_less_standard_hosts(self): endpoint = Endpoint.from_uri('http://123_server/') endpoint.clean() endpoint = Endpoint(host='456_desktop') endpoint.clean() endpoint = Endpoint(host='_invalid._host.com') endpoint.clean()
def clean_hosts_run(apps, change): def err_log(message, html_log, endpoint_html_log, endpoint): error_suffix = 'It is not possible to migrate it. Delete or edit this endpoint.' html_log.append({**endpoint_html_log, **{'message': message}}) logger.error('Endpoint (id={}) {}. {}'.format(endpoint.pk, message, error_suffix)) broken_endpoints.add(endpoint.pk) html_log = [] broken_endpoints = set() Endpoint_model = apps.get_model('dojo', 'Endpoint') Endpoint_Status_model = apps.get_model('dojo', 'Endpoint_Status') Product_model = apps.get_model('dojo', 'Product') for endpoint in Endpoint_model.objects.order_by('id'): endpoint_html_log = { 'view': reverse('view_endpoint', args=[endpoint.pk]), 'edit': reverse('edit_endpoint', args=[endpoint.pk]), 'delete': reverse('delete_endpoint', args=[endpoint.pk]), } if endpoint.host: if not re.match(r'^[A-Za-z][A-Za-z0-9\.\-\+]+$', endpoint.host): # is old host valid FQDN? try: validate_ipv46_address( endpoint.host) # is old host valid IPv4/6? except ValidationError: try: if '://' in endpoint.host: # is the old host full uri? parts = Endpoint.from_uri(endpoint.host) # can raise exception if the old host is not valid URL else: parts = Endpoint.from_uri('//' + endpoint.host) # can raise exception if there is no way to parse the old host if parts.protocol: if endpoint.protocol and (endpoint.protocol != parts.protocol): message = 'has defined protocol ({}) and it is not the same as protocol in host ' \ '({})'.format(endpoint.protocol, parts.protocol) err_log(message, html_log, endpoint_html_log, endpoint) else: if change: endpoint.protocol = parts.protocol if parts.userinfo: if change: endpoint.userinfo = parts.userinfo if parts.host: if change: endpoint.host = parts.host else: message = '"{}" use invalid format of host'.format( endpoint.host) err_log(message, html_log, endpoint_html_log, endpoint) if parts.port: try: if (endpoint.port is not None) and (int( endpoint.port) != parts.port): message = 'has defined port number ({}) and it is not the same as port number in ' \ 'host ({})'.format(endpoint.port, parts.port) err_log(message, html_log, endpoint_html_log, endpoint) else: if change: endpoint.port = parts.port except ValueError: message = 'uses non-numeric port: {}'.format( endpoint.port) err_log(message, html_log, endpoint_html_log, endpoint) if parts.path: if endpoint.path and (endpoint.path != parts.path): message = 'has defined path ({}) and it is not the same as path in host ' \ '({})'.format(endpoint.path, parts.path) err_log(message, html_log, endpoint_html_log, endpoint) else: if change: endpoint.path = parts.path if parts.query: if endpoint.query and (endpoint.query != parts.query): message = 'has defined query ({}) and it is not the same as query in host ' \ '({})'.format(endpoint.query, parts.query) err_log(message, html_log, endpoint_html_log, endpoint) else: if change: endpoint.query = parts.query if parts.fragment: if endpoint.fragment and (endpoint.fragment != parts.fragment): message = 'has defined fragment ({}) and it is not the same as fragment in host ' \ '({})'.format(endpoint.fragment, parts.fragment) err_log(message, html_log, endpoint_html_log, endpoint) else: if change: endpoint.fragment = parts.fragment if change and (endpoint.pk not in broken_endpoints ): # do not save broken endpoints endpoint.save() except ValidationError: message = '"{}" uses invalid format of host'.format( endpoint.host) err_log(message, html_log, endpoint_html_log, endpoint) try: Endpoint.clean( endpoint ) # still don't understand why 'endpoint.clean()' doesn't work if change: endpoint.save() except ValidationError as ves: for ve in ves: err_log(ve, html_log, endpoint_html_log, endpoint) if not endpoint.product: err_log('Missing product', html_log, endpoint_html_log, endpoint) if broken_endpoints: logger.error( 'It is not possible to migrate database because there is/are {} broken endpoint(s). ' 'Please check logs.'.format(len(broken_endpoints))) else: logger.info('There is not broken endpoint.') to_be_deleted = set() for product in Product_model.objects.all().distinct(): for endpoint in Endpoint_model.objects.filter( product=product).distinct(): if endpoint.id not in to_be_deleted: ep = endpoint_filter( protocol=endpoint.protocol, userinfo=endpoint.userinfo, host=endpoint.host, port=endpoint.port, path=endpoint.path, query=endpoint.query, fragment=endpoint.fragment, product_id=product.pk if product else None).order_by('id') if ep.count() > 1: ep_ids = [x.id for x in ep] to_be_deleted.update(ep_ids[1:]) if change: message = "Merging Endpoints {} into '{}'".format([ "{} (id={})".format(str(x), x.pk) for x in ep[1:] ], "{} (id={})".format(str(ep[0]), ep[0].pk)) html_log.append(message) logger.info(message) Endpoint_Status_model.objects\ .filter(endpoint__in=ep_ids[1:])\ .update(endpoint=ep_ids[0]) epss = Endpoint_Status_model.objects\ .filter(endpoint=ep_ids[0])\ .values('finding')\ .annotate(total=Count('id'))\ .filter(total__gt=1) for eps in epss: esm = Endpoint_Status_model.objects\ .filter(finding=eps['finding'])\ .order_by('-last_modified') message = "Endpoint Statuses {} will be replaced by '{}'".format( [ "last_modified: {} (id={})".format( x.last_modified, x.pk) for x in esm[1:] ], "last_modified: {} (id={})".format( esm[0].last_modified, esm[0].pk)) html_log.append(message) logger.info(message) esm.exclude(id=esm[0].pk).delete() if to_be_deleted: if change: message = "Removing endpoints: {}".format(list(to_be_deleted)) Endpoint_model.objects.filter(id__in=to_be_deleted).delete() else: message = "Redundant endpoints: {}, migration is required.".format( list(to_be_deleted)) html_log.append(message) logger.info(message) return html_log