def remove_from_trillian(pk): from measurements.models import InstanceRun try: # Try to find the InstanceRun multiple times, in case of a race condition run = retry_get(InstanceRun.objects.exclude(analysed=None), pk=pk) if not run.analysed: print_warning( _("InstanceRun {pk} has not yet been analysed").format(pk=pk)) return if not run.trillian_url: # Already cleaned up return print_message( _("Deleting InstanceRun {run.pk} ({run.url}) from {run.trillian.name}" ).format(run=run)) response = requests.request( method='DELETE', url=run.trillian_url, auth=TokenAuth(run.trillian.token), timeout=(5, 15), ) print(response) if response.status_code not in [204, 404]: # 204 = deleted, 404 = doesn't exist anymore print_error( _("{run.trillian.name} didn't accept our request ({response.status_code}), retrying later" ).format(run=run, response=response)) raise RetryTaskException run.trillian_url = '' run.save() print_message( _("Trillian {run.trillian.name} deleted completed InstanceRun {run.pk}" ).format(run=run)) except RetryTaskException: raise except InstanceRun.DoesNotExist: print_warning( _("InstanceRun {pk} does not exist anymore").format(pk=pk)) return except Exception as ex: print_error( _('{name} on line {line}: {msg}').format( name=type(ex).__name__, line=sys.exc_info()[-1].tb_lineno, msg=ex)) raise RetryTaskException
def analyse_instancerunresult(pk): from measurements.models import InstanceRunResult from measurements.utils import compare_base64_images try: result = retry_get(InstanceRunResult.objects.select_for_update(), pk=pk) if result.analysed: return print_notice( _("Analysing InstanceRunResult {result.pk} ({result.instance_type}: {result.instancerun.url}) " "on {result.instancerun.trillian.name}").format(result=result)) baseline = result.instancerun.get_baseline() if not baseline: result.image_score = 0 result.resource_score = 0 result.overall_score = 0 result.analysed = timezone.now() result.save() return # If we have multiple possible combinations then test them all and choose the most positive one result.image_score, base = max([ (compare_base64_images(base.web_response['image'], result.web_response['image']), base) for base in baseline ]) # Analyse the resources base_stats = get_resource_stats(base.web_response['resources']) my_stats = get_resource_stats(result.web_response['resources']) result.resource_score = min( 1.0, my_stats['total']['ok'] / (base_stats['total']['ok'] or 1)) # Determine the overall score result.overall_score = result.image_score * result.resource_score result.analysed = timezone.now() result.save() except RetryTaskException: raise except InstanceRunResult.DoesNotExist: print_warning( _("InstanceRunResult {pk} does not exist anymore").format(pk=pk)) return except Exception as ex: print_error( _('{name} on line {line}: {msg}').format( name=type(ex).__name__, line=sys.exc_info()[-1].tb_lineno, msg=ex)) raise RetryTaskException
def delegate_to_trillian(pk): from measurements.models import InstanceRun try: # Try to find the InstanceRun multiple times, in case of a race condition run = retry_get(InstanceRun.objects.all(), pk=pk) if run.trillian_url: print_warning(_("Trillian URL already exists for InstanceRun {pk}").format(pk=pk)) return print_message(_("Pushing InstanceRun {run.pk} ({run.url}) to {run.trillian.name}").format(run=run)) response = requests.request( method='POST', url='https://{hostname}/api/v1/instanceruns/'.format(hostname=run.trillian.hostname), auth=TokenAuth(run.trillian.token), timeout=(5, 15), json={ 'url': run.url, 'callback_url': 'https://{my_hostname}{path}'.format( my_hostname=settings.MY_HOSTNAME, path=reverse('v1:instancerun-detail', kwargs={'pk': run.pk}) ), 'requested': run.testrun.requested.isoformat(), } ) if response.status_code != 201: print_error( _("{run.trillian.name} didn't accept our request ({response.status_code}), retrying later").format( run=run, response=response ) ) raise RetryTaskException run.trillian_url = response.json()['_url'] run.save() print_message(_("Trillian {run.trillian.name} accepted the task as {run.trillian_url}").format(run=run)) except RetryTaskException: raise except InstanceRun.DoesNotExist: print_warning(_("InstanceRun {pk} does not exist anymore").format(pk=pk)) return except Exception as ex: print_error(_('{name} on line {line}: {msg}').format( name=type(ex).__name__, line=sys.exc_info()[-1].tb_lineno, msg=ex )) raise RetryTaskException
def analyse_instancerun(pk): from measurements.models import InstanceRun, InstanceRunResult try: children_finished = retry_all( InstanceRunResult.objects.filter(instancerun_id=pk).values_list( 'analysed', flat=True)) if not children_finished: return run = InstanceRun.objects.select_for_update().get(pk=pk) if run.analysed or not run.finished: return print_notice( _("Analysing InstanceRun {run.pk} ({run.url}) on {run.trillian.name}" ).format(run=run)) scores = InstanceRunResult.objects \ .filter(instancerun_id=pk) \ .values_list('image_score', 'resource_score', 'overall_score') run.image_score = mean([score[0] for score in scores]) run.resource_score = mean([score[1] for score in scores]) run.overall_score = mean([score[2] for score in scores]) run.analysed = timezone.now() run.save() except RetryTaskException: raise except InstanceRun.DoesNotExist: print_warning( _("InstanceRun {pk} does not exist anymore").format(pk=pk)) return except Exception as ex: print_error( _('{name} on line {line}: {msg}').format( name=type(ex).__name__, line=sys.exc_info()[-1].tb_lineno, msg=ex)) raise RetryTaskException
def get_marvins(instance_types, current_task): # We should now be the only spooler running this task marvins = find_marvins(instance_types) if not all(marvins.values()): timeout = randrange(5, 60) print_error( _("Not enough Marvins available, missing {types}: delaying by {timeout} seconds" ).format(types=[ instance_type for instance_type, marvin in marvins.items() if marvin is None ], timeout=timeout)) # Retry without lowering the retry count raise RetryTaskException(count=current_task.setup['retry_count'], timeout=timeout) print_message( _("Found Marvins: {}").format(', '.join([ '{}: {}'.format(instance_type, marvin.name) for instance_type, marvin in marvins.items() ]))) return marvins
def execute_instancerun(pk): from measurements.models import InstanceRun, InstanceRunResult current_task = get_current_task() try: # Make sure we need to start and we don't start twice with transaction.atomic(): run = retry_get(InstanceRun.objects.select_for_update(), pk=pk) if run.started: print_notice( _('InstanceRun {pk} has already started, skipping').format( pk=pk)) return now = timezone.now() if run.requested > now: print_notice( _('InstanceRun {pk} is requested to start in the future, skipping' ).format(pk=pk)) return # We are starting! run.started = now run.save() # Log which instancerun we're working on print_message( _("Start working on InstanceRun {run.pk} ({run.url})").format( run=run)) # Do a simple DNS lookup addresses = set() for info in socket.getaddrinfo(urlparse(run.url).hostname, port=80, proto=socket.IPPROTO_TCP): family, socktype, proto, canonname, sockaddr = info addresses.add(ipaddress.ip_address(sockaddr[0])) run.dns_results = list([str(address) for address in addresses]) # First determine a baseline marvin = get_marvins(['dual-stack'], current_task)['dual-stack'] with marvin: response = requests.request(method='POST', url='http://{}:3001/browse'.format( marvin.name), json={ 'url': run.url, }, timeout=(5, 65)) if response.status_code != 200: timeout = randrange(5, 120) print_error( _("Baseline test failed, retrying in {timeout} seconds"). format(timeout=timeout)) raise RetryTaskException(timeout=timeout) baseline = response.json() # Determine which protocols to check site_v4_addresses = [ address for address in addresses if address.version == 4 ] site_v6_addresses = [ address for address in addresses if address.version == 6 ] instance_types = {'nat64', 'dual-stack'} if site_v4_addresses: instance_types.add('v4only') else: InstanceRunMessage.objects.create( instancerun=run, severity=logging.WARNING, message=gettext_noop( 'This website has no IPv4 addresses so the IPv4-only test is skipped' ), ) if site_v6_addresses: instance_types.add('v6only') else: InstanceRunMessage.objects.create( instancerun=run, severity=logging.WARNING, message=gettext_noop( 'This website has no IPv6 addresses so the IPv6-only test is skipped' ), ) marvins = get_marvins(instance_types, current_task) with FuturesSession(executor=ThreadPoolExecutor( max_workers=2 * len(marvins))) as session: with ExitStack() as stack: # Signal usage of Marvins for marvin in marvins.values(): stack.enter_context(marvin) # Start requests browse_requests = {} for instance_type, marvin in marvins.items(): browse_requests[instance_type] = session.request( method='POST', url='http://{}:3001/browse'.format(marvin.name), json={ 'url': run.url, 'timeout': 30, }, timeout=(5, 65)) ping_requests = {} for instance_type, marvin in marvins.items(): marvin_has_v4 = instance_type in ('v4only', 'dual-stack') marvin_has_nat64 = instance_type in ('nat64', ) marvin_has_v6 = instance_type in ('v6only', 'dual-stack', 'nat64') if marvin_has_v4: for address in site_v4_addresses: address_str = str(address) ping_requests.setdefault( instance_type, {})[address_str] = session.request( method='POST', url='http://{}:3001/ping4'.format( marvin.name), json={'target': address_str}, timeout=(5, 65)) if marvin_has_nat64: for address in site_v4_addresses: address_str = str( IPv6Address('64:ff9b::') + int(address)) ping_requests.setdefault( instance_type, {})[address_str] = session.request( method='POST', url='http://{}:3001/ping6'.format( marvin.name), json={'target': address_str}, timeout=(5, 65)) if marvin_has_v6: for address in site_v6_addresses: address_str = str(address) ping_requests.setdefault( instance_type, {})[address_str] = session.request( method='POST', url='http://{}:3001/ping6'.format( marvin.name), json={'target': address_str}, timeout=(5, 65)) # Wait for all the responses to come back in browse_responses = {} for instance_type, request in browse_requests.items(): browse_responses[instance_type] = request.result() ping_responses = {} for instance_type, address_requests in ping_requests.items(): for address, request in address_requests.items(): ping_responses[(instance_type, address)] = request.result() for req, response in list(browse_responses.items()) + list( ping_responses.items()): if response.status_code >= 300: print_error("{req} {url} ({code}): {json}".format( code=response.status_code, req=req, url=response.url, json=response.json())) # Check if all tests succeeded if not all([ response.status_code == 200 for response in list(browse_responses.values()) + list(ping_responses.values()) ]): timeout = randrange(5, 120) print_error( _("Not all tests completed successfully, retrying in {timeout} seconds" ).format(timeout=timeout)) raise RetryTaskException(timeout=timeout) # Parse all JSON ping_responses_json = {} for (instance_type, address), response in ping_responses.items(): ping_responses_json.setdefault(instance_type, {}) ping_responses_json[instance_type][address] = response.json( object_pairs_hook=OrderedDict) browse_responses_json = { instance_type: response.json(object_pairs_hook=OrderedDict) for instance_type, response in browse_responses.items() } # Compare dual-stack to the baseline if len(baseline['resources']) != len(browse_responses_json['dual-stack']['resources']) or \ compare_base64_images(baseline['image'], browse_responses_json['dual-stack']['image']) < 0.98: InstanceRunMessage.objects.create( instancerun=run, severity=logging.WARNING, message=gettext_noop( 'Two identical requests returned different results. ' 'Results are going to be unpredictable.'), ) for instance_type in instance_types: InstanceRunResult.objects.update_or_create( defaults={ 'ping_response': ping_responses_json[instance_type], 'web_response': browse_responses_json[instance_type], }, instancerun=run, marvin=marvins[instance_type], ) # We are starting! run.finished = timezone.now() run.save() print_message( _("Work on InstanceRun {run.pk} ({run.url}) completed").format( run=run)) except RetryTaskException: # Clear the started timestamp and messages so it can be retried, and trigger retry InstanceRun.objects.filter(pk=pk).update(started=None, finished=None) InstanceRunMessage.objects.filter(instancerun_id=pk).delete() raise except InstanceRun.DoesNotExist: print_warning( _("InstanceRun {pk} does not exist anymore").format(pk=pk)) return except Exception as ex: print_error( _('{name} on line {line}: {msg}').format( name=type(ex).__name__, line=sys.exc_info()[-1].tb_lineno, msg=ex)) print_error(format_exc()) # Clear the started timestamp and messages so it can be retried, and trigger retry InstanceRun.objects.filter(pk=pk).update(started=None, finished=None) InstanceRunMessage.objects.filter(instancerun_id=pk).delete() raise RetryTaskException
def run(): print_notice("Starting telnet relay") try: redis = StrictRedis(**private_settings.WS4REDIS_CONNECTION) subscriber = redis.pubsub() subscriber.psubscribe('server:*/events') # noinspection PyProtectedMember redis_fd = subscriber.connection._sock.fileno() sockets = [redis_fd] sessions = {} while sockets: readable, writable, exceptional = select.select(sockets, [], []) for s in readable: if s is redis_fd: # Data from websocket to telnet message = subscriber.parse_response() msg_type = message.pop(0) if msg_type != b'pmessage': continue pattern, channel, message = message match = channel_pattern.match(channel) if not match: print_warning( _("Malformed channel name: {}").format(channel)) continue exercise_id = int(match.group(1)) data = json.loads(message) if 'type' not in data or 'node_id' not in data: print_warning( _("Malformed terminal input: {}").format(message)) continue if data['type'] != 'terminal-input': # Not for us continue node_id = int(data['node_id']) key = (exercise_id, node_id) if key in sessions: session = sessions[key] else: nodes = list( ExerciseNode.objects.filter( project_id=exercise_id, id=node_id).select_related('project')) if not nodes or not isinstance(nodes[0].template_node, (WorkNode, IRRNode)): print_warning( _("Invalid node-id {node_id} provided for exercise {exercise_id}" ).format(node_id=node_id, exercise_id=exercise_id)) node = nodes[0] gns3_node = get_gns3_node(node.project.gns3_id, node.gns3_id) if gns3_node['console_type'] != 'telnet': # We can only handle telnet consoles, put on the ignore list sessions[key] = None continue if gns3_node['console_host'] == '::': host = '::1' elif gns3_node['console_host'] == '0.0.0.0': host = '127.0.0.1' else: host = gns3_node['console_host'] port = gns3_node['console'] try: session = LabTelnet(key, host, port) print_notice( _("Telnet connection to {} {} established"). format(host, port)) except ConnectionRefusedError: print_warning( _("Connection refused by {} {}").format( host, port)) continue sockets.append(session) sessions[key] = session # If there is no session then ignore the data if not session: continue # Write the data to the session try: session.write(data['data'].encode()) except EOFError: # Connection closed, clean up print_warning( _("Telnet connection to {} {} closed").format( session.host, session.port)) sockets.remove(s) del sessions[s.key] except (OSError, IOError) as e: print_exc() print_error(e) else: # Data from telnet to websocket exercise_id, node_id = s.key try: data = s.read_eager() redis_publisher = LabPublisher( facility='{}/events'.format(exercise_id), broadcast=True) redis_publisher.publish_message( RedisMessage( json.dumps({ 'type': 'terminal-output', 'node_id': node_id, 'data': b64encode(data).decode('ascii'), }))) except EOFError: # Connection closed, clean up print_warning( _("Telnet connection to {} {} closed").format( s.host, s.port)) sockets.remove(s) del sessions[s.key] except (OSError, IOError) as e: print_exc() print_error(e) except Exception as e: print_exc() print_error(e)
def submit(self): # Remember name = self.current_section_name content = self.current_section # Clear the state vars self.current_section_name = None self.current_section = '' # Do we have any data? if not name: return # Process if name == 'UUID': # This is the UUID of the node self.uuid = content.strip().lower() if not self.uuid: # We can't do anything yet return if not self.node: try: self.node = ExerciseNode.objects.get(gns3_id=self.uuid) except ExerciseNode.DoesNotExist: print_error( _('Unable to submit {section} of {uuid}').format( section=name, uuid=self.uuid)) return if name in monitor_goal_types or name in irr_goal_types: state, created = ExerciseState.objects.select_for_update( ).get_or_create(defaults={ 'state': content, }, exercise_node=self.node, goal_type=name) if state.state != content: # State changed state.state = content state.save() redis_publisher = LabPublisher(facility='{}/events'.format( self.node.project_id), broadcast=True) redis_publisher.publish_message( RedisMessage( json.dumps( { 'type': 'state', 'goal_type': name, 'node': self.node.id, 'content': content, 'ts': state.last_update, }, cls=DjangoJSONEncoder))) elif name in ['QUERY-RESULT', 'UPDATE-RESULT']: redis_publisher = LabPublisher(facility='{}/events'.format( self.node.project_id), broadcast=True) redis_publisher.publish_message( RedisMessage( json.dumps( { 'type': 'irr-query-response' if name == 'QUERY-RESULT' else 'irr-update-response', 'node': self.node.id, 'request': 'irr-query' if name == 'QUERY-RESULT' else 'irr-update', 'response': content, }, cls=DjangoJSONEncoder))) elif name != 'UUID': print_error( _("Unknown section name: [{}] from {}").format( name, self.uuid))
def run(): try: print_notice( _("Listening for state updates on {addr}:{port}").format( addr=settings.STATE_COLLECTOR['ADDRESS'], port=settings.STATE_COLLECTOR['PORT'], )) listen_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP) listen_sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) listen_sock.setblocking(False) listen_sock.bind((settings.STATE_COLLECTOR['ADDRESS'], settings.STATE_COLLECTOR['PORT'])) listen_sock.listen(128) redis = StrictRedis(**private_settings.WS4REDIS_CONNECTION) subscriber = redis.pubsub() subscriber.psubscribe('server:*/events') # noinspection PyProtectedMember redis_fd = subscriber.connection._sock.fileno() sockets = [listen_sock, redis_fd] while sockets: readable, writable, exceptional = select.select(sockets, [], []) for s in readable: if s is listen_sock: connection, client_address = s.accept() print_message( "Incoming state connection from {addr}".format( addr=client_address)) connection.setblocking(False) sockets.append(StateConnection(connection)) # Ask for ID connection.send(b"*****[ ID ]*****\n") connection.send(b"*****[ END ]*****\n") continue if s is redis_fd: # Data from websocket to telnet message = subscriber.parse_response() msg_type = message.pop(0) if msg_type != b'pmessage': continue pattern, channel, message = message match = channel_pattern.match(channel) if not match: print_warning( _("Malformed channel name: {}").format(channel)) continue exercise_id = int(match.group(1)) data = json.loads(message) if 'type' not in data or 'node_id' not in data: print_warning( _("Malformed terminal input: {}").format(message)) continue if data['type'] not in ['irr-query', 'irr-update']: # Not for us continue node_id = int(data['node_id']) # Find the connection belonging to this exercise node for sc in sockets: if isinstance(sc, StateConnection) and sc.node and \ sc.node.project_id == exercise_id and sc.node.id == node_id: break else: print_warning( _("No existing connection found for exercise {exercise} node {node}" ).format(exercise=exercise_id, node=node_id)) redis_publisher = LabPublisher( facility='{}/events'.format(exercise_id), broadcast=True) redis_publisher.publish_message( RedisMessage( json.dumps( { 'type': data['type'] + '-response', 'node': node_id, 'response': 'Server is not yet available', }, cls=DjangoJSONEncoder))) continue sc.send_message(data) continue result = s.collect_data() if not result: # End of connection print_message("Lost state connection from {addr}".format( addr=s.connection.getpeername())) sockets.remove(s) s.close() except Exception as e: print_error(e)
def analyse_testrun(pk): from measurements.models import (TestRun, InstanceRun, InstanceRunResult, TestRunAverage) try: children_finished = retry_all( InstanceRun.objects.filter(testrun_id=pk).values_list('analysed', flat=True)) if not children_finished: return run = TestRun.objects.select_for_update().get(pk=pk) if run.analysed: return print_notice( _("Analysing TestRun {run.pk} ({run.url})").format(run=run)) scores = InstanceRun.objects \ .filter(testrun_id=pk) \ .values_list('image_score', 'resource_score', 'overall_score') run.image_score = mean([score[0] for score in scores]) run.resource_score = mean([score[1] for score in scores]) run.overall_score = mean([score[2] for score in scores]) averages = InstanceRunResult.objects \ .filter(instancerun__testrun_id=pk) \ .values('marvin__instance_type') \ .annotate(image_score=Avg('image_score'), resource_score=Avg('resource_score'), overall_score=Avg('overall_score')) for average in averages: TestRunAverage.objects.update_or_create( defaults={ 'image_score': average['image_score'], 'resource_score': average['resource_score'], 'overall_score': average['overall_score'], }, testrun_id=pk, instance_type=average['marvin__instance_type']) run.analysed = timezone.now() run.save() except RetryTaskException: raise except TestRun.DoesNotExist: print_warning(_("TestRun {pk} does not exist anymore").format(pk=pk)) return except Exception as ex: print_error( _('{name} on line {line}: {msg}').format( name=type(ex).__name__, line=sys.exc_info()[-1].tb_lineno, msg=ex)) raise RetryTaskException
def execute_update_zaphod(pk): from measurements.models import InstanceRun try: run = retry_get(InstanceRun.objects.all(), pk=pk) if not run.callback_url: print_warning( _("No callback URL provided for InstanceRun {pk}").format( pk=pk)) return print_message( _("Updating InstanceRun {run.pk} ({run.url}) on {run.callback_url}" ).format(run=run)) url = urlsplit(run.callback_url) try: zaphod = Zaphod.objects.get(hostname=url.netloc) auth = TokenAuth(zaphod.token) except Zaphod.DoesNotExist: print_warning( _("Unknown Zaphod at {url.netloc}, not authenticating").format( url=url)) auth = None response = requests.request( method='PUT', url=run.callback_url, auth=auth, timeout=(5, 15), json=InstanceRunSerializer( instance=run, context={ 'expand': {'messages', 'results__marvin'}, 'exclude': { 'id', '_url', 'results__id', 'results__instancerun', 'results__instancerun_id', 'results___url', 'results__marvin__id', 'results__marvin___url' } }).data) if response.status_code != 200: print_error( _("{run.callback_url} didn't accept our data ({response.status_code}), retrying later" ).format(run=run, response=response)) raise RetryTaskException except RetryTaskException: raise except InstanceRun.DoesNotExist: print_warning( _("InstanceRun {pk} does not exist anymore").format(pk=pk)) return except Exception as ex: print_error( _('{name} on line {line}: {msg}').format( name=type(ex).__name__, line=sys.exc_info()[-1].tb_lineno, msg=ex)) print_exc() raise RetryTaskException
def sync_projects_to_db(): print_debug("Synchronising projects") # Detect projects on the server server_projects = get_gns3_projects(session=session) # Sync projects projects = Project.objects.select_subclasses() for project in projects: try: for server_project in server_projects: if server_project['project_id'].lower() == str( project.gns3_id).lower(): break else: # Where did that one go?!? print_warning( "- " + _("Project {project.name} disappeared from GNS3 server" ).format(project=project)) # project.delete() continue # Open the project so its data becomes available in the API session.post(gns3_base_url + '/v2/projects/' + server_project['project_id'] + '/open') if server_project['name'] != project.name: print_message("- " + _("Project {old_name} renamed to {new_name}" ).format(old_name=project.name, new_name=server_project['name'])) project.name = server_project['name'] project.save() # Sync nodes server_nodes = get_gns3_nodes(server_project['project_id'], session=session) nodes = project.node_set.select_subclasses() for node in nodes: for server_node in server_nodes: if server_node['node_id'].lower() == str( node.gns3_id).lower(): break else: # Where did that one go?!? print_warning("- " + _( "Node {node.name} of project {project.name} disappeared from GNS3 server" ).format(node=node, project=project)) # node.delete() continue if server_node['name'] != node.name: print_message( "- " + _("Project {project.name} node {old} renamed to {new}" ).format(project=project, old=node.name, new=server_node['name'])) node.name = server_node['name'] node.save() if server_node['properties']['mac_address'] != node.mac_address: print_message( "- " + _("Project {project.name} node {node.name} " "change MAC fix_address from {old} to {new}").format( project=project, node=node, old=node.mac_address, new=server_node['properties']['mac_address'])) node.mac_address = server_node['properties']['mac_address'] node.save() if isinstance(node, ExerciseNode): node.gns3_update_monitor_option(session=session) if isinstance(project, Exercise): running = len([ node for node in server_nodes if node['status'] == 'started' ]) if running and project.deadline and project.deadline < timezone.now( ): print_notice( _("Stopping exercise {}").format(project.name)) project.gns3_stop(session=session) if project.deadline and project.deadline < timezone.now( ) - timedelta(weeks=1): print_notice( _("Deleting exercise {}").format(project.name)) project.delete() continue except IntegrityError: print_error(" - " + _("Template is still referenced, leaving it for now"))