def terminate_idle_pools(): """Terminate any pool that doesn't have any more tasks running on it """ ec2_pools = EC2Pool.objects.filter(auto_terminate=True) for ec2_pool in ec2_pools: try: copied_pools = EC2Pool.objects.filter(copy_of=ec2_pool) all_tasks = Task.objects.filter( condor_pool=ec2_pool) | Task.objects.filter( condor_pool__in=copied_pools) running_tasks = all_tasks.filter( status='running') | all_tasks.filter(status='new') if running_tasks.count() == 0 and all_tasks.count() > 0: pool_name = ec2_pool.name log.debug( 'Terminating pool %s since no other jobs running (auto terminate)' % pool_name) #Prune the tasks so that they are disassociated from the pool for task in all_tasks: task.condor_pool = None task.set_custom_field('condor_pool_name', pool_name) task.save() if ec2_pool.get_pool_type( ) == 'ec2' and ec2_pool.copy_of == None: error_list = [] #Remove the copied pools first for copied_pool in copied_pools: try: copied_pool.delete() except Exception as e: log.exception(e) error_list += [ 'Error deleting duplicate pool', str(e) ] #Remove from bosco try: condor_tools.remove_ec2_pool(ec2_pool) except Exception as e: log.exception(e) error_list += [ 'Error removing pool from bosco', str(e) ] try: email_tools.send_pool_auto_termination_email(ec2_pool) except: log.exception(e) ec2_tools.terminate_pool(ec2_pool) except Exception as e: log.exception('Error terminating pool') log.exception(e)
def terminate_idle_pools(): """Terminate any pool that doesn't have any more tasks running on it """ ec2_pools = EC2Pool.objects.filter(auto_terminate=True) for ec2_pool in ec2_pools: try: copied_pools = EC2Pool.objects.filter(copy_of=ec2_pool) all_tasks = Task.objects.filter(condor_pool=ec2_pool) | Task.objects.filter(condor_pool__in=copied_pools) running_tasks = all_tasks.filter(status='running') | all_tasks.filter(status='new') if running_tasks.count() == 0 and all_tasks.count() > 0: pool_name = ec2_pool.name log.debug('Terminating pool %s since no other jobs running (auto terminate)' % pool_name) #Prune the tasks so that they are disassociated from the pool for task in all_tasks: task.condor_pool = None task.set_custom_field('condor_pool_name', pool_name) task.save() if ec2_pool.get_pool_type() == 'ec2' and ec2_pool.copy_of == None: error_list = [] #Remove the copied pools first for copied_pool in copied_pools: try: copied_pool.delete() except Exception, e: log.exception(e) error_list += ['Error deleting duplicate pool', str(e)] #Remove from bosco try: condor_tools.remove_ec2_pool(ec2_pool) except Exception, e: log.exception(e) error_list += ['Error removing pool from bosco', str(e)] try: email_tools.send_pool_auto_termination_email(ec2_pool) except: log.exception(e) ec2_tools.terminate_pool(ec2_pool)
def dispatch(self, request, *args, **kwargs): key_id = self.kwargs['key_id'] key = AWSAccessKey.objects.get(id=key_id) kwargs['key'] = key assert key.user == request.user kwargs['show_loading_screen'] = True kwargs['loading_title'] = 'Removing key and associated VPC' kwargs['loading_description'] = 'Please be patient and do not navigate away from this page.' #Is this an original key or is it a copy if key.copy_of == None: original = True else: original = False if original: #Build a list of any pools and running jobs that will be terminated when this pool is terminated try: pools = EC2Pool.objects.filter(vpc__vpc_id = key.vpc.vpc_id) except: pools = [] shared_keys = AWSAccessKey.objects.filter(copy_of=key) shared_user_ids = [shared_key.user.id for shared_key in shared_keys] kwargs['shared_users'] = User.objects.filter(id__in=shared_user_ids) else: #A copy of a key. If so, we'll not be deleting the real vpc, adn so try: pools = EC2Pool.objects.filter(vpc__id=key.vpc.id) except: pools = [] kwargs['pools'] = pools errors=[] if kwargs['confirmed']: #Go through and terminate each of the running pools for pool in pools: #First, remove any running tasks running_tasks = pool.get_running_tasks() for task in running_tasks: for subtask in task.subtask_set.all(): condor_tools.remove_task(subtask) task.delete() other_tasks = Task.objects.filter(condor_pool=pool).exclude(pk__in=running_tasks) #Then 'prune' the remaining tasks to remove the pool as a foreignkey for task in other_tasks: task.condor_pool = None task.set_custom_field('condor_pool_name', pool.name) task.save() ec2_tools.terminate_pool(pool) if original: #We also need to delete the vpc (and any associated) try: related = AWSAccessKey.objects.filter(copy_of=key) for related_key in related: related_key.delete() if key.vpc != None: vpc_connection, ec2_connection = aws_tools.create_connections(key) errors += (vpc_tools.delete_vpc(key.vpc, vpc_connection, ec2_connection)) if errors != []: log.exception(errors) request.session['errors'] = aws_tools.process_errors(errors) except Exception, e: log.exception(e) #And delete the key key.delete() else: #Just delete the key object and the vpc key.delete() return HttpResponseRedirect(reverse_lazy('my_account_keys'))
def dispatch(self, request, *args, **kwargs): key_id = self.kwargs['key_id'] key = AWSAccessKey.objects.get(id=key_id) kwargs['key'] = key assert key.user == request.user kwargs['show_loading_screen'] = True kwargs['loading_title'] = 'Removing key and associated VPC' kwargs['loading_description'] = 'Please be patient and do not navigate away from this page.' #Is this an original key or is it a copy if key.copy_of == None: original = True else: original = False if original: #Build a list of any pools and running jobs that will be terminated when this pool is terminated try: pools = EC2Pool.objects.filter(vpc__vpc_id = key.vpc.vpc_id) except: pools = [] shared_keys = AWSAccessKey.objects.filter(copy_of=key) shared_user_ids = [shared_key.user.id for shared_key in shared_keys] kwargs['shared_users'] = User.objects.filter(id__in=shared_user_ids) else: #A copy of a key. If so, we'll not be deleting the real vpc, adn so try: pools = EC2Pool.objects.filter(vpc__id=key.vpc.id) except: pools = [] kwargs['pools'] = pools errors=[] if kwargs['confirmed']: #Go through and terminate each of the running pools for pool in pools: #First, remove any running tasks running_tasks = pool.get_running_tasks() for task in running_tasks: for subtask in task.subtask_set.all(): condor_tools.remove_task(subtask) task.delete() other_tasks = Task.objects.filter(condor_pool=pool).exclude(pk__in=running_tasks) #Then 'prune' the remaining tasks to remove the pool as a foreignkey for task in other_tasks: task.condor_pool = None task.set_custom_field('condor_pool_name', pool.name) task.save() ec2_tools.terminate_pool(pool) if original: #We also need to delete the vpc (and any associated) try: related = AWSAccessKey.objects.filter(copy_of=key) for related_key in related: related_key.delete() if key.vpc != None: vpc_connection, ec2_connection = aws_tools.create_connections(key) errors += (vpc_tools.delete_vpc(key.vpc, vpc_connection, ec2_connection)) if errors != []: log.exception(errors) request.session['errors'] = aws_tools.process_errors(errors) except Exception as e: log.exception(e) #And delete the key key.delete() else: #Just delete the key object and the vpc key.delete() return HttpResponseRedirect(reverse_lazy('my_account_keys')) return super(KeysDeleteView, self).dispatch(request, *args, **kwargs)
def dispatch(self, request, *args, **kwargs): pool_id = kwargs['pool_id'] pool = CondorPool.objects.get(id=pool_id) kwargs['pool'] = pool kwargs['pool_type']=pool.get_pool_type() if kwargs['pool_type'] == 'ec2': kwargs['node_count'] = EC2Pool.objects.get(id=pool_id).get_count() confirmed= kwargs['confirmed'] assert pool.user == request.user copied_pools = CondorPool.objects.filter(copy_of=pool) kwargs['copied_pools'] = copied_pools #Are there any other pools that are a copy of this one? pool_tasks = Task.objects.filter(condor_pool=pool) | Task.objects.filter(condor_pool__in=copied_pools) running_tasks = pool_tasks.filter(status='running')|pool_tasks.filter(status='new') other_tasks = pool_tasks.exclude(pk__in=running_tasks) if not confirmed: kwargs['show_loading_screen'] = True if pool.get_pool_type() == 'ec2' and pool.copy_of == None: kwargs['loading_title'] = 'Terminating pool' kwargs['loading_description'] = 'Please do not navigate away from this page. Terminating a pool can take several minutes.' kwargs['button_text']='Terminate pool' else: kwargs['loading_title'] = 'Removing pool' kwargs['loading_description'] = 'Please do not navigate away from this page. Removing a pool can take several minutes.' kwargs['button_text']='Remove pool' kwargs['running_tasks'] = running_tasks return super(PoolRemoveView, self).dispatch(request, *args, **kwargs) else: #Remove the pool #First, remove any running tasks for task in running_tasks: for subtask in task.subtask_set.all(): condor_tools.remove_task(subtask) task.status = 'cancelled' try: email_tools.send_task_cancellation_email(task) except: pass task.condor_pool = None task.set_custom_field('condor_pool_name', pool.name) task.save() #Then 'prune' the remaining tasks to remove the pool as a foreignkey for task in other_tasks: task.condor_pool = None task.set_custom_field('condor_pool_name', pool.name) task.save() if pool.get_pool_type() == 'ec2' and pool.copy_of == None: pool = EC2Pool.objects.get(id=pool.id) kwargs['pool'] = pool error_list = [] #Remove the copied pools first for copied_pool in copied_pools: try: copied_pool.delete() except Exception, e: log.exception(e) error_list += ['Error deleting duplicate pool', str(e)] #Remove from bosco try: condor_tools.remove_ec2_pool(pool) except Exception, e: log.exception(e) error_list += ['Error removing pool from bosco', str(e)] #Terminate the pool errors = ec2_tools.terminate_pool(pool) error_list += errors request.session['errors']=error_list
def dispatch(self, request, *args, **kwargs): pool_id = kwargs['pool_id'] pool = CondorPool.objects.get(id=pool_id) kwargs['pool'] = pool kwargs['pool_type']=pool.get_pool_type() if kwargs['pool_type'] == 'ec2': kwargs['node_count'] = EC2Pool.objects.get(id=pool_id).get_count() confirmed= kwargs['confirmed'] assert pool.user == request.user copied_pools = CondorPool.objects.filter(copy_of=pool) kwargs['copied_pools'] = copied_pools #Are there any other pools that are a copy of this one? pool_tasks = Task.objects.filter(condor_pool=pool) | Task.objects.filter(condor_pool__in=copied_pools) running_tasks = pool_tasks.filter(status='running')|pool_tasks.filter(status='new') other_tasks = pool_tasks.exclude(pk__in=running_tasks) if not confirmed: kwargs['show_loading_screen'] = True if pool.get_pool_type() == 'ec2' and pool.copy_of == None: kwargs['loading_title'] = 'Terminating pool' kwargs['loading_description'] = 'Please do not navigate away from this page. Terminating a pool can take several minutes.' kwargs['button_text']='Terminate pool' else: kwargs['loading_title'] = 'Removing pool' kwargs['loading_description'] = 'Please do not navigate away from this page. Removing a pool can take several minutes.' kwargs['button_text']='Remove pool' kwargs['running_tasks'] = running_tasks return super(PoolRemoveView, self).dispatch(request, *args, **kwargs) else: #Remove the pool #First, remove any running tasks for task in running_tasks: for subtask in task.subtask_set.all(): condor_tools.remove_task(subtask) task.status = 'cancelled' try: email_tools.send_task_cancellation_email(task) except: pass task.condor_pool = None task.set_custom_field('condor_pool_name', pool.name) task.save() #Then 'prune' the remaining tasks to remove the pool as a foreignkey for task in other_tasks: task.condor_pool = None task.set_custom_field('condor_pool_name', pool.name) task.save() if pool.get_pool_type() == 'ec2' and pool.copy_of == None: pool = EC2Pool.objects.get(id=pool.id) kwargs['pool'] = pool error_list = [] #Remove the copied pools first for copied_pool in copied_pools: try: copied_pool.delete() except Exception as e: log.exception(e) error_list += ['Error deleting duplicate pool', str(e)] #Remove from bosco try: condor_tools.remove_ec2_pool(pool) except Exception as e: log.exception(e) error_list += ['Error removing pool from bosco', str(e)] #Terminate the pool errors = ec2_tools.terminate_pool(pool) error_list += errors request.session['errors']=error_list elif pool.get_pool_type() == 'bosco' and pool.copy_of == None: try: #Remove any copied pools first for copied_pool in copied_pools: copied_pool.delete() #Only remove the pool from bosco if the same address is not registered with any other user if CondorPool.objects.filter(address=pool.address).count() == 1: condor_tools.remove_bosco_pool(pool.address) log.debug('Removing pool %s from bosco' % pool.address) else: log.debug('Not removing pool %s from bosco, since in use by another user' % pool.address) #However, still delete the db entry pool.delete() except Exception as e: log.exception(e) request.session['errors'] = [e] return HttpResponseRedirect(reverse_lazy('pool_details', pool_id = pool.id)) else: try: pool.delete() except Exception as e: log.exception(e) request.session['errors'] = [e] return HttpResponseRedirect(reverse_lazy('pool_details', pool_id = pool.id)) return HttpResponseRedirect(reverse_lazy('pool_list'))