def execute_event(store, log, event, deleted_policy_ids): """ Execute a single event :param store: `IScalingGroupCollection` provider :param log: A bound log for logging :param event: event dict to execute :param deleted_policy_ids: Set of policy ids that are deleted. Policy id will be added to this if its scaling group or policy has been deleted :return: a deferred with None. Any error occurred during execution is logged """ tenant_id, group_id, policy_id = event['tenantId'], event['groupId'], event['policyId'] log = log.bind(tenant_id=tenant_id, scaling_group_id=group_id, policy_id=policy_id) log.msg('Scheduler executing policy {policy_id}') group = store.get_scaling_group(log, tenant_id, group_id) d = group.modify_state(partial(maybe_execute_scaling_policy, log, generate_transaction_id(), policy_id=policy_id, version=event['version'])) d.addErrback(ignore_and_log, CannotExecutePolicyError, log, 'Scheduler cannot execute policy {policy_id}') def collect_deleted_policy(failure): failure.trap(NoSuchScalingGroupError, NoSuchPolicyError) deleted_policy_ids.add(policy_id) d.addErrback(collect_deleted_policy) d.addErrback(log.err, 'Scheduler failed to execute policy {policy_id}') return d
def execute_event(self, log, event, deleted_policy_ids): """ Execute a single event :param log: A bound log for logging :param event: event dict to execute :param deleted_policy_ids: Set of policy ids that are deleted. Policy id will be added to this if its scaling group or policy has been deleted :return: a deferred with the results of execution """ tenant_id, group_id, policy_id = event['tenantId'], event['groupId'], event['policyId'] log = log.bind(tenant_id=tenant_id, scaling_group_id=group_id, policy_id=policy_id) log.msg('Executing policy') group = get_store().get_scaling_group(log, tenant_id, group_id) d = group.modify_state(partial(maybe_execute_scaling_policy, log, generate_transaction_id(), policy_id=policy_id)) d.addErrback(ignore_and_log, CannotExecutePolicyError, log, 'Cannot execute policy') def collect_deleted_policy(failure): failure.trap(NoSuchScalingGroupError, NoSuchPolicyError) deleted_policy_ids.add(policy_id) d.addErrback(collect_deleted_policy) d.addErrback(log.err, 'Scheduler failed to execute policy') return d
def check_events(self, batchsize): """ Check for events occurring now and earlier """ if self.kz_partition.allocating: self.log.msg('Partition allocating') return if self.kz_partition.release: self.log.msg('Partition changed. Repartitioning') return self.kz_partition.release_set() if self.kz_partition.failed: self.log.msg('Partition failed. Starting new') self.kz_partition = self.kz_client.SetPartitioner( self.zk_partition_path, set=set(self.buckets), time_boundary=self.time_boundary) return if not self.kz_partition.acquired: self.log.err('Unknown state {}. This cannot happen. Starting new'.format( self.kz_partition.state)) self.kz_partition.finish() self.kz_partition = self.kz_client.SetPartitioner( self.zk_partition_path, set=set(self.buckets), time_boundary=self.time_boundary) return buckets = list(self.kz_partition) utcnow = datetime.utcnow() log = self.log.bind(scheduler_run_id=generate_transaction_id(), utcnow=utcnow) # TODO: This log might feel like spam since it'll occur on every tick. But # it'll be useful to debug partitioning problems (at least in initial deployment) log.msg('Got buckets {buckets}', buckets=buckets, path=self.zk_partition_path) return defer.gatherResults( [check_events_in_bucket( log, self.store, bucket, utcnow, batchsize) for bucket in buckets])
def fetch_and_process(self, batchsize): """ Fetch the events to be processed and process them. Also delete/update after processing them :return: a deferred that fires with list of events processed """ def process_events(events): if not len(events): return events, set() log.msg('Processing events', num_events=len(events)) deleted_policy_ids = set() def eb(failure, policy_id): failure.trap(NoSuchPolicyError, NoSuchScalingGroupError) deleted_policy_ids.add(policy_id) deferreds = [ self.execute_event(log, event).addErrback(eb, event['policyId']).addErrback(log.err) for event in events ] d = defer.gatherResults(deferreds, consumeErrors=True) return d.addCallback(lambda _: (events, deleted_policy_ids)) def update_delete_events((events, deleted_policy_ids)): """ Update events with cron entry with next trigger time Delete other events """ if not len(events): return events events_to_delete, events_to_update = [], [] for event in events: if event['cron'] and event['policyId'] not in deleted_policy_ids: event['trigger'] = next_cron_occurrence(event['cron']) events_to_update.append(event) else: events_to_delete.append(event['policyId']) log.msg('Deleting events', num_policy_ids_deleting=len(events_to_delete)) log.msg('Updating events', num_policy_ids_updating=len(events_to_update)) d = get_store().update_delete_events(events_to_delete, events_to_update) return d.addCallback(lambda _: events) # utcnow because of cass serialization issues utcnow = datetime.utcnow() log = self.log.bind(scheduler_run_id=generate_transaction_id(), utcnow=utcnow) log.msg('Checking for events') deferred = get_store().fetch_batch_of_events(utcnow, batchsize) deferred.addCallback(process_events) deferred.addCallback(update_delete_events) deferred.addErrback(log.err) return deferred
def _check_events(self, batchsize, buckets): """ Check for events occurring now and earlier """ utcnow = datetime.utcnow() log = self.log.bind(scheduler_run_id=generate_transaction_id(), utcnow=utcnow) return defer.gatherResults( [check_events_in_bucket(log, self.dispatcher, self.store, bucket, utcnow, batchsize) for bucket in buckets] )
def _check_events(self, batchsize, buckets): """ Check for events occurring now and earlier """ utcnow = datetime.utcnow() log = self.log.bind(scheduler_run_id=generate_transaction_id(), utcnow=utcnow) return defer.gatherResults([ check_events_in_bucket(log, self.dispatcher, self.store, bucket, utcnow, batchsize) for bucket in buckets ])
def _(self, request, *args, **kwargs): transaction_id = generate_transaction_id() request.setHeader('X-Response_Id', transaction_id) bound_log = log.bind(system=reflect.fullyQualifiedName(f), transaction_id=transaction_id) bound_log.bind(method=request.method, uri=request.uri, clientproto=request.clientproto, referer=request.getHeader('referer'), useragent=request.getHeader('user-agent')).msg( 'Recieved request') return f(self, request, bound_log, *args, **kwargs)
def _(request, *args, **kwargs): transaction_id = generate_transaction_id() request.setHeader('X-Response-Id', transaction_id) bound_log = log.bind(system=reflect.fullyQualifiedName(f), transaction_id=transaction_id) bound_log.bind(method=request.method, uri=request.uri, clientproto=request.clientproto, referer=request.getHeader("referer"), useragent=request.getHeader("user-agent")).msg( "Received request") return bind_log(f)(request, bound_log, *args, **kwargs)
def _(self, request, *args, **kwargs): transaction_id = generate_transaction_id() request.setHeader('X-Response-Id', transaction_id) self.log = self.log.bind(system=reflect.fullyQualifiedName(f), transaction_id=transaction_id) self.log.msg("Received request", method=request.method, uri=request.uri, clientproto=request.clientproto, referer=request.getHeader("referer"), useragent=request.getHeader("user-agent"), request_status="received") return f(self, request, *args, **kwargs)
def _(self, request, *args, **kwargs): transaction_id = generate_transaction_id() request.setHeader('X-Response-Id', transaction_id) bound_log = log.bind( system=reflect.fullyQualifiedName(f), transaction_id=transaction_id) bound_log.bind( method=request.method, uri=request.uri, clientproto=request.clientproto, referer=request.getHeader("referer"), useragent=request.getHeader("user-agent") ).msg("Received request") return bind_log(f)(self, request, bound_log, *args, **kwargs)
def execute_event(self, log, event): """ Execute a single event :param log: A bound log for logging :return: a deferred with the results of execution """ tenant_id, group_id, policy_id = event['tenantId'], event['groupId'], event['policyId'] log = log.bind(tenant_id=tenant_id, scaling_group_id=group_id, policy_id=policy_id) log.msg('Executing policy') group = get_store().get_scaling_group(log, tenant_id, group_id) d = group.modify_state(partial(maybe_execute_scaling_policy, log, generate_transaction_id(), policy_id=policy_id)) d.addErrback(ignore_and_log, CannotExecutePolicyError, log, 'Cannot execute policy') return d
def execute_event(dispatcher, store, log, event, deleted_policy_ids): """ Execute a single event :param dispatcher: Effect dispatcher :param store: `IScalingGroupCollection` provider :param log: A bound log for logging :param event: event dict to execute :param deleted_policy_ids: Set of policy ids that are deleted. Policy id will be added to this if its scaling group or policy has been deleted :return: a deferred with None. Any error occurred during execution is logged """ tenant_id = event['tenantId'] group_id = event['groupId'] policy_id = event['policyId'] log = log.bind(tenant_id=tenant_id, scaling_group_id=group_id, policy_id=policy_id, scheduled_time=event["trigger"].isoformat() + "Z") log.msg('sch-exec-pol', cloud_feed=True) group = store.get_scaling_group(log, tenant_id, group_id) d = modify_and_trigger(dispatcher, group, bound_log_kwargs(log), partial(maybe_execute_scaling_policy, log, generate_transaction_id(), policy_id=policy_id, version=event['version']), modify_state_reason='scheduler.execute_event') d.addErrback(ignore_and_log, CannotExecutePolicyError, log, "sch-cannot-exec", cloud_feed=True) def collect_deleted_policy(failure): failure.trap(NoSuchScalingGroupError, NoSuchPolicyError) deleted_policy_ids.add(policy_id) d.addErrback(collect_deleted_policy) d.addErrback(log.err, "sch-exec-pol-err", cloud_feed=True) return d
def execute_event(dispatcher, store, log, event, deleted_policy_ids): """ Execute a single event :param dispatcher: Effect dispatcher :param store: `IScalingGroupCollection` provider :param log: A bound log for logging :param event: event dict to execute :param deleted_policy_ids: Set of policy ids that are deleted. Policy id will be added to this if its scaling group or policy has been deleted :return: a deferred with None. Any error occurred during execution is logged """ tenant_id = event["tenantId"] group_id = event["groupId"] policy_id = event["policyId"] log = log.bind( tenant_id=tenant_id, scaling_group_id=group_id, policy_id=policy_id, scheduled_time=event["trigger"].isoformat() + "Z", ) log.msg("sch-exec-pol", cloud_feed=True) group = store.get_scaling_group(log, tenant_id, group_id) d = modify_and_trigger( dispatcher, group, bound_log_kwargs(log), partial( maybe_execute_scaling_policy, log, generate_transaction_id(), policy_id=policy_id, version=event["version"] ), modify_state_reason="scheduler.execute_event", ) d.addErrback(ignore_and_log, CannotExecutePolicyError, log, "sch-cannot-exec", cloud_feed=True) def collect_deleted_policy(failure): failure.trap(NoSuchScalingGroupError, NoSuchPolicyError) deleted_policy_ids.add(policy_id) d.addErrback(collect_deleted_policy) d.addErrback(log.err, "sch-exec-pol-err", cloud_feed=True) return d