class QueueController(ModelController): resource = resources.Queue version = (1, 0) mapping = 'id subject name status' model = Queue schema = SchemaDependency('platoon') def create(self, request, response, subject, data): session = self.schema.session subject = self.model.create(session, **data) session.commit() response({'id': subject.id}) def update(self, request, response, subject, data): if not data: return response({'id': subject.id}) session = self.schema.session subject.update(session, **data) session.commit() response({'id': subject.id}) def _annotate_resource(self, request, model, resource, data): endpoint = model.endpoint if endpoint: resource['endpoint'] = endpoint.extract_dict( exclude='id endpoint_id', drop_none=True)
class ScheduleController(ModelController): resource = resources.Schedule version = (1, 0) model = Schedule mapping = 'id name schedule anchor interval' schema = SchemaDependency('platoon')
class ProductController(ModelController): resource = ProductResource version = (1, 0) model = Product mapping = 'id title description' schema = SchemaDependency('lattice')
class ComponentController(ModelController): resource = ComponentResource version = (1, 0) model = Component mapping = 'id name version status description timestamp' schema = SchemaDependency('lattice') def _annotate_model(self, model, data): repository = data.get('repository') if repository: model.repository = ComponentRepository.polymorphic_create( repository) builds = data.get('builds') if builds: self.schema.session.query(Build).filter( Build.component_id == model.id).delete() for name, build in builds.iteritems(): build['name'] = name model.builds.append(Build.polymorphic_create(build)) def _annotate_resource(self, model, resource, data): repository = model.repository if repository: resource['repository'] = repository.extract_dict( exclude=['id', 'component_id']) builds = resource['builds'] = {} for build in model.builds: builds[build.name] = build.extract_dict( exclude=['id', 'component_id', 'name']) if data and 'include' in data and 'dependencies' in data['include']: resource['dependencies'] = [d.id for d in model.dependencies]
class IntentController(BaseEntityController): resource = resources.Intent version = (1, 0) model = Intent schema = SchemaDependency('docket') mapping = 'id name designation description created modified exclusive'
class ConfigurationController(ModelController): resource = resources.Configuration version = (1, 0) model = Configuration schema = SchemaDependency('harp') mapping = [('id', 'name'), 'filepath', 'pidfile', 'chroot', 'daemon', 'group', 'log_tag', 'user', 'default_mode', 'default_connect_timeout', 'default_client_timeout', 'default_server_timeout', 'include_globals', 'include_defaults'] def acquire(self, subject): try: query = self.schema.session.query(self.model) return query.filter(Configuration.name == subject).one() except NoResultFound: return None def update(self, request, response, subject, data): commit = data.pop('commit', False) super(ConfigurationController, self).update(request, response, subject, data) if commit: subject.commit()
class RuleController(ElementController): resource = resources.Rule version = (1, 0) model = Rule schema = SchemaDependency('harp') mapping = 'name rule content'
class ExecutorController(ModelController): resource = resources.Executor version = (1, 0) mapping = 'id name status' model = Executor schema = SchemaDependency('platoon') def create(self, request, response, subject, data): session = self.schema.session subject = self.model.create(session, **data) session.commit() response({'id': subject.id}) def update(self, request, response, subject, data): if not data: return response({'id': subject.id}) session = self.schema.session subject.update(session, **data) session.commit() response({'id': subject.id}) def _annotate_resource(self, request, model, resource, data): endpoints = model.endpoints if endpoints: resource['endpoints'] = {} for subject, endpoint in endpoints.iteritems(): resource['endpoints'][subject] = endpoint.extract_dict( exclude='id endpoint_id')
class ProfileController(ModelController): resource = ProfileResource version = (1, 0) model = Profile mapping = 'id product_id version' schema = SchemaDependency('lattice') def _annotate_model(self, model, data): components = data.get('components') if components: self.schema.session.query(ProfileComponent).filter( ProfileComponent.profile_id == model.id).delete() for component in components: component['component_id'] = component.pop('id') model.components.append(ProfileComponent(**component)) def _annotate_resource(self, model, resource, data): product = model.product resource['product'] = { 'title': product.title, 'description': product.description, } components = resource['components'] = [] for component in model.components: components.append({ 'id': component.component_id, }) if data and 'include' in data and 'sequence' in data['include']: print model.collate_components()
class ArchetypeController(BaseArchetypeController): resource = resources.Archetype version = (1, 0) model = Archetype registry = Dependency(ArchetypeRegistry) schema = SchemaDependency('docket') mapping = 'id name designation description created modified resource properties'
class RecurringTaskController(TaskController, ModelController): resource = resources.RecurringTask version = (1, 0) model = RecurringTask mapping = 'id tag description status schedule_id retry_backoff retry_limit retry_timeout created' idler = Dependency(Idler) schema = SchemaDependency('platoon')
class FrontendController(ProxyController): resource = resources.Frontend version = (1, 0) model = Frontend schema = SchemaDependency('harp') mapping = [ 'name', 'mode', 'connect_timeout', 'client_timeout', 'server_timeout', 'forwardfor', 'forwardfor_header', 'http_close', 'http_server_close', 'http_log', 'log_global', 'bind', 'default_backend' ]
class SubscribedTaskController(TaskController, ModelController): resource = resources.SubscribedTask version = (1, 0) model = SubscribedTask mapping = ( 'id tag description topic aspects activation_limit retry_backoff' ' retry_limit retry_timeout created activated timeout') idler = Dependency(Idler) schema = SchemaDependency('platoon')
class ServerController(ElementController): resource = resources.Server version = (1, 0) model = Server schema = SchemaDependency('harp') mapping = [ 'name', 'address', 'addr', 'backup', 'check', 'cookie', 'disabled', 'error_limit', 'fall', 'inter', 'fastinter', 'downinter', 'maxconn', 'maxqueue', 'minconn', 'observe', 'on_error', 'port', 'redir', 'rise', 'slowstart', 'track', 'weight' ]
class EntryController(ModelController): resource = resources.Entry version = (1, 0) model = Entry schema = SchemaDependency('narrative') def task(self, request, response, subject, data): session = self.schema.session task = data['task'] if task == 'purge-entries': Entry.purge(session)
class EventController(ModelController): resource = resources.Event version = (1, 0) model = Event mapping = 'id topic aspects' idler = Dependency(Idler) schema = SchemaDependency('platoon') def create(self, request, response, subject, data): subject = self.model.create(self.schema.session, **data) self.schema.session.commit() self.idler.interrupt() response({'id': subject.id})
class AssociationController(ModelController): resource = AssociationResource version = (1, 0) model = Association schema = SchemaDependency('docket') mapping = ('id', ('subject', 'subject_id'), 'intent', ('target', 'target_id')) def create(self, request, response, subject, data): session = self.schema.session subject = self.model.create(session, data['subject'], data['intent'], data['target']) session.commit() response({'id': self._get_id_value(subject)})
class RegistrationController(ModelController): resource = RegistrationResource version = (1, 0) model = Registration registry = Dependency(EntityRegistry) schema = SchemaDependency('docket') mapping = 'id name title url is_container specification canonical_version change_event' def acquire(self, subject): try: query = self.schema.session.query(self.model).options( undefer('specification')) return query.get(subject) except NoResultFound: return None def create(self, request, response, subject, data): session = self.schema.session subject = self.model.create(session, **data) session.commit() response({'id': subject.id}) def delete(self, request, response, subject, data): session = self.schema.session session.delete(subject) session.commit() response({'id': subject.id}) self.registry.unregister(subject) def update(self, request, response, subject, data): if not data: return response({'id': subject.id}) session = self.schema.session subject.update(session, **data) session.commit() response({'id': subject.id}) def _annotate_resource(self, request, model, resource, data): resource['cached_attributes'] = {} for name, attribute in model.cached_attributes.iteritems(): resource['cached_attributes'][name] = attribute.extract_dict( exclude='id registration_id name')
class ScheduledTaskController(TaskController, ModelController): resource = resources.ScheduledTask version = (1, 0) model = ScheduledTask mapping = 'id tag description status occurrence retry_backoff retry_limit retry_timeout created' idler = Dependency(Idler) schema = SchemaDependency('platoon') def _annotate_resource(self, request, model, resource, data): TaskController._annotate_resource(self, request, model, resource, data) if data and 'include' in data and 'executions' in data['include']: resource['executions'] = [ execution.extract_dict(exclude='id task_id') for execution in model.executions ]
class ProjectController(ModelController): resource = ProjectResource version = (1, 0) model = Project mapping = 'id status description' schema = SchemaDependency('lattice') def _annotate_model(self, model, data): repository = data.get('repository') if repository: model.repository = ProjectRepository.polymorphic_create(repository) def _annotate_resource(self, model, resource, data): repository = model.repository if repository: if repository.type == 'git': resource['repository'] = {'type': 'git', 'url': repository.url} elif repository.type == 'svn': resource['repository'] = {'type': 'svn', 'url': repository.url}
class EntityController(BaseEntityController): resource = EntityResource version = (1, 0) model = Entity registry = Dependency(EntityRegistry) schema = SchemaDependency('docket') mapping = 'id entity name designation description created modified' def task(self, request, response, subject, data): registry = self.registry session = self.schema.session task = data['task'] if task == 'synchronize-all-entities': self.model.synchronize_entities(registry, session) elif task == 'synchronize-entities': for identifier in data['ids']: try: subject = self.model.load(session, id=data['id'], lockmode='update') except NoResultFound: continue else: subject.synchronize(registry, session) session.commit() elif task == 'synchronize-changed-entity': event = data.get('event') if not event: return try: subject = self.model.load(session, id=event['id'], lockmode='update') except NoResultFound: return else: subject.synchronize(registry, session) session.commit()
class ExecutionController(ModelController): """A step execution controller""" model = WorkflowExecutionModel resource = Execution schema = SchemaDependency('flux') version = (1, 0) flux = MeshDependency('flux') platoon = MeshDependency('platoon') def update(self, request, response, subject, data): session = self.schema.session task = subject.update(session, **data) if task == 'abort': subject.initiate_abort(session) session.call_after_commit( ScheduledTask.queue_http_task, 'abort-run', self.flux.prepare('flux/1.0/execution', 'task', None, { 'task': 'abort-run', 'id': subject.id })) session.commit() response({'id': subject.id}) def task(self, request, response, subject, data): session = self.schema.session if 'id' in data: try: subject = self.model.load(session, id=data['id'], lockmode='update') except NoResultFound: return task = data['task'] if task == 'abort-run': subject.run.abort_executions(session) session.commit()
class Docket(Component): api = MeshServer.deploy(bundles=BUNDLES) schema = SchemaDependency('docket') archetype_registry = Dependency(ArchetypeRegistry) entity_registry = Dependency(EntityRegistry) docket = MeshDependency('docket') platoon = MeshDependency('platoon') @onstartup() def bootstrap(self): self.entity_registry.bootstrap() self.archetype_registry.bootstrap() self.api.server.configure_endpoints() self.schema.purge() @onstartup(service='docket') def startup_docket(self): EVERY_SIX_HOURS.put() SYNC_ALL_ENTITIES.set_http_task( self.docket.prepare('docket/1.0/entity', 'task', None, {'task': 'synchronize-all-entities'})) SYNC_ALL_ENTITIES.put() self.entity_registry.subscribe_to_changes() return {'status': 'yielding', 'stage': 'dependents-ready'} @onstartup(service='docket', stage='dependents-ready') def restart_when_dependents_ready(self): current_runtime().reload() return {'status': 'restarting', 'stage': 'docket-ready'} @onstartup(service='docket', stage='docket-ready') def finish_docket_startup(self): self.entity_registry.synchronize_entities() return {'status': 'ready'}
class ExecutionController(ModelController): """A step execution controller""" model = WorkflowExecutionModel resource = Execution schema = SchemaDependency('flux') version = (1, 0) flux = MeshDependency('flux') platoon = MeshDependency('platoon') def update(self, request, response, subject, data): session = self.schema.session status = data.pop('status') if status == 'aborted' and subject.is_active: subject.abort(session) session.commit() self.flux.execute('flux/1.0/run', 'update', subject.run_id, {'status': 'aborted'}) response({'id': subject.id})
class QueueManager(Unit): """The queue manager.""" flux = MeshDependency('flux') platoon = MeshDependency('platoon') schema = SchemaDependency('flux') def bootstrap(self): session = self.schema.session for operation in session.query(Operation): self._register_queue(operation) def initiate(self, operation, tag, input=None, id=None, timeout=None): params = {'queue_id': operation.queue_id, 'tag': tag} if id is not None: params['id'] = id if input is not None: params['input'] = input if timeout is not None: params['timeout'] = timeout Process.create(**params) def register(self, operation): self._register_queue(operation) def _register_queue(self, operation): endpoint = self.flux.prepare('flux/1.0/operation', 'process', operation.id, preparation={'type': 'http'}) Queue(id=operation.queue_id, subject=operation.id, name=operation.name, endpoint=endpoint).put()
class ProcessController(ModelController): resource = resources.Process version = (1, 0) mapping = 'id queue_id tag timeout status input output progress state started ended' model = Process schema = SchemaDependency('platoon') def create(self, request, response, subject, data): session = self.schema.session subject = self.model.create(session, **data) session.commit() response({'id': subject.id}) def update(self, request, response, subject, data): if not data: return response({'id': subject.id}) session = self.schema.session subject.update(session, **data) session.commit() response({'id': subject.id})
class RunController(ModelController): resource = RunResource version = (1, 0) model = Run mapping = 'id workflow_id name status parameters started ended' schema = SchemaDependency('flux') flux = MeshDependency('flux') platoon = MeshDependency('platoon') @support_returning def create(self, request, response, subject, data): session = self.schema.session subject = self.model.create(session, **data) session.commit() ScheduledTask.queue_http_task( 'initiate-run', self.flux.prepare('flux/1.0/run', 'task', None, { 'task': 'initiate-run', 'id': subject.id })) notify = data.get('notify') if notify: SubscribedTask.queue_http_task('run-completion', self.flux.prepare( 'flux/1.0/run', 'task', None, { 'task': 'run-completion', 'id': subject.id, 'notify': notify }), topic='run:completed', aspects={'id': subject.id}) return subject @support_returning def update(self, request, response, subject, data): session = self.schema.session status = data.pop('status') if status == 'aborted' and subject.is_active: subject.initiate_abort(session) session.commit() ScheduledTask.queue_http_task( 'abort-run', self.flux.prepare('flux/1.0/run', 'task', None, { 'task': 'abort-executions', 'id': subject.id })) return subject def task(self, request, response, subject, data): session = self.schema.session if 'id' in data: try: subject = self.model.load(session, id=data['id'], lockmode='update') except NoResultFound: return task = data['task'] if task == 'initiate-run': subject.initiate(session) session.commit() elif task == 'abort-executions': subject.abort_executions(session) session.commit() elif task == 'run-completion': self._send_completion_email(subject, data) def _annotate_resource(self, request, model, resource, data): if not data: return include = data.get('include') if include and 'executions' in include: attrs = ( 'id', 'execution_id', 'ancestor_id', 'step', 'name', 'status', 'started', 'ended', ) executions = [ e.extract_dict(attrs=attrs) for e in model.executions.all() ] resource['executions'] = executions def _send_completion_email(self, subject, data): recipients = [{'to': data['notify'].split(',')}] email_subject = 'Workflow run "%s" completed' % subject.name body = 'The workflow run "%s" completed and is available for review.' % subject.name Message.create(recipients=recipients, subject=email_subject, body=body)
class BaseInstanceController(BaseEntityController): schema = SchemaDependency('docket')
class ServiceRegistry(Unit): """The service registry.""" configuration = Configuration({ 'required_services': Sequence(Token(segments=1, nonempty=True), unique=True), }) schema = SchemaDependency('nucleus') threads = Dependency(ThreadPool) def bootstrap(self): session = self.schema.session query = session.query(Service) self.schema.lock_tables(session, 'service') try: for id in self.configuration.get('required_services', []): service = query.get(id) if not service: Service.create(session, id=id) for service in query: service.reset() session.commit() finally: session.close() self.schema.purge() current_runtime().register_mule('service-registry', self.manage) def manage(self, mule): session = self.schema.session try: self._manage_services(session) finally: session.close() def _manage_services(self, session): attempt = 1 while True: log('info', 'attempting to verify registrations (attempt %d)', attempt) attempt += 1 try: registered = self._verify_registrations(session) finally: session.rollback() if registered: log('info', 'all services registered') break timeout = next_timeout(attempt) if timeout is not None: time.sleep(timeout) else: raise Exception('missing services') attempt = 1 while True: log('info', 'attempting to start up all services (attempt %d)', attempt) attempt += 1 try: ready = self._start_services(session) finally: session.rollback() if ready: log('info', 'all services ready') break timeout = next_timeout(attempt) if timeout is not None: time.sleep(timeout) else: raise Exception('service startup failed') def _enumerate_services(self, session): graph = {} query = session.query(Service) for service in list(query): graph[service] = set() if service.dependencies: for id in service.dependencies: dependency = query.get(id) if dependency: graph[service].add(dependency) else: raise InvalidDependencyError(id) return topological_sort(graph) def _start_services(self, session): yielding = [] for service in self._enumerate_services(session): if service.status == 'ready' or not service.registered: continue status = service.status if status == 'unknown': service.instruct({'status': 'starting'}) elif status in ('starting', 'restarting'): service.instruct({ 'status': 'starting', 'stage': service.stage }) elif status == 'yielding': yielding.append(service) for service in yielding: if service.dependents_ready(session): service.instruct({ 'status': 'starting', 'stage': service.stage }) session.commit() ready = True statuses = [] for service in session.query(Service).order_by('id'): statuses.append('%s=%s' % (service.id, service.status)) if service.status != 'ready': ready = False log('info', 'service status: %s' % ', '.join(statuses)) return ready def _verify_registrations(self, session): registered = True registrations = [] for service in session.query(Service).order_by('id'): registrations.append('%s=%r' % (service.id, service.registered)) if not service.registered: registered = False log('info', 'registration status: %s' % ', '.join(registrations)) return registered
class TestDependency(Unit): schema = SchemaDependency('flux')