class QueueController(ModelController):
resource = resources.Queue
version = (1, 0)
mapping = 'id subject name status'
model = Queue
schema = SchemaDependency('platoon')
def create(self, request, response, subject, data):
session = self.schema.session
subject = self.model.create(session, **data)
session.commit()
response({'id': subject.id})
def update(self, request, response, subject, data):
if not data:
return response({'id': subject.id})
session = self.schema.session
subject.update(session, **data)
session.commit()
response({'id': subject.id})
def _annotate_resource(self, request, model, resource, data):
endpoint = model.endpoint
if endpoint:
resource['endpoint'] = endpoint.extract_dict(
exclude='id endpoint_id', drop_none=True)
class ScheduleController(ModelController):
resource = resources.Schedule
version = (1, 0)
model = Schedule
mapping = 'id name schedule anchor interval'
schema = SchemaDependency('platoon')
class ProductController(ModelController):
resource = ProductResource
version = (1, 0)
model = Product
mapping = 'id title description'
schema = SchemaDependency('lattice')
class ComponentController(ModelController):
resource = ComponentResource
version = (1, 0)
model = Component
mapping = 'id name version status description timestamp'
schema = SchemaDependency('lattice')
def _annotate_model(self, model, data):
repository = data.get('repository')
if repository:
model.repository = ComponentRepository.polymorphic_create(
repository)
builds = data.get('builds')
if builds:
self.schema.session.query(Build).filter(
Build.component_id == model.id).delete()
for name, build in builds.iteritems():
build['name'] = name
model.builds.append(Build.polymorphic_create(build))
def _annotate_resource(self, model, resource, data):
repository = model.repository
if repository:
resource['repository'] = repository.extract_dict(
exclude=['id', 'component_id'])
builds = resource['builds'] = {}
for build in model.builds:
builds[build.name] = build.extract_dict(
exclude=['id', 'component_id', 'name'])
if data and 'include' in data and 'dependencies' in data['include']:
resource['dependencies'] = [d.id for d in model.dependencies]
class IntentController(BaseEntityController):
resource = resources.Intent
version = (1, 0)
model = Intent
schema = SchemaDependency('docket')
mapping = 'id name designation description created modified exclusive'
class ConfigurationController(ModelController):
resource = resources.Configuration
version = (1, 0)
model = Configuration
schema = SchemaDependency('harp')
mapping = [('id', 'name'), 'filepath', 'pidfile', 'chroot', 'daemon',
'group', 'log_tag', 'user', 'default_mode',
'default_connect_timeout', 'default_client_timeout',
'default_server_timeout', 'include_globals', 'include_defaults']
def acquire(self, subject):
try:
query = self.schema.session.query(self.model)
return query.filter(Configuration.name == subject).one()
except NoResultFound:
return None
def update(self, request, response, subject, data):
commit = data.pop('commit', False)
super(ConfigurationController, self).update(request, response, subject,
data)
if commit:
subject.commit()
class RuleController(ElementController):
resource = resources.Rule
version = (1, 0)
model = Rule
schema = SchemaDependency('harp')
mapping = 'name rule content'
class ExecutorController(ModelController):
resource = resources.Executor
version = (1, 0)
mapping = 'id name status'
model = Executor
schema = SchemaDependency('platoon')
def create(self, request, response, subject, data):
session = self.schema.session
subject = self.model.create(session, **data)
session.commit()
response({'id': subject.id})
def update(self, request, response, subject, data):
if not data:
return response({'id': subject.id})
session = self.schema.session
subject.update(session, **data)
session.commit()
response({'id': subject.id})
def _annotate_resource(self, request, model, resource, data):
endpoints = model.endpoints
if endpoints:
resource['endpoints'] = {}
for subject, endpoint in endpoints.iteritems():
resource['endpoints'][subject] = endpoint.extract_dict(
exclude='id endpoint_id')
class ProfileController(ModelController):
resource = ProfileResource
version = (1, 0)
model = Profile
mapping = 'id product_id version'
schema = SchemaDependency('lattice')
def _annotate_model(self, model, data):
components = data.get('components')
if components:
self.schema.session.query(ProfileComponent).filter(
ProfileComponent.profile_id == model.id).delete()
for component in components:
component['component_id'] = component.pop('id')
model.components.append(ProfileComponent(**component))
def _annotate_resource(self, model, resource, data):
product = model.product
resource['product'] = {
'title': product.title,
'description': product.description,
}
components = resource['components'] = []
for component in model.components:
components.append({
'id': component.component_id,
})
if data and 'include' in data and 'sequence' in data['include']:
print model.collate_components()
class ArchetypeController(BaseArchetypeController):
resource = resources.Archetype
version = (1, 0)
model = Archetype
registry = Dependency(ArchetypeRegistry)
schema = SchemaDependency('docket')
mapping = 'id name designation description created modified resource properties'
class RecurringTaskController(TaskController, ModelController):
resource = resources.RecurringTask
version = (1, 0)
model = RecurringTask
mapping = 'id tag description status schedule_id retry_backoff retry_limit retry_timeout created'
idler = Dependency(Idler)
schema = SchemaDependency('platoon')
class FrontendController(ProxyController):
resource = resources.Frontend
version = (1, 0)
model = Frontend
schema = SchemaDependency('harp')
mapping = [
'name', 'mode', 'connect_timeout', 'client_timeout', 'server_timeout',
'forwardfor', 'forwardfor_header', 'http_close', 'http_server_close',
'http_log', 'log_global', 'bind', 'default_backend'
]
class SubscribedTaskController(TaskController, ModelController):
resource = resources.SubscribedTask
version = (1, 0)
model = SubscribedTask
mapping = (
'id tag description topic aspects activation_limit retry_backoff'
' retry_limit retry_timeout created activated timeout')
idler = Dependency(Idler)
schema = SchemaDependency('platoon')
class ServerController(ElementController):
resource = resources.Server
version = (1, 0)
model = Server
schema = SchemaDependency('harp')
mapping = [
'name', 'address', 'addr', 'backup', 'check', 'cookie', 'disabled',
'error_limit', 'fall', 'inter', 'fastinter', 'downinter', 'maxconn',
'maxqueue', 'minconn', 'observe', 'on_error', 'port', 'redir', 'rise',
'slowstart', 'track', 'weight'
]
class EntryController(ModelController):
resource = resources.Entry
version = (1, 0)
model = Entry
schema = SchemaDependency('narrative')
def task(self, request, response, subject, data):
session = self.schema.session
task = data['task']
if task == 'purge-entries':
Entry.purge(session)
class EventController(ModelController):
resource = resources.Event
version = (1, 0)
model = Event
mapping = 'id topic aspects'
idler = Dependency(Idler)
schema = SchemaDependency('platoon')
def create(self, request, response, subject, data):
subject = self.model.create(self.schema.session, **data)
self.schema.session.commit()
self.idler.interrupt()
response({'id': subject.id})
class AssociationController(ModelController):
resource = AssociationResource
version = (1, 0)
model = Association
schema = SchemaDependency('docket')
mapping = ('id', ('subject', 'subject_id'), 'intent', ('target',
'target_id'))
def create(self, request, response, subject, data):
session = self.schema.session
subject = self.model.create(session, data['subject'], data['intent'],
data['target'])
session.commit()
response({'id': self._get_id_value(subject)})
class RegistrationController(ModelController):
resource = RegistrationResource
version = (1, 0)
model = Registration
registry = Dependency(EntityRegistry)
schema = SchemaDependency('docket')
mapping = 'id name title url is_container specification canonical_version change_event'
def acquire(self, subject):
try:
query = self.schema.session.query(self.model).options(
undefer('specification'))
return query.get(subject)
except NoResultFound:
return None
def create(self, request, response, subject, data):
session = self.schema.session
subject = self.model.create(session, **data)
session.commit()
response({'id': subject.id})
def delete(self, request, response, subject, data):
session = self.schema.session
session.delete(subject)
session.commit()
response({'id': subject.id})
self.registry.unregister(subject)
def update(self, request, response, subject, data):
if not data:
return response({'id': subject.id})
session = self.schema.session
subject.update(session, **data)
session.commit()
response({'id': subject.id})
def _annotate_resource(self, request, model, resource, data):
resource['cached_attributes'] = {}
for name, attribute in model.cached_attributes.iteritems():
resource['cached_attributes'][name] = attribute.extract_dict(
exclude='id registration_id name')
class ScheduledTaskController(TaskController, ModelController):
resource = resources.ScheduledTask
version = (1, 0)
model = ScheduledTask
mapping = 'id tag description status occurrence retry_backoff retry_limit retry_timeout created'
idler = Dependency(Idler)
schema = SchemaDependency('platoon')
def _annotate_resource(self, request, model, resource, data):
TaskController._annotate_resource(self, request, model, resource, data)
if data and 'include' in data and 'executions' in data['include']:
resource['executions'] = [
execution.extract_dict(exclude='id task_id')
for execution in model.executions
]
class ProjectController(ModelController):
resource = ProjectResource
version = (1, 0)
model = Project
mapping = 'id status description'
schema = SchemaDependency('lattice')
def _annotate_model(self, model, data):
repository = data.get('repository')
if repository:
model.repository = ProjectRepository.polymorphic_create(repository)
def _annotate_resource(self, model, resource, data):
repository = model.repository
if repository:
if repository.type == 'git':
resource['repository'] = {'type': 'git', 'url': repository.url}
elif repository.type == 'svn':
resource['repository'] = {'type': 'svn', 'url': repository.url}
class EntityController(BaseEntityController):
resource = EntityResource
version = (1, 0)
model = Entity
registry = Dependency(EntityRegistry)
schema = SchemaDependency('docket')
mapping = 'id entity name designation description created modified'
def task(self, request, response, subject, data):
registry = self.registry
session = self.schema.session
task = data['task']
if task == 'synchronize-all-entities':
self.model.synchronize_entities(registry, session)
elif task == 'synchronize-entities':
for identifier in data['ids']:
try:
subject = self.model.load(session,
id=data['id'],
lockmode='update')
except NoResultFound:
continue
else:
subject.synchronize(registry, session)
session.commit()
elif task == 'synchronize-changed-entity':
event = data.get('event')
if not event:
return
try:
subject = self.model.load(session,
id=event['id'],
lockmode='update')
except NoResultFound:
return
else:
subject.synchronize(registry, session)
session.commit()
class ExecutionController(ModelController):
"""A step execution controller"""
model = WorkflowExecutionModel
resource = Execution
schema = SchemaDependency('flux')
version = (1, 0)
flux = MeshDependency('flux')
platoon = MeshDependency('platoon')
def update(self, request, response, subject, data):
session = self.schema.session
task = subject.update(session, **data)
if task == 'abort':
subject.initiate_abort(session)
session.call_after_commit(
ScheduledTask.queue_http_task, 'abort-run',
self.flux.prepare('flux/1.0/execution', 'task', None, {
'task': 'abort-run',
'id': subject.id
}))
session.commit()
response({'id': subject.id})
def task(self, request, response, subject, data):
session = self.schema.session
if 'id' in data:
try:
subject = self.model.load(session,
id=data['id'],
lockmode='update')
except NoResultFound:
return
task = data['task']
if task == 'abort-run':
subject.run.abort_executions(session)
session.commit()
class Docket(Component):
api = MeshServer.deploy(bundles=BUNDLES)
schema = SchemaDependency('docket')
archetype_registry = Dependency(ArchetypeRegistry)
entity_registry = Dependency(EntityRegistry)
docket = MeshDependency('docket')
platoon = MeshDependency('platoon')
@onstartup()
def bootstrap(self):
self.entity_registry.bootstrap()
self.archetype_registry.bootstrap()
self.api.server.configure_endpoints()
self.schema.purge()
@onstartup(service='docket')
def startup_docket(self):
EVERY_SIX_HOURS.put()
SYNC_ALL_ENTITIES.set_http_task(
self.docket.prepare('docket/1.0/entity', 'task', None,
{'task': 'synchronize-all-entities'}))
SYNC_ALL_ENTITIES.put()
self.entity_registry.subscribe_to_changes()
return {'status': 'yielding', 'stage': 'dependents-ready'}
@onstartup(service='docket', stage='dependents-ready')
def restart_when_dependents_ready(self):
current_runtime().reload()
return {'status': 'restarting', 'stage': 'docket-ready'}
@onstartup(service='docket', stage='docket-ready')
def finish_docket_startup(self):
self.entity_registry.synchronize_entities()
return {'status': 'ready'}
class ExecutionController(ModelController):
"""A step execution controller"""
model = WorkflowExecutionModel
resource = Execution
schema = SchemaDependency('flux')
version = (1, 0)
flux = MeshDependency('flux')
platoon = MeshDependency('platoon')
def update(self, request, response, subject, data):
session = self.schema.session
status = data.pop('status')
if status == 'aborted' and subject.is_active:
subject.abort(session)
session.commit()
self.flux.execute('flux/1.0/run', 'update', subject.run_id,
{'status': 'aborted'})
response({'id': subject.id})
class QueueManager(Unit):
"""The queue manager."""
flux = MeshDependency('flux')
platoon = MeshDependency('platoon')
schema = SchemaDependency('flux')
def bootstrap(self):
session = self.schema.session
for operation in session.query(Operation):
self._register_queue(operation)
def initiate(self, operation, tag, input=None, id=None, timeout=None):
params = {'queue_id': operation.queue_id, 'tag': tag}
if id is not None:
params['id'] = id
if input is not None:
params['input'] = input
if timeout is not None:
params['timeout'] = timeout
Process.create(**params)
def register(self, operation):
self._register_queue(operation)
def _register_queue(self, operation):
endpoint = self.flux.prepare('flux/1.0/operation',
'process',
operation.id,
preparation={'type': 'http'})
Queue(id=operation.queue_id,
subject=operation.id,
name=operation.name,
endpoint=endpoint).put()
class ProcessController(ModelController):
resource = resources.Process
version = (1, 0)
mapping = 'id queue_id tag timeout status input output progress state started ended'
model = Process
schema = SchemaDependency('platoon')
def create(self, request, response, subject, data):
session = self.schema.session
subject = self.model.create(session, **data)
session.commit()
response({'id': subject.id})
def update(self, request, response, subject, data):
if not data:
return response({'id': subject.id})
session = self.schema.session
subject.update(session, **data)
session.commit()
response({'id': subject.id})
class RunController(ModelController):
resource = RunResource
version = (1, 0)
model = Run
mapping = 'id workflow_id name status parameters started ended'
schema = SchemaDependency('flux')
flux = MeshDependency('flux')
platoon = MeshDependency('platoon')
@support_returning
def create(self, request, response, subject, data):
session = self.schema.session
subject = self.model.create(session, **data)
session.commit()
ScheduledTask.queue_http_task(
'initiate-run',
self.flux.prepare('flux/1.0/run', 'task', None, {
'task': 'initiate-run',
'id': subject.id
}))
notify = data.get('notify')
if notify:
SubscribedTask.queue_http_task('run-completion',
self.flux.prepare(
'flux/1.0/run', 'task', None, {
'task': 'run-completion',
'id': subject.id,
'notify': notify
}),
topic='run:completed',
aspects={'id': subject.id})
return subject
@support_returning
def update(self, request, response, subject, data):
session = self.schema.session
status = data.pop('status')
if status == 'aborted' and subject.is_active:
subject.initiate_abort(session)
session.commit()
ScheduledTask.queue_http_task(
'abort-run',
self.flux.prepare('flux/1.0/run', 'task', None, {
'task': 'abort-executions',
'id': subject.id
}))
return subject
def task(self, request, response, subject, data):
session = self.schema.session
if 'id' in data:
try:
subject = self.model.load(session,
id=data['id'],
lockmode='update')
except NoResultFound:
return
task = data['task']
if task == 'initiate-run':
subject.initiate(session)
session.commit()
elif task == 'abort-executions':
subject.abort_executions(session)
session.commit()
elif task == 'run-completion':
self._send_completion_email(subject, data)
def _annotate_resource(self, request, model, resource, data):
if not data:
return
include = data.get('include')
if include and 'executions' in include:
attrs = (
'id',
'execution_id',
'ancestor_id',
'step',
'name',
'status',
'started',
'ended',
)
executions = [
e.extract_dict(attrs=attrs) for e in model.executions.all()
]
resource['executions'] = executions
def _send_completion_email(self, subject, data):
recipients = [{'to': data['notify'].split(',')}]
email_subject = 'Workflow run "%s" completed' % subject.name
body = 'The workflow run "%s" completed and is available for review.' % subject.name
Message.create(recipients=recipients, subject=email_subject, body=body)
class BaseInstanceController(BaseEntityController):
schema = SchemaDependency('docket')
class ServiceRegistry(Unit):
"""The service registry."""
configuration = Configuration({
'required_services':
Sequence(Token(segments=1, nonempty=True), unique=True),
})
schema = SchemaDependency('nucleus')
threads = Dependency(ThreadPool)
def bootstrap(self):
session = self.schema.session
query = session.query(Service)
self.schema.lock_tables(session, 'service')
try:
for id in self.configuration.get('required_services', []):
service = query.get(id)
if not service:
Service.create(session, id=id)
for service in query:
service.reset()
session.commit()
finally:
session.close()
self.schema.purge()
current_runtime().register_mule('service-registry', self.manage)
def manage(self, mule):
session = self.schema.session
try:
self._manage_services(session)
finally:
session.close()
def _manage_services(self, session):
attempt = 1
while True:
log('info', 'attempting to verify registrations (attempt %d)',
attempt)
attempt += 1
try:
registered = self._verify_registrations(session)
finally:
session.rollback()
if registered:
log('info', 'all services registered')
break
timeout = next_timeout(attempt)
if timeout is not None:
time.sleep(timeout)
else:
raise Exception('missing services')
attempt = 1
while True:
log('info', 'attempting to start up all services (attempt %d)',
attempt)
attempt += 1
try:
ready = self._start_services(session)
finally:
session.rollback()
if ready:
log('info', 'all services ready')
break
timeout = next_timeout(attempt)
if timeout is not None:
time.sleep(timeout)
else:
raise Exception('service startup failed')
def _enumerate_services(self, session):
graph = {}
query = session.query(Service)
for service in list(query):
graph[service] = set()
if service.dependencies:
for id in service.dependencies:
dependency = query.get(id)
if dependency:
graph[service].add(dependency)
else:
raise InvalidDependencyError(id)
return topological_sort(graph)
def _start_services(self, session):
yielding = []
for service in self._enumerate_services(session):
if service.status == 'ready' or not service.registered:
continue
status = service.status
if status == 'unknown':
service.instruct({'status': 'starting'})
elif status in ('starting', 'restarting'):
service.instruct({
'status': 'starting',
'stage': service.stage
})
elif status == 'yielding':
yielding.append(service)
for service in yielding:
if service.dependents_ready(session):
service.instruct({
'status': 'starting',
'stage': service.stage
})
session.commit()
ready = True
statuses = []
for service in session.query(Service).order_by('id'):
statuses.append('%s=%s' % (service.id, service.status))
if service.status != 'ready':
ready = False
log('info', 'service status: %s' % ', '.join(statuses))
return ready
def _verify_registrations(self, session):
registered = True
registrations = []
for service in session.query(Service).order_by('id'):
registrations.append('%s=%r' % (service.id, service.registered))
if not service.registered:
registered = False
log('info', 'registration status: %s' % ', '.join(registrations))
return registered
class TestDependency(Unit):
schema = SchemaDependency('flux')