def flow_factory(job):
"""
@param job: class: sqlalchemy:session
"""
retryer = None
if job.retry:
retryer = Times(attempts=job.retry, revert_all=job.revertall)
main_flow = lf.Flow('scheduler_taskflow', retry=retryer)
for index, step in enumerate(job.steps):
_executor = getattr(executor, step.executor)
_analyzer = getattr(analyzer, step.executor)
task_executor = _executor or _analyzer
task = task_executor.builder('%d-%d' % (job.job_id, index), step)
main_flow.add(task)
return main_flow
def mysql_flow_factory(app,
store,
create_cls=MysqlCreate,
backup_cls=MysqlDump,
update_cls=MysqlUpdate):
if not app.databases:
return None
middleware = app.middleware
endpoint_name = middleware.endpoint
entity = middleware.entity
uflow = uf.Flow('db_cbu_%s_%d' % (endpoint_name, entity))
for index, database in enumerate(app.databases):
retry = None
if database.retry:
retry = Times(attempts=database.retry,
name='db_retry_%s_%d_%d' %
(endpoint_name, entity, index))
lfow = lf.Flow(name='db_%s_%d_%d' % (endpoint_name, entity, index),
retry=retry)
if database.create:
lfow.add(create_cls(middleware, database))
if database.backup:
lfow.add(backup_cls(middleware, database))
if database.update:
if database.update.rollback and not database.backup:
raise ValueError('Database rollback need backup')
lfow.add(update_cls(middleware, database))
if len(lfow):
uflow.add(lfow)
else:
del lfow
if len(uflow):
return uflow
del uflow
return None
def merge_entitys(appendpoint, uuid, entity, databases):
datadb = databases[common.DATADB]
mergepath = 'merge-%s' % uuid
mergeroot = os.path.join(appendpoint.endpoint_backup, mergepath)
stepsfile = os.path.join(mergeroot, 'steps.dat')
initfile = os.path.join(mergeroot, 'init.sql')
if not os.path.exists(stepsfile):
raise exceptions.MergeException('Steps file not exist')
with open(stepsfile, 'rb') as f:
data = cPickle.load(f)
steps = data['steps']
prepares = []
for _entity, step in six.iteritems(steps):
# 一些post sql执行错误对整体无影响情况下
# 可以直接讲step改为FINISHED避免重复合服步骤
if step == FINISHED:
for _step in six.itervalues(steps):
if _step != FINISHED:
raise exceptions.MergeException('Steps is finish?')
appendpoint.client.finish_merge(uuid)
appendpoint.flush_config(entity,
databases,
opentime=data['opentime'],
chiefs=data['chiefs'])
return
if step != INSERT:
prepares.append(_entity)
mini_entity = min(prepares)
if prepares:
name = 'prepare-merge-at-%d' % int(time.time())
book = LogBook(name=name)
store = dict(timeout=5,
dtimeout=600,
mergeroot=mergeroot,
entity=entity)
taskflow_session = build_session(
'sqlite:///%s' % os.path.join(mergeroot, '%s.db' % name))
connection = Connection(taskflow_session)
prepare_uflow = uf.Flow(name)
for _entity in prepares:
entity_flow = lf.Flow('prepare-%d' % _entity)
entity_flow.add(Swallow(uuid, steps, _entity, appendpoint))
entity_flow.add(
DumpData(uuid, steps, _entity, appendpoint,
_entity != mini_entity))
entity_flow.add(Swallowed(uuid, steps, _entity, appendpoint))
prepare_uflow.add(entity_flow)
engine = load(connection,
prepare_uflow,
store=store,
book=book,
engine_cls=ParallelActionEngine,
max_workers=4)
try:
engine.run()
except Exception as e:
if LOG.isEnabledFor(logging.DEBUG):
LOG.exception('Prepare merge task execute fail')
raise exceptions.MergeException(
'Prepare merge task execute fail, %s %s' %
(e.__class__.__name__, str(e)))
finally:
connection.session = None
taskflow_session.close()
with open(stepsfile, 'wb') as f:
cPickle.dump(data, f)
for _entity, step in six.iteritems(steps):
if step != INSERT:
raise exceptions.MergeException('Some step not on %s' % INSERT)
if not os.path.exists(os.path.join(mergeroot, sqlfile(_entity))):
raise exceptions.MergeException('Entity %d sql file not exist' %
_entity)
if not os.path.exists(initfile):
LOG.error('Init database file not exist')
raise exceptions.MergeException('Init database file not exist')
LOG.info('Prepare merge success, try merge database')
now = int(time.time())
name = 'merge-at-%d' % now
book = LogBook(name=name)
store = dict(timeout=1800, root=mergeroot, database=datadb, timeline=now)
taskflow_session = build_session('sqlite:///%s' %
os.path.join(mergeroot, '%s.db' % name))
connection = Connection(taskflow_session)
merge_flow = lf.Flow('merge-to')
merge_flow.add(SafeCleanDb())
merge_flow.add(InitDb())
insert_lflow = lf.Flow('insert-db')
stoper = [0]
for _entity in steps:
insert_lflow.add(InserDb(_entity, stoper))
merge_flow.add(insert_lflow)
merge_flow.add(PostDo(uuid, appendpoint))
engine = load(connection,
merge_flow,
store=store,
book=book,
engine_cls=ParallelActionEngine,
max_workers=4)
try:
engine.run()
except Exception as e:
if LOG.isEnabledFor(logging.DEBUG):
LOG.exception('Merge database task execute fail')
raise exceptions.MergeException(
'Merge database task execute fail, %s %s' %
(e.__class__.__name__, str(e)))
else:
for _entity in steps:
steps[_entity] = FINISHED
with open(stepsfile, 'wb') as f:
cPickle.dump(data, f)
appendpoint.client.finish_merge(uuid)
appendpoint.flush_config(entity,
databases,
opentime=data['opentime'],
chiefs=data['chiefs'])
LOG.info('Merge task %s all finish' % uuid)
finally:
connection.session = None
taskflow_session.close()
def test_flow_factory():
return lf.Flow('example').add(TestTask(name='first'),
UnfortunateTask(name='boom'),
TestTask(name='second'))
class CallJoe(task.Task):
def execute(self, joe_number, *args, **kwargs):
print("Calling joe %s." % joe_number)
def revert(self, joe_number, *args, **kwargs):
print("Calling joe %s and apologizing." % joe_number)
class CallJim(task.Task):
def execute(self, jim_number):
print("Calling jim %s." % jim_number)
if jim_number != 5551:
raise Exception("Wrong number! of jim")
else:
print("Hello Jim!")
def revert(self, jim_number, **kwargs):
print("Wrong number jim, apologizing.")
retryer = retry.ParameterizedForEach(rebind=['phone_directory'], provides='jim_number')
lflow = lf.Flow('retrying-linear', retry=retryer)
lflow.add(CallJoe())
lflow.add(CallJim())
# Create your flow and associated tasks (the work to be done).
# Now run that flow using the provided initial data (store below).
result = api.run(session, lflow, store={'phone_directory': [333, 444, 555, 666], 'joe_number': 111})
print result
}
from simpleservice.ormdb.argformater import connformater
sql_connection = connformater % dst
session = build_session(sql_connection)
class Adder(task.Task):
def execute(self, x, y):
print 'do!!!', x, y
return x + y
flow = gf.Flow('root').add(
lf.Flow('nested_linear').add(
# x2 = y3+y4 = 12
Adder("add2", provides='x2', rebind=['y3', 'y4']),
# x1 = y1+y2 = 4
Adder("add1", provides='x1', rebind=['y1', 'y2'])),
# x5 = x1+x3 = 20
Adder("add5", provides='x5', rebind=['x1', 'x3']),
# x3 = x1+x2 = 16
Adder("add3", provides='x3', rebind=['x1', 'x2']),
# x4 = x2+y5 = 21
Adder("add4", provides='x4', rebind=['x2', 'y5']),
# x6 = x5+x4 = 41
Adder("add6", provides='x6', rebind=['x5', 'x4']),
# x7 = x6+x6 = 82
Adder("add7", provides='x7', rebind=['x6', 'x6']))
# Provide the initial variable inputs using a storage dictionary.
store = {
global revert_all
if revert_all:
return retry.REVERT_ALL
return retry.REVERT
return retry.RETRY
def execute(self, history, *args, **kwargs):
print 'RetryFlow exec', args, kwargs, 'history:', len(history)
return 2
def revert(self, history, *args, **kwargs):
return 'RetryFlow revert', args, kwargs, 'history:', len(history)
retryer = RetryFlow()
lflow = lf.Flow('xxx')
lflow.add(A('a', rebind=['va']))
lflow.add(A('b', rebind=['vb']))
# retry应该是写在这里,如果在顶层flow
# 那么RETRY和REVERT_ALL是没有区别的
gflow = gf.Flow('ogm', retry=retryer)
gflow.add(A('c', rebind=['vc'])) # 因为使用的是graph_flow, c和d是没有顺序的
gflow.add(A('d', rebind=['vd']))
lflow.add(gflow)
# 设置是否回滚全部
revert_all = True
api.run(session,
lflow,
def execute(self, joe_number, *args, **kwargs):
print("Calling joe %s." % joe_number)
def revert(self, joe_number, *args, **kwargs):
print("Calling %s and apologizing." % joe_number)
class CallSuzzie(task.Task):
def execute(self, suzzie_number, *args, **kwargs):
raise Exception("Suzzie not home right now.")
# Create your flow and associated tasks (the work to be done).
flow = lf.Flow('simple-linear').add(
CallJim(),
CallJoe(),
CallSuzzie()
)
stone = dict(joe_number=444,
jim_number=555,
suzzie_number=666)
try:
# Now run that flow using the provided initial data (store below).
api.run(session, flow, store=stone)
except Exception as e:
# NOTE(harlowja): This exception will be the exception that came out of the
# 'CallSuzzie' task instead of a different exception, this is useful since
# typically surrounding code wants to handle the original exception and not
def flow_factory(session, book,
applications,
upgradefile=None,
backupfile=None,
store=None,
db_flow_factory=database.mysql_flow_factory,
**kwargs):
"""
@param session: class: sqlalchemy:session
@param middlewares: class:list EntityMiddleware
@param upgradefile: class:AppUpgradeFile app upgrade file
@param backupfile: class:basestring of path/AppRemoteBackupFile app backup file
@param store: class:dict
@param db_flow_factory: class:function 默认database.mysql_flow_factory
@param create_cls: class:class 数据库创建任务类 参考database.MysqlCreate
@param backup_cls: class:class 数据库备份任务类 参考database.MysqlDump
@param update_cls: class:class 数据库更新任务类 参考database.MysqlUpdate
"""
if not applications:
raise RuntimeError('No application found')
if upgradefile and not isinstance(upgradefile, TaskPublicFile):
raise TypeError('upgradefile not TaskPublicFile')
if backupfile and not isinstance(backupfile, TaskPublicFile):
raise TypeError('backupfile not TaskPublicFile')
store = store or {}
if store.get('backupfile') or store.get('upgradefile'):
raise RuntimeError('Backupfile or Upgradefile in store')
# choice one entity by randomizing
# 随机选择一个app
app = applications[random.randint(0, len(applications)-1)]
# 获取endpoint的名称
endpoint_name = app.middleware.endpoint
main_flow = lf.Flow('%s_taskflow' % endpoint_name)
# prepare file for app update and database
# 准备工作
prepare_uflow = uf.Flow('%s_prepare' % endpoint_name)
# 下载程序更新文件
if upgradefile:
rebind = ['download_timeout']
format_store_rebind(store, rebind)
# get app update file, all middlewares use same app upload file
prepare_uflow.add(application.AppUpgradeFileGet(app.middleware, upgradefile, rebind=rebind))
# 备份程序文件
if backupfile:
rebind = ['download_timeout']
format_store_rebind(store, rebind)
prepare_uflow.add(application.AppBackUp(app.middleware, backupfile, rebind=rebind))
# 下载数据库更新文件
if app.databases and not all([False if d.update else True for d in app.databases]):
rebind = ['download_timeout']
format_store_rebind(store, rebind)
# get database upload file, all middlewares use same database upload file
prepare_uflow.add(database.DbUpdateSqlGet(app.middleware, app.databases, rebind=rebind))
if len(prepare_uflow):
main_flow.add(prepare_uflow)
else:
del prepare_uflow
entitys_taskflow = uf.Flow('%s_entitys_task' % endpoint_name)
# 批量更新操作
for app in applications:
# all entity task
entitys_taskflow.add(entity_factory(session, book, app, store,
upgradefile, backupfile,
db_flow_factory, **kwargs))
eventlet.sleep(0)
main_flow.add(entitys_taskflow)
return main_flow
def entity_factory(session, book, app, store,
upgradefile, backupfile,
db_flow_factory, **kwargs):
"""
@param session: class: sqlalchemy:session
@param book: class: taskflow book
@param store: class: dict
@param db_flow_factory: class: function
@param upgradefile: class: TaskPublicFile
@param backupfile: class: TaskPublicFile
@param kwargs: class: create_cls,backup_cls,update_cls
"""
endpoint_name = app.middleware.endpoint
entity = app.middleware.entity
entity_flow = lf.Flow('entity_%s_%d' % (endpoint_name, entity))
# 为创建更新和备份提供文件
entity_flow.add(ProvidesTask(name='provides_%s_%d' % (endpoint_name, entity),
upgradefile=upgradefile,
backupfile=backupfile))
# 创建任务,串行
if app.createtask:
if not upgradefile:
raise ValueError('No file found for createtask')
entity_flow.add(app.createtask)
# 停止任务,串行
if app.stoptask:
# kill if stop fail
prepare_flow = uf.Flow('recheck_stop_%s_%d' % (endpoint_name, entity),
retry=application.AppKill('kill_%s_%d' % (endpoint_name, entity)))
# sure entity stoped
prepare_flow.add(app.stoptask)
entity_flow.add(prepare_flow)
# 创建一个并行工作流
upflow = uf.Flow('up_%s_%d' % (endpoint_name, entity))
# 更新任务(与其他任务并行)
if app.upgradetask:
if not upgradefile:
raise ValueError('No file found for upgradetask')
# upgrade app file
upflow.add(app.upgradetask)
# 数据库备份与升级任务, 与更新任务并行
database_flow = db_flow_factory(app, store, **kwargs)
if database_flow:
upflow.add(database_flow)
# 并行工作流插入到主工作流中
if len(upflow):
entity_flow.add(upflow)
else:
del upflow
# 其他串行任务(更新完成后)
if app.updatetask:
entity_flow.add(app.updatetask)
# start appserver
if app.startstak:
entity_flow.add(app.startstak)
# delete appserver
if app.deletetask:
entity_flow.add(app.deletetask)
# entity task is independent event
return EntityTask(session, book, entity_flow, store)
class MysqlDumper(task.Task):
def execute(self, server_id):
print 'execute server_id', server_id
return server_id + 1
def revert(self, *args, **kwargs):
print 'revert', args, kwargs
atask = MysqlDumper('s1', provides='s2', rebind=['s1'])
btask = MysqlDumper('s2', provides='s3', rebind=['s2'])
ctask = MysqlDumper('s3', provides='s4', rebind=['s3'])
dtask = MysqlDumper('s4', provides='s5', rebind=['s4'])
etask = MysqlDumper('s5', provides='s6', rebind=['s5'])
lflow = lf.Flow('lftest')
lflow.add(atask)
lflow.add(btask)
lflow.add(ctask)
lflow.add(dtask)
lflow.add(etask)
data = {'s1': 1}
result = api.run(session, lflow, store=data, engine_cls=ParallelActionEngine)
# result = api.run(session, lflow, store=data)
print 'all success',
print result
