from simpleflow.types import failure import eventlet eventlet.monkey_patch() failure.TRACEBACK = True dst = { 'host': '172.20.0.3', 'port': 3304, 'schema': 'simpleflow', 'user': '******', 'passwd': '111111' } from simpleservice.ormdb.argformater import connformater sql_connection = connformater % dst session = build_session(sql_connection) connection = Connection(session) class MysqlDumper(task.Task): def execute(self, server_id): if server_id % 2 != 0: raise Exception('server id %d error' % server_id) print 'success', server_id def revert(self, *args, **kwargs): print 'revert', args, kwargs servers = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
def merge_entitys(appendpoint, uuid, entity, databases): datadb = databases[common.DATADB] mergepath = 'merge-%s' % uuid mergeroot = os.path.join(appendpoint.endpoint_backup, mergepath) stepsfile = os.path.join(mergeroot, 'steps.dat') initfile = os.path.join(mergeroot, 'init.sql') if not os.path.exists(stepsfile): raise exceptions.MergeException('Steps file not exist') with open(stepsfile, 'rb') as f: data = cPickle.load(f) steps = data['steps'] prepares = [] for _entity, step in six.iteritems(steps): # 一些post sql执行错误对整体无影响情况下 # 可以直接讲step改为FINISHED避免重复合服步骤 if step == FINISHED: for _step in six.itervalues(steps): if _step != FINISHED: raise exceptions.MergeException('Steps is finish?') appendpoint.client.finish_merge(uuid) appendpoint.flush_config(entity, databases, opentime=data['opentime'], chiefs=data['chiefs']) return if step != INSERT: prepares.append(_entity) mini_entity = min(prepares) if prepares: name = 'prepare-merge-at-%d' % int(time.time()) book = LogBook(name=name) store = dict(timeout=5, dtimeout=600, mergeroot=mergeroot, entity=entity) taskflow_session = build_session( 'sqlite:///%s' % os.path.join(mergeroot, '%s.db' % name)) connection = Connection(taskflow_session) prepare_uflow = uf.Flow(name) for _entity in prepares: entity_flow = lf.Flow('prepare-%d' % _entity) entity_flow.add(Swallow(uuid, steps, _entity, appendpoint)) entity_flow.add( DumpData(uuid, steps, _entity, appendpoint, _entity != mini_entity)) entity_flow.add(Swallowed(uuid, steps, _entity, appendpoint)) prepare_uflow.add(entity_flow) engine = load(connection, prepare_uflow, store=store, book=book, engine_cls=ParallelActionEngine, max_workers=4) try: engine.run() except Exception as e: if LOG.isEnabledFor(logging.DEBUG): LOG.exception('Prepare merge task execute fail') raise exceptions.MergeException( 'Prepare merge task execute fail, %s %s' % (e.__class__.__name__, str(e))) finally: connection.session = None taskflow_session.close() with open(stepsfile, 'wb') as f: cPickle.dump(data, f) for _entity, step in six.iteritems(steps): if step != INSERT: raise exceptions.MergeException('Some step not on %s' % INSERT) if not os.path.exists(os.path.join(mergeroot, sqlfile(_entity))): raise exceptions.MergeException('Entity %d sql file not exist' % _entity) if not os.path.exists(initfile): LOG.error('Init database file not exist') raise exceptions.MergeException('Init database file not exist') LOG.info('Prepare merge success, try merge database') now = int(time.time()) name = 'merge-at-%d' % now book = LogBook(name=name) store = dict(timeout=1800, root=mergeroot, database=datadb, timeline=now) taskflow_session = build_session('sqlite:///%s' % os.path.join(mergeroot, '%s.db' % name)) connection = Connection(taskflow_session) merge_flow = lf.Flow('merge-to') merge_flow.add(SafeCleanDb()) merge_flow.add(InitDb()) insert_lflow = lf.Flow('insert-db') stoper = [0] for _entity in steps: insert_lflow.add(InserDb(_entity, stoper)) merge_flow.add(insert_lflow) merge_flow.add(PostDo(uuid, appendpoint)) engine = load(connection, merge_flow, store=store, book=book, engine_cls=ParallelActionEngine, max_workers=4) try: engine.run() except Exception as e: if LOG.isEnabledFor(logging.DEBUG): LOG.exception('Merge database task execute fail') raise exceptions.MergeException( 'Merge database task execute fail, %s %s' % (e.__class__.__name__, str(e))) else: for _entity in steps: steps[_entity] = FINISHED with open(stepsfile, 'wb') as f: cPickle.dump(data, f) appendpoint.client.finish_merge(uuid) appendpoint.flush_config(entity, databases, opentime=data['opentime'], chiefs=data['chiefs']) LOG.info('Merge task %s all finish' % uuid) finally: connection.session = None taskflow_session.close()
import time from simpleflow.utils.storage_utils import build_session from simpleflow.storage.impl import Connection dst = { 'host': '172.20.0.3', 'port': 3304, 'schema': 'simpleflow', 'user': '******', 'passwd': '111111' } session = build_session(dst) conn = Connection(session) s = time.time() for book in conn.get_logbooks(): print book print time.time() - s conn.clear_all() print time.time() - s