def get_client( src: typing.Union[Node, RunContext, NetworkIdentifier] ) -> pyclx.CasperLabsClient: """Factory method to return configured clabs client. :param src: The source form which a network node will be derived. :returns: A configured clabs client ready for use. """ # Pull node information from cache. if isinstance(src, Node): node = src elif isinstance(src, NetworkIdentifier): node = cache.get_node_by_network_id(src) elif isinstance(src, RunContext): node = cache.get_run_node(src) if not node: raise ValueError( "Network nodeset is empty, therefore cannot dispatch a deploy.") logger.log( f"PYCLX :: connecting to node :: {node.network} :: {node.host}:{node.port}" ) # TODO: get node id / client ssl cert. return pyclx.CasperLabsClient( host=node.host, port=node.port, )
def main(args): """Entry point. :param args: Parsed CLI arguments. """ # Unpack. host = args.address.split(':')[0] index = int(args.node.split(':')[-1]) network = args.node.split(':')[0] port = int(args.address.split(':')[-1]) typeof = NodeType[args.typeof.upper()] # Instantiate. node = factory.create_node(host=host, index=index, network_id=factory.create_network_id(network), port=port, typeof=typeof) # Push. cache.set_network_node(node) # Notify. logger.log(f"Node {args.node} was successfully registered")
def main(args): """Entry point. :param args: Parsed CLI arguments. """ # Unpack. network_id = factory.create_network_id(args.node.split(':')[0]) node_id = factory.create_node_id(network_id, int(args.node.split(':')[-1])) # Pull. node = cache.get_node(node_id) if node is None: raise ValueError("Unregistered node.") # Set key pair. pvk, pbk = crypto.get_key_pair_from_pvk_pem_file(args.pem_path, crypto.KeyEncoding.HEX) # Set bonding account. node.account = factory.create_account(index=-node_id.index, private_key=pvk, public_key=pbk, status=AccountStatus.ACTIVE, typeof=AccountType.BOND) # Push. cache.set_network_node(node) # Inform. logger.log(f"Node {args.node} bonding key was successfully registered")
def main(args): """Entry point. :param args: Parsed CLI arguments. """ # Pull data. network_id = factory.create_network_id(args.network) data = cache.orchestration.get_info_list(network_id, args.run_type) data = [i for i in data if i.aspect == ExecutionAspect.RUN] if not data: logger.log("No run information found.") return # Set cols/rows. cols = ["Network", "Type", "ID", "Start Time", "Duration (s)", "Status"] rows = map( lambda i: [ network_id.name, i.run_type, i.index_label.strip(), i.ts_start, i.tp_elapsed_label, i. status_label ], sorted(data, key=lambda i: f"{i.run_type}.{i.index_label}")) # Set table. t = get_table(cols, rows) t.column_alignments['Start Time'] = BeautifulTable.ALIGN_LEFT t.column_alignments['Duration (s)'] = BeautifulTable.ALIGN_RIGHT # Render. print(t) print(f"total runs = {len(data)}")
def on_run_end(ctx: ExecutionContext): """Ends a workflow. :param ctx: Execution context information. """ # Update ctx. ctx.status = ExecutionStatus.COMPLETE # Set info/state. run_state = factory.create_state(ExecutionAspect.RUN, ctx) # Update cache. cache.orchestration.set_context(ctx) cache.orchestration.set_state(run_state) cache.orchestration.update_run_info(ctx) # Locks can now be flushed. cache.orchestration.flush_locks(ctx) # Inform. logger.log(f"WFLOW :: {ctx.run_type} :: {ctx.run_index_label} -> ends") # Loop. if ctx.loop_count != 0: do_run_loop(ctx)
def main(args): """Entry point. :param args: Parsed CLI arguments. """ # Pull data. network_id = factory.create_network_id(args.network) data = cache.state.get_deploys(network_id, args.run_type, args.run_index) if not data: logger.log("No run deploys found.") return # Set table cols/rows. cols = [i for i, _ in COLS] rows = map(lambda i: [ i.deploy_hash, i.typeof.name, i.status.name, i.dispatch_node, i.dispatch_ts, i.label_finalization_time, i.block_hash or "--" ], sorted(data, key=lambda i: i.dispatch_ts)) # Set table. t = get_table(cols, rows, max_width=1080) # Set table alignments. for key, aligmnent in COLS: t.column_alignments[key] = aligmnent # Render. print(t) print(f"{network_id.name} - {args.run_type} - Run {args.run_index}")
def main(args): """Entry point. :param args: Parsed CLI arguments. """ # Pull. network = cache.get_network_by_name(args.network) if network is None: raise ValueError("Unregistered network.") # Set key pair. pvk, pbk = crypto.get_key_pair_from_pvk_pem_file(args.pem_path, crypto.KeyEncoding.HEX) # Set faucet. network.faucet = factory.create_account(index=0, private_key=pvk, public_key=pbk, typeof=AccountType.FAUCET, status=AccountStatus.ACTIVE) # Push. cache.set_network(network) # Inform. logger.log( f"Network {args.network} faucet key was successfully registered")
def do_step_end(ctx: ExecutionContext): """Ends a workflow step. :param ctx: Execution context information. """ # Set step. step = Workflow.get_phase_step(ctx, ctx.phase_index, ctx.step_index) # Set info/state. step_state = factory.create_state(ExecutionAspect.STEP, ctx, ExecutionStatus.COMPLETE) # Update cache. cache.orchestration.set_state(step_state) cache.orchestration.update_step_info(ctx, ExecutionStatus.COMPLETE) # Inform. logger.log( f"WFLOW :: {ctx.run_type} :: {ctx.run_index_label} :: {ctx.phase_index_label} :: {ctx.step_index_label} :: {step.label} -> end" ) # Enqueue either end of phase or next step. if step.is_last: on_phase_end.send(ctx) else: do_step.send(ctx)
def do_run(ctx: ExecutionContext): """Runs a workflow. :param ctx: Execution context information. """ # Escape if unexecutable. if not predicates.can_start_run(ctx): return # Update ctx. ctx.status = ExecutionStatus.IN_PROGRESS # Set info/state. run_info = factory.create_info(ExecutionAspect.RUN, ctx) run_state = factory.create_state(ExecutionAspect.RUN, ctx) # Update cache. cache.flush_by_run(ctx) cache.orchestration.set_context(ctx) cache.orchestration.set_info(run_info) cache.orchestration.set_state(run_state) # Inform. logger.log(f"WFLOW :: {ctx.run_type} :: {ctx.run_index_label} -> starts") # Run phase. do_phase.send(ctx)
def do_step(ctx: ExecutionContext): """Runs a workflow step. :param ctx: Execution context information. """ # Escape if unexecutable. if not predicates.can_start_step(ctx): return # Set step. step = Workflow.get_phase_step(ctx, ctx.phase_index, ctx.step_index + 1) # Update ctx. ctx.step_index += 1 ctx.step_label = step.label # Set info/state. step_info = factory.create_info(ExecutionAspect.STEP, ctx) step_state = factory.create_state(ExecutionAspect.STEP, ctx, ExecutionStatus.IN_PROGRESS) # Update cache. cache.orchestration.set_context(ctx) cache.orchestration.set_info(step_info) cache.orchestration.set_state(step_state) # Inform. logger.log( f"WFLOW :: {ctx.run_type} :: {ctx.run_index_label} :: {ctx.phase_index_label} :: {ctx.step_index_label} :: {step.label} -> starts" ) # Execute. do_step_execute(ctx, step)
def on_phase_end(ctx: ExecutionContext): """Ends a workflow phase. :param ctx: Execution context information. """ # Set phase. phase = Workflow.get_phase_(ctx, ctx.phase_index) # Set info/state. phase_state = factory.create_state(ExecutionAspect.PHASE, ctx, status=ExecutionStatus.COMPLETE) # Update cache. cache.orchestration.set_state(phase_state) cache.orchestration.update_phase_info(ctx, ExecutionStatus.COMPLETE) # Inform. logger.log( f"WFLOW :: {ctx.run_type} :: {ctx.run_index_label} :: {ctx.phase_index_label} -> ends" ) # Enqueue either end of workflow or next phase. if phase.is_last: on_run_end.send(ctx) else: do_phase.send(ctx)
def main(args: argparse.Namespace): """Entry point. """ # Import initialiser to setup upstream services / actors. import stests.initialiser # Unpack args. network_id = factory.create_network_id(args.network_name) node_id = factory.create_node_id(network_id, args.node_index) # Set execution context. ctx = factory.create_run_info(args=Arguments.create(args), loop_count=args.loop_count, loop_interval=args.loop_interval, network_id=network_id, node_id=node_id, run_index=args.run_index, run_type=constants.TYPE, use_stored_contracts=True) # Abort if a run lock cannot be acquired. if is_run_locked(ctx): logger.log_warning( f"{constants.TYPE} :: run {args.run_index} aborted as it is currently executing." ) # Start run. else: from stests.orchestration.actors import do_run do_run.send(ctx) logger.log(f"{constants.TYPE} :: run {args.run_index} started")
def main(args): """Entry point. :param args: Parsed CLI arguments. """ # Pull data. network_id = factory.create_network_id(args.network) data = cache.orchestration.get_info_list(network_id, args.run_type, args.run_index) if not data: logger.log("No run information found.") return # Set cols/rows. cols = ["Phase / Step", "Start Time", "Duration (s)", "Action", "Status"] rows = map(lambda i: [ i.index_label, i.ts_start, i.tp_elapsed_label, i.step_label if i.step_label else '--' , i.status.name, ], sorted(data, key=lambda i: i.index_label)) # Set table. t = get_table(cols, rows) t.column_alignments['Phase / Step'] = BeautifulTable.ALIGN_LEFT t.column_alignments['Start Time'] = BeautifulTable.ALIGN_LEFT t.column_alignments['Duration (s)'] = BeautifulTable.ALIGN_RIGHT t.column_alignments['Action'] = BeautifulTable.ALIGN_RIGHT t.column_alignments['Status'] = BeautifulTable.ALIGN_RIGHT # Render. print(t) print(f"{network_id.name} - {args.run_type} - Run {args.run_index}")
def main(args: argparse.Namespace): """Entry point. """ # Set run context. network_id = factory.create_network_id(args.network) node_id = factory.create_node_id(network_id, args.node) ctx = factory.create_run_context(args=Arguments.create(args), network_id=network_id, node_id=node_id, run=args.run, run_type=constants.TYPE) # Flush previous cache data. cache.flush_run(ctx) # Initialise MQ broker. mq.initialise() # Import actors. import stests.generators.correlator import stests.generators.wg_100.phase_1 import stests.generators.wg_100.phase_2 import stests.generators.wg_100.step_incrementor # Start workflow. logger.log("... workload generator begins") # Execute first actor in pipeline. from stests.generators.wg_100.step_incrementor import PIPELINE PIPELINE[0].send(ctx)
def do_phase(ctx: ExecutionContext): """Runs a workflow phase. :param ctx: Execution context information. """ # Escape if unexecutable. if not predicates.can_start_phase(ctx): return # Update ctx. ctx.phase_index += 1 ctx.step_index = 0 # Set info/state. phase_info = factory.create_info(ExecutionAspect.PHASE, ctx) phase_state = factory.create_state(ExecutionAspect.PHASE, ctx, ExecutionStatus.IN_PROGRESS) # Update cache. cache.orchestration.set_context(ctx) cache.orchestration.set_info(phase_info) cache.orchestration.set_state(phase_state) # Inform. logger.log( f"WFLOW :: {ctx.run_type} :: {ctx.run_index_label} :: {ctx.phase_index_label} -> starts" ) # Run step. do_step.send(ctx)
def do_transfer( ctx: RunContext, cp1: Account, cp2: Account, amount: int, is_refundable: bool = True ) -> typing.Tuple[Deploy, Transfer]: """Executes a transfer between 2 counter-parties & returns resulting deploy hash. :param ctx: Generator run contextual information. :param cp1: Account information of counter party 1. :param cp2: Account information of counter party 2. :param amount: Amount in motes to be transferred. :param is_refundable: Flag indicating whether a refund is required. :returns: Dispatched deploy. """ deploy_hash = get_client(ctx).transfer( amount=amount, from_addr=cp1.public_key, private_key=cp1.private_key_as_pem_filepath, target_account_hex=cp2.public_key, # TODO: allow these to be passed in via standard arguments payment_amount=defaults.CLX_TX_FEE, gas_price=defaults.CLX_TX_GAS_PRICE ) logger.log(f"PYCLX :: transfer :: {amount} CLX :: {cp1.public_key[:8]} -> {cp2.public_key[:8]} :: {deploy_hash}") return ( factory.create_deploy_for_run(ctx, deploy_hash, DeployStatus.DISPATCHED, DeployType.TRANSFER), factory.create_transfer(ctx, amount, "CLX", cp1, cp2, deploy_hash, is_refundable) )
def before_process_message(self, broker, message): """Called before a message is processed. :param broker: Message broker to which message was dispatched. :param message: A message being processed. """ msg = f"ACTOR :: {_get_actor_name(message)} :: executing ..." _logger.log(msg)
def do_transfer( ctx: ExecutionContext, cp1: Account, cp2: Account, amount: int, contract: ClientContract = None, is_refundable: bool = True, deploy_type: DeployType = DeployType.TRANSFER ) -> typing.Tuple[Deploy, Transfer]: """Executes a transfer between 2 counter-parties & returns resulting deploy hash. :param ctx: Execution context information. :param cp1: Account information of counter party 1. :param cp2: Account information of counter party 2. :param amount: Amount in motes to be transferred. :param contract: The transfer contract to call (if any). :param is_refundable: Flag indicating whether a refund is required. :param deploy_type: The type of deploy to dispatch. :returns: Dispatched deploy & transfer. """ # Set client. node, client = utils.get_client(ctx) # Transfer using called contract - does not dispatch wasm. if contract: session_args = ABI.args([ ABI.account("address", cp2.public_key), ABI.big_int("amount", amount) ]) dhash = client.deploy( session_hash=bytes.fromhex(contract.chash), session_args=session_args, from_addr=cp1.public_key, private_key=cp1.private_key_as_pem_filepath, # TODO: allow these to be passed in via standard arguments payment_amount=defaults.CLX_TX_FEE, gas_price=defaults.CLX_TX_GAS_PRICE) # Transfer using stored contract - dispatches wasm. else: dhash = client.transfer( amount=amount, from_addr=cp1.public_key, private_key=cp1.private_key_as_pem_filepath, target_account_hex=cp2.public_key, # TODO: allow these to be passed in via standard arguments payment_amount=defaults.CLX_TX_FEE, gas_price=defaults.CLX_TX_GAS_PRICE) logger.log( f"PYCLX :: transfer :: {dhash} :: {amount} CLX :: {cp1.public_key[:8]} -> {cp2.public_key[:8]}" ) return (node, dhash)
def _yield_events(network_id: NetworkIdentifier, on_block_added, on_block_finalized): """Yields events from event source (i.e. a CLX chain). """ # TODO: handle client disconnects. logger.log(f"PYCLX :: stream_events :: connecting ...") for event in get_client(network_id).stream_events( block_added=on_block_added is not None, block_finalized=on_block_finalized is not None): yield event
def main(args): """Entry point. :param args: Parsed CLI arguments. """ # Instantiate. network = factory.create_network(args.network) # Push. cache.infra.set_network(network) # Inform. logger.log(f"Network {args.network} was successfully registered")
def do_deploy_contract(ctx: RunContext, account: Account, wasm_filepath: str): """Deploys a smart contract to chain. :param ctx: Generator run contextual information. :param account: Account to be associated with contract. :param wasm_filepath: Path to smart contract's wasm file. :returns: Deploy hash (in hex format). """ pyclx = get_client(ctx) logger.log(f"TODO :: deploy-contract :: {account.key_pair.public_key.as_hex} :: {wasm_filepath}") return "TODO: dispatch contract deploy"
def get_broker() -> Broker: """Returns an MQ broker instance for integration with dramatiq framework. :returns: A configured message broker. """ # factory = FACTORIES["REDIS"] try: factory = FACTORIES[EnvVars.TYPE] except KeyError: raise InvalidEnvironmentVariable("BROKER_TYPE", EnvVars.TYPE, FACTORIES) broker = factory.get_broker() logger.log(f"... established connection to {EnvVars.TYPE} MQ broker") return broker
def main(args): """Entry point. :param args: Parsed CLI arguments. """ network_id = factory.create_network_id(args.network) network = cache.infra.get_network(network_id) if network is None: logger.log_warning(f"Network {args.network} is unregistered.") return logger.log( f"""NETWORK: {network.name} -> faucet pvk {network.faucet.private_key}""" ) logger.log( f"""NETWORK: {network.name} -> faucet pbk {network.faucet.public_key}""" )
def do_step_execute(ctx: ExecutionContext, step: WorkflowStep): """Performs step execution. :param ctx: Execution context information. :param step: Step related execution information. """ # Execute step. step.execute() # Process errors. if step.error: do_step_error.send(ctx, str(step.error)) # Async steps are processed by deploy listeners. elif step.is_async: if step.result is not None: if inspect.isfunction(step.result): message_factory = step.result() group = dramatiq.group(message_factory) group.run() else: raise TypeError( "Expecting either none or a message factory from a step function." ) logger.log( f"WFLOW :: {ctx.run_type} :: {ctx.run_index_label} :: {ctx.phase_index_label} :: {ctx.step_index_label} :: {step.label} -> listening for chain events" ) # Sync steps are processed inline. else: if step.result is None: do_step_end.send(ctx) elif inspect.isfunction(step.result): message_factory = step.result() group = dramatiq.group(message_factory) group.add_completion_callback(do_step_end.message(ctx)) group.run() else: raise TypeError( "Expecting either none or a message factory from a step function." )
def wrapper(*args, **kwargs): # Pre log. messages = { "get_block": lambda args: f"bhash={args[-1]}", "get_deploys": lambda args: f"bhash={args[-1]}", "get_balance": lambda args: f"pbk={args[-1].public_key}", } try: message = messages[func.__name__] except KeyError: logger.log(f"PYCLX :: {func.__name__} :: executing ...") else: logger.log(f"PYCLX :: {func.__name__} :: {message(args)}") try: return func(*args, **kwargs) except Exception as err: logger.log_error(f"PYCLX :: {err}") raise err
def on_finalized_deploy(network_id: NetworkIdentifier, bhash: str, dhash: str, finalization_ts: float): """Event: raised whenever a deploy is finalized. :param network_id: Identifier of network upon which a block has been finalized. :param bhash: Hash of finalized block. :param dhash: Hash of finalized deploy. :param finalization_ts: Moment in time when finalization occurred. """ # Set network deploy. deploy = factory.create_deploy(network_id, bhash, dhash, DeployStatus.FINALIZED) # Encache - skip duplicates. _, encached = cache.monitoring.set_deploy(deploy) if not encached: return logger.log(f"processing finalized deploy: {bhash} :: {dhash}") # Pull run deploy - escape if none found. deploy = cache.state.get_run_deploy(dhash) if not deploy: return # Update deploy. deploy.update_on_finalization(bhash, finalization_ts) cache.state.set_run_deploy(deploy) # Increment deploy counts. ctx = cache.orchestration.get_context(deploy.network, deploy.run_index, deploy.run_type) cache.orchestration.increment_deploy_counts(ctx) # Update transfers. transfer = cache.state.get_run_transfer(dhash) if transfer: transfer.update_on_completion() cache.state.set_run_transfer(transfer) # Signal to orchestrator. on_step_deploy_finalized.send(ctx, dhash)
def after_process_message(self, broker, message, *, result=None, exception=None): """Called after a message has been processed. :param broker: Message broker to which message was dispatched. :param message: A message being processed. """ if exception is None: return msg = f"ACTOR :: {_get_actor_name(message)} :: complete" _logger.log(msg) else: msg = f"ACTOR :: {_get_actor_name(message)} :: ERROR :: err={exception}" _logger.log_error(msg)
def main(args): """Entry point. :param args: Parsed CLI arguments. """ # Unpack. network_id = factory.create_network_id(args.node.split(':')[0]) node_id = factory.create_node_id(network_id, int(args.node.split(':')[-1])) # Pull. node = cache.infra.get_node(node_id) if node is None: raise ValueError("Unregistered node.") # Inform. logger.log( f"""NODE: {node.label} -> bonding pvk {node.account.private_key}""") logger.log( f"""NODE: {node.label} -> bonding pbk {node.account.public_key}""")
def main(args): """Entry point. :param args: Parsed CLI arguments. """ # Pull. network = cache.infra.get_network_by_name(args.network) if network is None: raise ValueError("Unregistered network.") # Update. network.status = NetworkStatus[args.status.upper()] # Push. cache.infra.set_network(network) # Notify. logger.log( f"Network {args.network} status was updated --> {network.status}")
def stream_events(src: typing.Union[NodeIdentifier, NetworkIdentifier], on_block_added: typing.Callable = None, on_block_finalized: typing.Callable = None): """Hooks upto network streaming events. :param src: The source from which a network node will be derived. :param on_block_added: Callback to invoke whenever a block is added to chain. :param on_block_finalized: Callback to invoke whenever a block is finalized. """ for node, event in _yield_events(src, on_block_added, on_block_finalized): if on_block_added and event.HasField("block_added"): bhash = event.block_added.block.summary.block_hash.hex() logger.log(f"PYCLX :: stream_events :: block added :: {bhash}") on_block_added(node, bhash) elif on_block_finalized and event.HasField("new_finalized_block"): bhash = event.new_finalized_block.block_hash.hex() logger.log(f"PYCLX :: stream_events :: block finalized :: {bhash}") on_block_finalized(node, bhash)