def main(): context = get_context() try: m = context['message_dict']['text'] reducer = context['message_dict']['reducer'] print("Got JSON.") except Exception: print("Didnt get JSON, using raw message.") reducer = None m = context.raw_message wordcount = {} for word in m.split(): w = word.lower() if w in wordcount: wordcount[w] += 1 else: wordcount[w] = 1 print("Final word count:") for k, v in wordcount.items(): print("{}: {}".format(k, v)) if reducer: print("Sending message to reducer {}".format(reducer)) ag = get_client() ag.actors.sendMessage(actorId=reducer, body={'message': { 'count': wordcount }})
def main(): ITERATIONS_DEFAULT = 6 SLEEP_DEFAULT = 10 context = get_context() try: msg_dict = context['message_dict'] except KeyError as e: print("KeyError getting message_dict".format(e)) msg_dict = None iterations = ITERATIONS_DEFAULT sleep_time_s = SLEEP_DEFAULT if msg_dict: try: iterations = int(msg_dict.get('iterations', ITERATIONS_DEFAULT)) sleep_time_s = int(msg_dict.get('sleep', SLEEP_DEFAULT)) except Exception as e: print( "Got exception {} trying to read iterations and sleep from msg_dict".format(e)) for i in range(iterations): print("starting iteration {}".format(i)) time.sleep(sleep_time_s) print("sleep_loop completed.")
def main(): context = get_context() print("Contents of context:") for k, v in context.items(): print("key: {}. value: {}".format(k, v)) print("Contents of env: {}".format(os.environ))
def get_context_with_mock_support(agave_client): ''' Return the current Actor context Return the Abaco actor's environment context if running deployed. Creates a test context based on inferred or mocked values if running in local or debug mode. ''' _context = get_context() if os.environ.get('_abaco_actor_id') is None: _phony_actor_id = uniqueid.get_id() _phony_exec_id = uniqueid.get_id() _username = os.environ.get('_abaco_username', None) if _username is None: try: _username = agave_client.username except Exception: pass __context = AttrDict({'raw_message': os.environ.get('MSG', ''), 'content_type': 'application/json', 'username': _username, 'actor_dbid': _phony_actor_id, 'actor_id': _phony_actor_id, 'execution_id': _phony_exec_id, 'state': {}}) # Merge new values from __context _context = _context + __context # Update environment set_os_environ_from_mock(__context) set_os_environ_from_client(agave_client) return AttrDict(_context)
def main(): # default size, in GB, to test print("start of mem_limit") SIZE_DEFAULT = 2 context = get_context() try: msg_dict = context['message_dict'] except KeyError as e: print("KeyError getting message_dict".format(e)) msg_dict = None if msg_dict: try: size = int(msg_dict.get('size', SIZE_DEFAULT)) except Exception as e: print("Got exception {} trying to read size from msg_dict".format(e)) if size > 10: print("Invalid size parameter - received a value larger than the max value of 10.") raise Exception() print("mem_limit using size: {}".format(size)) # try to create a python object of a certain size - ONE_G = 1000000000 factor = size * ONE_G try: s = factor*'a' except MemoryError as e: print("got a memory error; e: {}".format(e)) except Exception as e: print("Got a non-MemoryError exception; e: {}".format(e)) print("actual size of var: {}".format(sys.getsizeof(s))) print("mem_limit completed.")
def main(): context = get_context() try: count = context['message_dict']['count'] print("Got JSON. Count: {}".format(count)) except Exception: print("Didnt get JSON, aborting.") sys.exit() state = context['state'] print("Type of state: {}".format(type(state))) print("Initial state: {}".format(state)) if type(state) == str: try: state = ast.literal_eval(state) except ValueError: state = {} else: state = {} # iterate over all keys in the count for k, v in count.items(): if k in state: state[k] += v else: state[k] = v print("Final word count:") for k, v in state.items(): print("{}: {}".format(k, v)) update_state(state)
def main(): start = time.time() logs = {} logs['messages'] = f'Max Cores: {os.cpu_count()}' # intializes core count, bit difficulty, and blockchain length context = get_context() try: n_bits = int(context['message_dict']['bits']) except Exception: logs[ 'messages'] += 'Didnt get any/valid values.. using default values..' n_bits = 10 logs[ 'messages'] = f' Mining a {n_bits}-bit block to the next block on 1 core..' ##### genesis_block = randomBlock(bits=n_bits) next_block = mineNextBlock(genesis_block) ##### hashrate = next_block['hashes'] / (end - start) end = time.time() # writing logs logs['genesis_block'] = str(genesis_block) logs['next_block'] = str(next_block) logs['runtime'] = end - start logs['hashrate'] = hashrate logs['hashes'] = next_block['hashes'] # sends logs to execution/logs print(logs, end='')
def main(): print("executing") context = get_context() path = context['message_dict'].get('path', '/data/samples') system_id = context['message_dict'].get('system_id', 'reactors.storage.sample') ag = get_client() try: fs = ag.files.list(systemId=system_id, filePath=path) except Exception as e: try: print("files error; content: {}".format(e.response.content)) except Exception as e2: print("files original: {}; 2nd: {}".format(e, e2)) try: p = ag.profiles.get() except Exception as e: try: print("profiles error; content: {}".format(e.response.content)) except Exception as e2: print("profiles original: {}; 2nd: {}".format(e, e2)) try: a = ag.apps.list() except Exception as e: try: print("apps error; content: {}".format(e.response.content)) except Exception as e2: print("apps original: {}; 2nd: {}".format(e, e2)) try: print(len(fs)) print(p) print(a) except Exception as e: print("failure: {}".format(e))
def main(): context = get_context() print("Contents of context:") for k, v in context.items(): print("key: {}. value: {}".format(k, v)) print("Contents of env: {}".format(os.environ)) # check identity: print("*** Posix Identity Information ***") try: print("Posix username: {}".format(getpass.getuser())) except Exception as e: # it is possible to get an exception trying to look up the username because the uid may not # exist there. Swallow it and keep going. print("Got an exception trying to look up the username: {}".format(e)) print("Posix uid: {}".format(os.getuid())) print("Posix gid: {}".format(os.getgid())) # instantiate the agavepy client: try: ag = get_client() except Exception as e: print("Got exception {} trying to get client.".format(e)) sys.exit() try: prof = ag.profiles.get() print("This actor was executed by the user with profile: {}".format( prof)) except Exception as e: print( "Got an exception trying to get the profile. Exception: {}".format( e))
def main(): context = get_context() path = context['message_dict'].get('path', '/data/samples') system_id = context['message_dict'].get('system_id', 'reactors.storage.sample') ag = get_client() fs = ag.files.list(systemId=system_id, filePath=path) print("Files: {}".format(fs))
def main(): context = get_context() ag = get_client() try: job = ag.jobs.submit(body=context['message_dict']) print("jobs {} submitted successfully.".format(job)) except Exception as e: print("Exception submitting job: {}".format(e))
def main(): context = get_context() print("Contents of context:") for k, v in context.items(): print("key: {}. value: {}".format(k, v)) print("Contents of env: {}".format(os.environ)) cli = docker.AutoVersionClient(base_url='unix://var/run/docker.sock') print("Containers: {}".format(cli.containers()))
def main(): context = get_context() try: length = context['message_dict']['length'] print("Got JSON: {}".format(context['message_dict'])) except Exception: print("Didnt get JSON, using defaults.") length = 100000 log = 'a'*length print(log)
def load_context(enable_mocks=True): # Replaces the byzantine get_context_with_mock_support # which predates the inbuilt capabilities of AgavePy to # bootstrap from environment variables. A current side # effect of this implementation is that the mocked # context is not *exactly* the same as a real context # It is missing the # Diagnose whether we are running under Abaco actor_id_val = os.environ.get('_abaco_actor_id') if actor_id_val is not None and actor_id_val != '': return AttrDict(get_context()) else: if enable_mocks: # Build a mock context ab_ctxt = get_context() ctxt = { 'raw_message': os.environ.get('MSG', ''), 'username': os.environ.get('USER', 'reactor'), 'content_type': 'application/json', 'actor_dbid': new_hashid(), 'actor_id': new_hashid(), 'execution_id': new_hashid(), 'worker_id': new_hashid(), 'actor_repo': new_hashid(), 'actor_name': new_hashid(), 'state': {} } # Merge in mock values if not provided by Abaco context merged_ctxt = ab_ctxt for k, v in ctxt.items(): if ab_ctxt.get(k, None) is not None: merged_ctxt[k] = ab_ctxt.get(k) else: merged_ctxt[k] = v return AttrDict(merged_ctxt) else: # TODO - raise a more specific exception raise Exception( "Unable to load Reactor context from environment (or return a mock value)" )
def main(): ag = get_client() context = get_context() m = context.message_dict file_m = m.get('file') if not file_m: print "Not a file event." sys.exit() status = file_m['status'] path = file_m['path'] system_id = file_m['systemId'] native_format = file_m['nativeFormat'] print "Status: {} path:{} system_id:{} format:{} ".format( status, path, system_id, native_format) try: rsp = ag.files.list(systemId=system_id, filePath=path) except Exception as e: print "Got an exception trying to list files: {}".format(e) print "URL on the request: {}".format(e.request.url) print "Request headers: {}".format(e.request.headers) print "Request body: {}".format(e.request.body) print "Agave files response: {}".format(rsp) if status == 'STAGING_COMPLETED' or status == 'TRANSFORMING_COMPLETED': if native_format == 'dir': # new project directory, let's add our project skeleton ag.files.manage(systemId=system_id, filePath=path, body={ 'action': 'mkdir', 'path': 'data' }) ag.files.manage(systemId=system_id, filePath=path, body={ 'action': 'mkdir', 'path': 'analysis' }) ag.files.manage(systemId=system_id, filePath=path, body={ 'action': 'mkdir', 'path': 'logs' }) else: print "Skipping since native_format was {}, not 'dir'".format( native_format) else: print "Skipping since status was {}".format(status)
def main(): print("top of main") context = get_context() message = get_message(context) std_dev = message.get('std_dev', 100) size = message.get('size', 4096) try: std_dev = int(std_dev) except: std_dev = 100 try: size = int(size) except: size = 4096 print("using std_dev {} and size {}".format(std_dev, size)) r = f(std_dev, size) print("result: {}".format(r)) send_python_result(r)
def main(): ag = get_client() context = get_context() m = context.message_dict print "new_project actor running for message: {}".format(m) file_m = m.get('file') if not file_m: print "Not a file event." sys.exit() status = file_m.get('status') if status == 'STAGING_COMPLETED': native_format = file_m['nativeFormat'] if native_format == 'raw': path = file_m['path'] system_id = file_m['systemId'] process_image(ag, path, system_id) else: print "Skipping since native_format was {}, not 'raw'".format( native_format) else: print "Skipping since status was {}".format(status)
def main(): context = get_context() d = context['message_dict'] print("Got JSON: {}".format(d)) try: # message variables -- project_id = d['project_id'] site_id = d['site_id'] inst_id = d['inst_id'] time = d['time'] except KeyError as ke: print(f"Required field missing: {ke}") raise ke except Exception as e: print(f"Unexpected exception: {e}.") raise e start_datetime, end_datetime = get_datetime_range(time) csv_data = get_measurements(project_id, site_id, inst_id, start_datetime, end_datetime) df = create_dataframe(csv_data) generate_plot_from_df(df) upload_plot(time)
def get_context_with_mock_support(): """ Return the Abaco actor's environment context if running as a function. Otherwise, return a mock environment for debug and test purposes """ _context = get_context() if os.environ.get('_abaco_actor_id') is None: _phony_actor_id = get_id() + '.local' __context = AttrDict({ 'raw_message': os.environ.get('MSG', ''), 'content_type': 'application/json', 'execution_id': get_id() + '.local', 'username': os.environ.get('_abaco_username'), 'state': {}, 'actor_dbid': _phony_actor_id, 'actor_id': _phony_actor_id }) # Merge new values from __context _context = _context + __context return _context
def main(): context = get_context() message = context['message_dict'] audit_required_fields(context, message) conn, ip = get_ssh_connection(context, message) user_name, container_name = get_user_data(context, message) ag = get_agave_client(message) m = TerminalMetadata(user_name, ag) command = message.get('command', 'START') if command == 'START': url = launch_terminal(user_name, container_name, conn, ip) if not url: m.set_error("Actor got an error trying to launch terminal. Check the logs") else: print("Setting actor to ready status for URL: {}".format(url)) m.set_ready(url) print("Status updated. Actor exiting.") elif command == 'STOP': stop_terminal(container_name, conn) m.set_stopped() elif command == "SYNC": get_terminal_status()
def get_context_with_mock_support(): ''' Return the current Actor context Return the Abaco actor's environment context if running deployed. Creates a test context based on inferred or mocked values if running in local or debug mode. ''' _context = get_context() if os.environ.get('_abaco_actor_id') is None: _phony_actor_id = uniqueid.get_id() __context = AttrDict({ 'raw_message': os.environ.get('MSG', ''), 'content_type': 'application/json', 'execution_id': uniqueid.get_id(), 'username': os.environ.get('_abaco_username'), 'state': {}, 'actor_dbid': _phony_actor_id, 'actor_id': _phony_actor_id }) # Merge new values from __context _context = _context + __context return _context
def main(): context = get_context() message = get_message(context) ag = get_apy_client(context) # if the message has a start field, this is an initiation message to the first actor; we need to do # some bootstrapping. if 'start' in message: rsp = create_second_actor(context, ag) actor2_id = rsp['id'] rsp = create_nonce(context, ag) nonce_id = rsp['id'] count = 0 else: # if the message does not have a start field then this message is a # get the target actor id out of the message actor2_id = message['target_actor_id'] count = int(message['count']) nonce_id = message['nonce_id'] # get nonce out of the env if 'x-nonce' in context: nonce = context['x-nonce'] print("Got nonce: {}".format(nonce)) else: print("Did not get nonce, exiting.") sys.exit() # get count and tot out of message try: count = context['message_dict']['count'] tot = context['message_dict']['tot'] print("Got JSON. Count: {} tot: {}".format(count, tot)) except Exception: print("Didnt get JSON, aborting.") sys.exit() try: count = int(count) tot = int(tot) except Exception: print("Got bad count or tot data, exiting.") sys.exit() # check recursion depth and send message: if count > 0: count -= 1 tot += 1 message = {'count': count, 'tot': tot} url = '{}/actors/{}/messages'.format(context['_abaco_api_server']) # else, we are done counting; do some basic checks: print("*** Posix Identity Information ***") try: print("Posix username: {}".format(getpass.getuser())) except Exception as e: # it is possible to get an exception trying to look up the username because the uid may not # exist there. Swallow it and keep going. print("Got an exception trying to look up the username: {}".format(e)) print("Posix uid: {}".format(os.getuid())) print("Posix gid: {}".format(os.getgid())) try: prof = ag.profiles.get() print("This actor was executed by the user with profile: {}".format( prof)) except Exception as e: print( "Got an exception trying to get the profile. Exception: {}".format( e))
def main(): context = get_context() message = context['raw_message'] echo_message(message)
from io import TextIOWrapper from os.path import exists from agavepy import actors import os, json from agavepy.agave import AttrDict if exists("/container_scripts/offline.env"): print("offline.env exists, skipping set_env.py") exit(0) # Pull variables from abaco message. # msg is a python dictionary, returned from the JSON-parsing get_context(). msg_base: AttrDict = actors.get_context() if "json" in msg_base['content_type']: msg: dict = msg_base['message_dict'] # Set location for agavepy to look for the "currents" file when running Agave.restore() os.environ['TAPIS_CACHE_DIR'] = msg['TAPIS_CACHE_DIR'] # Build the "currents" file. currents: dict = { "refresh token": "", "expires_in": "", "expires_at": "", "created_at": "", "username": msg['GATEWAY_USERNAME'], "token_username": None, "client_name": msg['GATEWAY_CLIENT_NAME'], "use_nonce": False, "verify": True, "proxies": {},
def get_vars(agave_context): '''Returns container, system, and outdir variables''' # check if passed as JSON if type(context.message_dict) is dict: c, s, o = get_vars_json(agave_context) else: c, s, o = get_vars_env(agave_context) return (c, s, o) if __name__ == '__main__': # get context and client context = get_context() ag = get_client() # container, agave system, and path to outdir container, system, outdir = get_vars(context) # execute d2s with bash d2s_cmd = 'bash /docker2singularity.sh {}'.format(container) process = Popen(d2s_cmd.split()).wait() assert int( process) == 0, 'd2s command finished with non-zero status: {}'.format( str(process)) # find image file produced files = ' '.join(listdir('/output')) regex = container.replace('/', '_').replace(