def main():
context = get_context()
try:
m = context['message_dict']['text']
reducer = context['message_dict']['reducer']
print("Got JSON.")
except Exception:
print("Didnt get JSON, using raw message.")
reducer = None
m = context.raw_message
wordcount = {}
for word in m.split():
w = word.lower()
if w in wordcount:
wordcount[w] += 1
else:
wordcount[w] = 1
print("Final word count:")
for k, v in wordcount.items():
print("{}: {}".format(k, v))
if reducer:
print("Sending message to reducer {}".format(reducer))
ag = get_client()
ag.actors.sendMessage(actorId=reducer,
body={'message': {
'count': wordcount
}})
def main():
ITERATIONS_DEFAULT = 6
SLEEP_DEFAULT = 10
context = get_context()
try:
msg_dict = context['message_dict']
except KeyError as e:
print("KeyError getting message_dict".format(e))
msg_dict = None
iterations = ITERATIONS_DEFAULT
sleep_time_s = SLEEP_DEFAULT
if msg_dict:
try:
iterations = int(msg_dict.get('iterations', ITERATIONS_DEFAULT))
sleep_time_s = int(msg_dict.get('sleep', SLEEP_DEFAULT))
except Exception as e:
print(
"Got exception {} trying to read iterations and sleep from msg_dict".format(e))
for i in range(iterations):
print("starting iteration {}".format(i))
time.sleep(sleep_time_s)
print("sleep_loop completed.")
def main():
context = get_context()
print("Contents of context:")
for k, v in context.items():
print("key: {}. value: {}".format(k, v))
print("Contents of env: {}".format(os.environ))
def get_context_with_mock_support(agave_client):
'''
Return the current Actor context
Return the Abaco actor's environment context if running deployed. Creates
a test context based on inferred or mocked values if running in local or
debug mode.
'''
_context = get_context()
if os.environ.get('_abaco_actor_id') is None:
_phony_actor_id = uniqueid.get_id()
_phony_exec_id = uniqueid.get_id()
_username = os.environ.get('_abaco_username', None)
if _username is None:
try:
_username = agave_client.username
except Exception:
pass
__context = AttrDict({'raw_message': os.environ.get('MSG', ''),
'content_type': 'application/json',
'username': _username,
'actor_dbid': _phony_actor_id,
'actor_id': _phony_actor_id,
'execution_id': _phony_exec_id,
'state': {}})
# Merge new values from __context
_context = _context + __context
# Update environment
set_os_environ_from_mock(__context)
set_os_environ_from_client(agave_client)
return AttrDict(_context)
def main():
# default size, in GB, to test
print("start of mem_limit")
SIZE_DEFAULT = 2
context = get_context()
try:
msg_dict = context['message_dict']
except KeyError as e:
print("KeyError getting message_dict".format(e))
msg_dict = None
if msg_dict:
try:
size = int(msg_dict.get('size', SIZE_DEFAULT))
except Exception as e:
print("Got exception {} trying to read size from msg_dict".format(e))
if size > 10:
print("Invalid size parameter - received a value larger than the max value of 10.")
raise Exception()
print("mem_limit using size: {}".format(size))
# try to create a python object of a certain size -
ONE_G = 1000000000
factor = size * ONE_G
try:
s = factor*'a'
except MemoryError as e:
print("got a memory error; e: {}".format(e))
except Exception as e:
print("Got a non-MemoryError exception; e: {}".format(e))
print("actual size of var: {}".format(sys.getsizeof(s)))
print("mem_limit completed.")
def main():
context = get_context()
try:
count = context['message_dict']['count']
print("Got JSON. Count: {}".format(count))
except Exception:
print("Didnt get JSON, aborting.")
sys.exit()
state = context['state']
print("Type of state: {}".format(type(state)))
print("Initial state: {}".format(state))
if type(state) == str:
try:
state = ast.literal_eval(state)
except ValueError:
state = {}
else:
state = {}
# iterate over all keys in the count
for k, v in count.items():
if k in state:
state[k] += v
else:
state[k] = v
print("Final word count:")
for k, v in state.items():
print("{}: {}".format(k, v))
update_state(state)
def main():
start = time.time()
logs = {}
logs['messages'] = f'Max Cores: {os.cpu_count()}'
# intializes core count, bit difficulty, and blockchain length
context = get_context()
try:
n_bits = int(context['message_dict']['bits'])
except Exception:
logs[
'messages'] += 'Didnt get any/valid values.. using default values..'
n_bits = 10
logs[
'messages'] = f' Mining a {n_bits}-bit block to the next block on 1 core..'
#####
genesis_block = randomBlock(bits=n_bits)
next_block = mineNextBlock(genesis_block)
#####
hashrate = next_block['hashes'] / (end - start)
end = time.time()
# writing logs
logs['genesis_block'] = str(genesis_block)
logs['next_block'] = str(next_block)
logs['runtime'] = end - start
logs['hashrate'] = hashrate
logs['hashes'] = next_block['hashes']
# sends logs to execution/logs
print(logs, end='')
def main():
print("executing")
context = get_context()
path = context['message_dict'].get('path', '/data/samples')
system_id = context['message_dict'].get('system_id',
'reactors.storage.sample')
ag = get_client()
try:
fs = ag.files.list(systemId=system_id, filePath=path)
except Exception as e:
try:
print("files error; content: {}".format(e.response.content))
except Exception as e2:
print("files original: {}; 2nd: {}".format(e, e2))
try:
p = ag.profiles.get()
except Exception as e:
try:
print("profiles error; content: {}".format(e.response.content))
except Exception as e2:
print("profiles original: {}; 2nd: {}".format(e, e2))
try:
a = ag.apps.list()
except Exception as e:
try:
print("apps error; content: {}".format(e.response.content))
except Exception as e2:
print("apps original: {}; 2nd: {}".format(e, e2))
try:
print(len(fs))
print(p)
print(a)
except Exception as e:
print("failure: {}".format(e))
def main():
context = get_context()
print("Contents of context:")
for k, v in context.items():
print("key: {}. value: {}".format(k, v))
print("Contents of env: {}".format(os.environ))
# check identity:
print("*** Posix Identity Information ***")
try:
print("Posix username: {}".format(getpass.getuser()))
except Exception as e:
# it is possible to get an exception trying to look up the username because the uid may not
# exist there. Swallow it and keep going.
print("Got an exception trying to look up the username: {}".format(e))
print("Posix uid: {}".format(os.getuid()))
print("Posix gid: {}".format(os.getgid()))
# instantiate the agavepy client:
try:
ag = get_client()
except Exception as e:
print("Got exception {} trying to get client.".format(e))
sys.exit()
try:
prof = ag.profiles.get()
print("This actor was executed by the user with profile: {}".format(
prof))
except Exception as e:
print(
"Got an exception trying to get the profile. Exception: {}".format(
e))
def main():
context = get_context()
path = context['message_dict'].get('path', '/data/samples')
system_id = context['message_dict'].get('system_id',
'reactors.storage.sample')
ag = get_client()
fs = ag.files.list(systemId=system_id, filePath=path)
print("Files: {}".format(fs))
def main():
context = get_context()
ag = get_client()
try:
job = ag.jobs.submit(body=context['message_dict'])
print("jobs {} submitted successfully.".format(job))
except Exception as e:
print("Exception submitting job: {}".format(e))
def main():
context = get_context()
print("Contents of context:")
for k, v in context.items():
print("key: {}. value: {}".format(k, v))
print("Contents of env: {}".format(os.environ))
cli = docker.AutoVersionClient(base_url='unix://var/run/docker.sock')
print("Containers: {}".format(cli.containers()))
def main():
context = get_context()
try:
length = context['message_dict']['length']
print("Got JSON: {}".format(context['message_dict']))
except Exception:
print("Didnt get JSON, using defaults.")
length = 100000
log = 'a'*length
print(log)
def load_context(enable_mocks=True):
# Replaces the byzantine get_context_with_mock_support
# which predates the inbuilt capabilities of AgavePy to
# bootstrap from environment variables. A current side
# effect of this implementation is that the mocked
# context is not *exactly* the same as a real context
# It is missing the
# Diagnose whether we are running under Abaco
actor_id_val = os.environ.get('_abaco_actor_id')
if actor_id_val is not None and actor_id_val != '':
return AttrDict(get_context())
else:
if enable_mocks:
# Build a mock context
ab_ctxt = get_context()
ctxt = {
'raw_message': os.environ.get('MSG', ''),
'username': os.environ.get('USER', 'reactor'),
'content_type': 'application/json',
'actor_dbid': new_hashid(),
'actor_id': new_hashid(),
'execution_id': new_hashid(),
'worker_id': new_hashid(),
'actor_repo': new_hashid(),
'actor_name': new_hashid(),
'state': {}
}
# Merge in mock values if not provided by Abaco context
merged_ctxt = ab_ctxt
for k, v in ctxt.items():
if ab_ctxt.get(k, None) is not None:
merged_ctxt[k] = ab_ctxt.get(k)
else:
merged_ctxt[k] = v
return AttrDict(merged_ctxt)
else:
# TODO - raise a more specific exception
raise Exception(
"Unable to load Reactor context from environment (or return a mock value)"
)
def main():
ag = get_client()
context = get_context()
m = context.message_dict
file_m = m.get('file')
if not file_m:
print "Not a file event."
sys.exit()
status = file_m['status']
path = file_m['path']
system_id = file_m['systemId']
native_format = file_m['nativeFormat']
print "Status: {} path:{} system_id:{} format:{} ".format(
status, path, system_id, native_format)
try:
rsp = ag.files.list(systemId=system_id, filePath=path)
except Exception as e:
print "Got an exception trying to list files: {}".format(e)
print "URL on the request: {}".format(e.request.url)
print "Request headers: {}".format(e.request.headers)
print "Request body: {}".format(e.request.body)
print "Agave files response: {}".format(rsp)
if status == 'STAGING_COMPLETED' or status == 'TRANSFORMING_COMPLETED':
if native_format == 'dir':
# new project directory, let's add our project skeleton
ag.files.manage(systemId=system_id,
filePath=path,
body={
'action': 'mkdir',
'path': 'data'
})
ag.files.manage(systemId=system_id,
filePath=path,
body={
'action': 'mkdir',
'path': 'analysis'
})
ag.files.manage(systemId=system_id,
filePath=path,
body={
'action': 'mkdir',
'path': 'logs'
})
else:
print "Skipping since native_format was {}, not 'dir'".format(
native_format)
else:
print "Skipping since status was {}".format(status)
def main():
print("top of main")
context = get_context()
message = get_message(context)
std_dev = message.get('std_dev', 100)
size = message.get('size', 4096)
try:
std_dev = int(std_dev)
except:
std_dev = 100
try:
size = int(size)
except:
size = 4096
print("using std_dev {} and size {}".format(std_dev, size))
r = f(std_dev, size)
print("result: {}".format(r))
send_python_result(r)
def main():
ag = get_client()
context = get_context()
m = context.message_dict
print "new_project actor running for message: {}".format(m)
file_m = m.get('file')
if not file_m:
print "Not a file event."
sys.exit()
status = file_m.get('status')
if status == 'STAGING_COMPLETED':
native_format = file_m['nativeFormat']
if native_format == 'raw':
path = file_m['path']
system_id = file_m['systemId']
process_image(ag, path, system_id)
else:
print "Skipping since native_format was {}, not 'raw'".format(
native_format)
else:
print "Skipping since status was {}".format(status)
def main():
context = get_context()
d = context['message_dict']
print("Got JSON: {}".format(d))
try:
# message variables --
project_id = d['project_id']
site_id = d['site_id']
inst_id = d['inst_id']
time = d['time']
except KeyError as ke:
print(f"Required field missing: {ke}")
raise ke
except Exception as e:
print(f"Unexpected exception: {e}.")
raise e
start_datetime, end_datetime = get_datetime_range(time)
csv_data = get_measurements(project_id, site_id, inst_id, start_datetime,
end_datetime)
df = create_dataframe(csv_data)
generate_plot_from_df(df)
upload_plot(time)
def get_context_with_mock_support():
"""
Return the Abaco actor's environment
context if running as a function. Otherwise,
return a mock environment for debug
and test purposes
"""
_context = get_context()
if os.environ.get('_abaco_actor_id') is None:
_phony_actor_id = get_id() + '.local'
__context = AttrDict({
'raw_message': os.environ.get('MSG', ''),
'content_type': 'application/json',
'execution_id': get_id() + '.local',
'username': os.environ.get('_abaco_username'),
'state': {},
'actor_dbid': _phony_actor_id,
'actor_id': _phony_actor_id
})
# Merge new values from __context
_context = _context + __context
return _context
def main():
context = get_context()
message = context['message_dict']
audit_required_fields(context, message)
conn, ip = get_ssh_connection(context, message)
user_name, container_name = get_user_data(context, message)
ag = get_agave_client(message)
m = TerminalMetadata(user_name, ag)
command = message.get('command', 'START')
if command == 'START':
url = launch_terminal(user_name, container_name, conn, ip)
if not url:
m.set_error("Actor got an error trying to launch terminal. Check the logs")
else:
print("Setting actor to ready status for URL: {}".format(url))
m.set_ready(url)
print("Status updated. Actor exiting.")
elif command == 'STOP':
stop_terminal(container_name, conn)
m.set_stopped()
elif command == "SYNC":
get_terminal_status()
def get_context_with_mock_support():
'''
Return the current Actor context
Return the Abaco actor's environment context if running deployed. Creates
a test context based on inferred or mocked values if running in local or
debug mode.
'''
_context = get_context()
if os.environ.get('_abaco_actor_id') is None:
_phony_actor_id = uniqueid.get_id()
__context = AttrDict({
'raw_message': os.environ.get('MSG', ''),
'content_type': 'application/json',
'execution_id': uniqueid.get_id(),
'username': os.environ.get('_abaco_username'),
'state': {},
'actor_dbid': _phony_actor_id,
'actor_id': _phony_actor_id
})
# Merge new values from __context
_context = _context + __context
return _context
def main():
context = get_context()
message = get_message(context)
ag = get_apy_client(context)
# if the message has a start field, this is an initiation message to the first actor; we need to do
# some bootstrapping.
if 'start' in message:
rsp = create_second_actor(context, ag)
actor2_id = rsp['id']
rsp = create_nonce(context, ag)
nonce_id = rsp['id']
count = 0
else:
# if the message does not have a start field then this message is a
# get the target actor id out of the message
actor2_id = message['target_actor_id']
count = int(message['count'])
nonce_id = message['nonce_id']
# get nonce out of the env
if 'x-nonce' in context:
nonce = context['x-nonce']
print("Got nonce: {}".format(nonce))
else:
print("Did not get nonce, exiting.")
sys.exit()
# get count and tot out of message
try:
count = context['message_dict']['count']
tot = context['message_dict']['tot']
print("Got JSON. Count: {} tot: {}".format(count, tot))
except Exception:
print("Didnt get JSON, aborting.")
sys.exit()
try:
count = int(count)
tot = int(tot)
except Exception:
print("Got bad count or tot data, exiting.")
sys.exit()
# check recursion depth and send message:
if count > 0:
count -= 1
tot += 1
message = {'count': count, 'tot': tot}
url = '{}/actors/{}/messages'.format(context['_abaco_api_server'])
# else, we are done counting; do some basic checks:
print("*** Posix Identity Information ***")
try:
print("Posix username: {}".format(getpass.getuser()))
except Exception as e:
# it is possible to get an exception trying to look up the username because the uid may not
# exist there. Swallow it and keep going.
print("Got an exception trying to look up the username: {}".format(e))
print("Posix uid: {}".format(os.getuid()))
print("Posix gid: {}".format(os.getgid()))
try:
prof = ag.profiles.get()
print("This actor was executed by the user with profile: {}".format(
prof))
except Exception as e:
print(
"Got an exception trying to get the profile. Exception: {}".format(
e))
def main():
context = get_context()
message = context['raw_message']
echo_message(message)
from io import TextIOWrapper
from os.path import exists
from agavepy import actors
import os, json
from agavepy.agave import AttrDict
if exists("/container_scripts/offline.env"):
print("offline.env exists, skipping set_env.py")
exit(0)
# Pull variables from abaco message.
# msg is a python dictionary, returned from the JSON-parsing get_context().
msg_base: AttrDict = actors.get_context()
if "json" in msg_base['content_type']:
msg: dict = msg_base['message_dict']
# Set location for agavepy to look for the "currents" file when running Agave.restore()
os.environ['TAPIS_CACHE_DIR'] = msg['TAPIS_CACHE_DIR']
# Build the "currents" file.
currents: dict = {
"refresh token": "",
"expires_in": "",
"expires_at": "",
"created_at": "",
"username": msg['GATEWAY_USERNAME'],
"token_username": None,
"client_name": msg['GATEWAY_CLIENT_NAME'],
"use_nonce": False,
"verify": True,
"proxies": {},
def get_vars(agave_context):
'''Returns container, system, and outdir variables'''
# check if passed as JSON
if type(context.message_dict) is dict:
c, s, o = get_vars_json(agave_context)
else:
c, s, o = get_vars_env(agave_context)
return (c, s, o)
if __name__ == '__main__':
# get context and client
context = get_context()
ag = get_client()
# container, agave system, and path to outdir
container, system, outdir = get_vars(context)
# execute d2s with bash
d2s_cmd = 'bash /docker2singularity.sh {}'.format(container)
process = Popen(d2s_cmd.split()).wait()
assert int(
process) == 0, 'd2s command finished with non-zero status: {}'.format(
str(process))
# find image file produced
files = ' '.join(listdir('/output'))
regex = container.replace('/', '_').replace(
