def test_repeat_offer(master, agent):
time.sleep(10)
#declare a framework and request offers
test_framework = Framework('test', '127.0.0.1', 8080, 8080)
offers_1 = test_framework.getOffers()
offers_2 = test_framework.getOffers()
assert len(offers_1) == 1
cpu = False
mem = False
for r in offers_1[0].resources:
if r.name == 'mem':
assert r.scalar.value == 125000000
mem = True
elif r.name == 'cpus':
assert r.scalar.value == 0.4
cpu = True
assert cpu and mem
assert len(offers_2) == 0
#are they still running?
assert master.poll() is None
assert agent.poll() is None
def test_kill_task(master, agent):
time.sleep(10)
#declare a framework and request offers
test_framework = Framework('test', '127.0.0.1', 8080, 8080)
offers = test_framework.getOffers()
valid_offers = test_framework.findAgents(offers, {
'cpus': 0.2,
'mem': 100000000
})
#run a docker task on the found agent
task_id = test_framework.runTask('test',
valid_offers[0],
docker_image='jnoor/dummy:v1')
time.sleep(5)
#get the task - there should only be one
tasks = test_framework.getTasks()
print(tasks)
assert len(tasks) == 1
assert tasks[0]['taskId'] == task_id
#now wait some time for the task to start running - maybe two minutes
t = time.time()
complete = False
while time.time() - t < 120:
tasks = test_framework.getTasks()
print(tasks)
time.sleep(1)
if tasks[0]['state'] == 'RUNNING':
complete = True
break
assert complete
#now kill the task
test_framework.killTask(task_id)
#now wait for it to die
t = time.time()
complete = False
while time.time() - t < 120:
tasks = test_framework.getTasks()
print(tasks)
time.sleep(1)
if tasks[0]['state'] == 'KILLED':
complete = True
break
assert complete
#are they still running?
assert master.poll() is None
assert agent.poll() is None
def main(host, port): # pragma: no cover
#declare a framework
framework = Framework("DDFlow", host, port)
# get offers
offers = framework.getOffers()
# First we need to launch the server task if it's not already running
agent = framework.getAgentInfoForRunningTask('DDFlow-Coordinator')
domain = None
if agent is None:
#launch the task
server_agents = framework.findAgents(offers, {
'domain': None,
'cpus': 0.5,
'mem': 100000000
})
if len(server_agents) == 0:
print("No available server agents.", file=sys.stderr)
return
framework.runTask(
"DDFlow-Coordinator",
server_agents[0],
# {'cpus':0.5,'mem':'100000000'},
docker_image='jnoor/ddflow-coordinator:v1',
docker_port_mappings={3005: 3005},
environment={'SERVER_PORT': '3005'})
print("Started ddflow coordinator task on agent: {}".format(
server_agents[0]))
domain = framework.getAgentProperty(server_agents[0], 'domain')
else:
print("DDFlow coordinator task already running.")
domain = framework.getAgentProperty(agent, 'domain')
#Find a device agent
device_agents = framework.findAgents(offers, {
'cpus': 1.0,
'mem': 100000000
})
if (len(device_agents) == 0):
print("No available ddflow device agents.", file=sys.stderr)
return
# Run the device manager task
framework.runTask(
"DDFlow-Device",
device_agents[0],
# {'cpus':1.0,'mem':'100000000'},
docker_image='jnoor/ddflow-device:v1',
docker_port_mappings={3000: 3000},
environment={'SERVER_PORT': '3000'})
print("Started ddflow device task on agent: {}".format(device_agents[0]))
args = parser.parse_args()
#extract an ID for this task
id = None
if args.id:
id = args.id
else:
id = str(args.map) + '-' + str(
args.reduce) + '-' + args.sensor + args.period
h = hashlib.sha256()
h.update(id.encode('utf-8'))
id = h.hexdigest()[0:20]
#declare a framework
framework = Framework("Map Reduce", args.host, args.port)
# check if the reduce task is already running
offers = framework.getOffers()
#get the map function
wasm_file = compile_map(args.map, args.period)
#schedule the reduce/result parts of the code
result_ip, result_port = schedule_result_server()
ip, port = schedule_reduce_server(framework, offers, result_ip,
result_port, id)
path = issue_reduce_code(args.reduce, ip, port, id, args.reissue)
#if reissue, kill all currently running map tasks
if args.reissue:
def main(host, port, client, sensor, period, func, val): # pragma: no cover
#declare a framework
framework = Framework("Sensor Framework", host, port)
# First we need to launch the server task if it's not already running
agent = framework.getAgentInfoForRunningTask('SensorServer')
#Get offers
offers = framework.getOffers()
domain = None
if agent is None:
#launch the task
server_agents = framework.findAgents(offers, {'domain':None,'cpus':0.5,'mem':100000000})
if len(server_agents) == 0:
print("No available server agents.", file=sys.stderr)
return
framework.runTask("SensorServer",server_agents[0],
docker_image='jnoor/coapserver:v1',
docker_port_mappings={3002:3002},
environment={'SERVER_PORT':'3002'})
print("Started server task on agent: {}".format(server_agents[0].agent_id))
domain = framework.getAgentProperty(server_agents[0].agent_id,'domain')
else:
print("Sensor server task already running.")
domain = framework.getAgentProperty(agent,'domain')
# okay now do a dns lookup for this domain
ip = pydig.query(domain,'A')[0]
#construct the wasm environment
env = {}
env['IP'] = ip
env['PORT'] = 3002
env['PERIOD'] = int(args.period)
#Filters
if (args.ffunc == "G"):
env['FILT_FUNC'] = 'G'
env['FILT_VAL'] = int(args.fval)
elif args.ffunc == "L":
env['FILT_FUNC'] = 'L'
env['FILT_VAL'] = int(args.fval)
#Sensor
sensor_name = None
if args.sensor == "temperature":
env['PATH'] = args.client+'-t'
env['SENSOR'] = 'temp'
sensor_name = "temperature_sensor"
elif args.sensor == "pressure":
env['PATH'] = args.client+'-p'
env['SENSOR'] = 'press'
sensor_name = "pressure_sensor"
elif args.sensor == "humidity":
env['PATH'] = args.client+'-h'
env['SENSOR'] = 'humidity'
sensor_name = "humidity_sensor"
else:
print("Not a valid sensor.",file=sys.stderr)
return
#Find a wasm agent
wasm_agents = framework.findAgents(offers, {'executors':'WASM','cpus':0.1,sensor_name:None})
if(len(wasm_agents) == 0):
print("No available sensor agents.",file=sys.stderr)
return
wasm_index = random.randrange(0,len(wasm_agents))
print("Using WASM agent " + wasm_agents[wasm_index].agent_id)
# Run the WASM task
__location__ = os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__)))
wasm_file = open(os.path.join(__location__,'./wasm/wasm-send/out.wasm'),'rb')
print("Running task with environment:")
print(env)
framework.runTask(args.client + ':' + "SensorSample",wasm_agents[wasm_index],wasm_binary=wasm_file.read(),environment=env)
print("Started sensor task on agent: {}".format(wasm_agents[wasm_index].agent_id))
def main(host, port): # pragma: no cover
#declare a framework
framework = Framework("Profiler Framework", host, port)
# First we need to launch the server task if it's not already running
agent = framework.getAgentInfoForRunningTask('ProfilerServer')
#Get offers
offers = framework.getOffers()
# find all domains
allDomains = []
for agentWithDomain in offers:
for att in agentWithDomain.attributes:
if att.name == 'domain':
allDomains.append(att.text.value)
print(allDomains)
domain = None
if agent is None:
#launch the task
server_agents = framework.findAgents(offers, {
'domain': None,
'cpus': 0.5,
'mem': 100000000
})
if len(server_agents) == 0:
print("No available server agents.")
return
framework.runTask("ProfilerServer",
server_agents[0],
docker_image='jnoor/profilerserver:v1',
docker_port_mappings={3001: 3001},
environment={'SERVER_PORT': '3001'})
print("Started server task on agent: {}".format(
server_agents[0].agent_id))
domain = framework.getAgentProperty(server_agents[0].agent_id,
'domain')
else:
print("Sensor server task already running.")
domain = framework.getAgentProperty(agent, 'domain')
# okay now do a dns lookup for this domain
print("Lookup: " + domain)
ip = pydig.query(domain, 'A')[0]
#construct the profile task environment
env = {}
env['HOST'] = domain
env['PORT'] = 3001
env['TS'] = int(time.time())
env['ENDPOINT'] = "http://" + str(ip) + ":3001/profile"
for i in range(len(allDomains)):
env['DOMAIN' + str(i)] = allDomains[i]
agents = framework.findAgents(offers, {'cpus': 0.5, "mem": 10000000})
for agent in agents:
imageToUse = None
for attribute in agent.attributes:
if attribute.name == 'OS':
# print(agent.agent_id, attribute.name, attribute.text.value)
if "x86" in attribute.text.value:
imageToUse = 'jnoor/profiler-x86:v1'
elif 'arm' in attribute.text.value:
imageToUse = 'jnoor/profiler-arm:v1'
if not imageToUse:
print("No profiler image for ")
print(agent)
continue
# if agent.agent_id != '14038003386728':
# continue
env['AGENT'] = agent.agent_id
print("Starting profiler task on agent: {}".format(agent.agent_id))
framework.runTask("profiler-" + str(agent.agent_id),
agent,
docker_image=imageToUse,
environment=env)
def main(host, port, client, camera): # pragma: no cover
#declare a framework
framework = Framework("Camera Framework", host, port)
# First we need to launch the server task if it's not already running
agent = framework.getAgentInfoForRunningTask('HTTP endpoint')
#Get offers
offers = framework.getOffers()
#Find a classify agent
classify_agents = framework.findAgents(offers, {
'cpus': 1.0,
"mem": 200000000,
'domain': None
})
if (len(classify_agents) == 0):
print("No available classify agents.", file=sys.stderr)
return
#Find a camera agent
cam_agents = framework.findAgents(offers, {
'cpus': 0.5,
"mem": 100000000,
"picam": None,
'agent_id': camera
})
if (len(cam_agents) == 0):
print("No available camera agents.", file=sys.stderr)
return
domain = None
if agent is None:
#launch the task
server_agents = framework.findAgents(offers, {
'domain': None,
'cpus': 0.5,
'mem': 100000000
})
if len(server_agents) == 0:
print("No available server agents.", file=sys.stderr)
return
framework.runTask("HTTP endpoint",
server_agents[0],
docker_image='jnoor/hellocameraserver:v1',
docker_port_mappings={3003: 3003},
environment={'SERVER_PORT': '3003'})
print("Started server task on agent: {}".format(
server_agents[0].agent_id))
domain = framework.getAgentProperty(server_agents[0].agent_id,
'domain')
else:
print("HTTP server task already running.")
domain = framework.getAgentProperty(agent, 'domain')
# Run the classify task
unique_key = client + '-' + str(randint(0, 1000000))
framework.runTask(
client + ':' + "image classification",
classify_agents[0],
docker_image='jnoor/classify:v1',
environment={
'INPUT_URL':
'http://' + domain + ':3003/' + unique_key + '-latest.jpg',
'OUTPUT_URL':
'http://' + domain + ':3003/' + client + '-predictions.jpg',
'OUTPUTRESULT_URL':
'http://' + domain + ':3003/' + client + '-results.json'
})
# Run the classify task
framework.runTask(client + ':' + "camera task",
cam_agents[0],
docker_image='jnoor/cameraalpine:v1',
environment={
'SERVER_HOST':
'http://' + domain + ':3003/' + unique_key +
'-latest.jpg'
})
print("Started classify task on agent: {}".format(
classify_agents[0].agent_id))
print("Started camera task on agent: {}".format(cam_agents[0].agent_id))