def complete(self):
completion_time = str(datetime.datetime.now())
set_job(self._job_id, {'job_status': 'completed', 'completion_time': completion_time})
submitter_url = get_job(self._job_id)['submitter_url']
submitter_port = get_job(self._job_id)['submitter_port']
orchestrator_url = get_job(self._job_id)['orchestrator_url']
orchestrator_port = get_job(self._job_id)['orchestrator_port']
requests.post(f'http://{submitter_url}:{submitter_port}/complete',
params={'job_id': self._job_id,
'completion_time': completion_time,
'executor_name': get_global('agent_name'),
'executor_url': get_global('agent_url'),
'executor_port': get_global('agent_port')
})
requests.post(f'http://{orchestrator_url}:{orchestrator_port}/complete',
params={'job_id': self._job_id,
'agent_name': get_global('agent_name'),
'completion_time': completion_time})
self.report_job(self._job_id, 'completed')
def report(self, message, target=None, job_id=None):
if target is None:
target = get_global("tracker_host")
requests.get(f'http://{target}:3000/log_report',
params={
'agent_name': get_global('agent_name'),
'agent_log': message,
'job_id': job_id
}
)
def connectivity():
current_time = datetime.now()
try:
heartbeat_last = datetime.strptime(get_global('heartbeat_last'),
"%Y-%m-%d %H:%M:%S.%f")
except Exception:
return {'status': 'disconnected'}
if (current_time - heartbeat_last).seconds > 10:
return {'status': 'disconnected'}
else:
return {'status': get_global('agent_status')}
def orchestrate(job):
job_id = job.job_id
agent = Agent(job_id=job_id, role='orchestrate')
job.set('start_time', time.time())
git_repo = job.get('git_repo')
file_name = job.get('file_name')
agent.log(f'orchestrating', report=True, job_id=job_id)
exec_agents = json.loads(
requests.get(f'http://{get_global("tracker_host")}:3000/assign_agents',
params={
'source': job.get('submitter_name'),
'orchestrator': get_global('agent_name'),
'required': 2
}).content.decode("ascii"))
job.set('executors', exec_agents)
agent.report_job(job_id, f'executors: {exec_agents}')
for exec_agent in exec_agents:
agent.log(f'sending to executor: {exec_agent["name"]}',
report=True,
job_id=job_id)
time.sleep(5)
try:
requests.get(
f'http://{exec_agent["url"]}:{exec_agent["port"]}/execute',
params={
'git_repo': git_repo,
'file_name': file_name,
'job_id': job_id,
'submission_time': job.get('submission_time'),
'submitter_name': job.get('submitter_name'),
'submitter_url': job.get('submitter_url'),
'submitter_port': job.get('submitter_port'),
'orchestrator_name': get_global('agent_name'),
'orchestrator_url': get_global('agent_url'),
'orchestrator_port': get_global('agent_port')
},
timeout=0.0000000001)
except requests.exceptions.ReadTimeout:
pass
time.sleep(3)
Thread(target=sync, kwargs={'job_id': job_id}).start()
set_global('agent_status', 'connected')
return f'sent job {job.job_id} to executors: {exec_agents}'
def status():
current_time = datetime.now()
agent_status = {'agent_name': get_global('agent_name')}
try:
heartbeat_last = datetime.strptime(get_global('heartbeat_last'),
"%Y-%m-%d %H:%M:%S.%f")
if (current_time - heartbeat_last).seconds > 10:
agent_status.update({'status': 'disconnected'})
else:
agent_status.update({'status': get_global('agent_status')})
except Exception:
agent_status.update({'status': 'disconnected'})
return agent_status
def submit(job):
agent = Agent(job_id=job.job_id, role='submit')
try:
agent.report_job(job.job_id, 'submitting')
# todo - add tracker object
orchestrator_agent = request_orchestrator(agent, 1, job_id=job.job_id)
job.set('assigned_agent', orchestrator_agent)
# agent.report(f'sending job: {job.job_id}, to orchestrator: {orchestrator_agent}', job_id=job.job_id)
agent.report_job(job.job_id, f'sending to orchestrator: {orchestrator_agent}')
submission_time = str(datetime.datetime.now())
# todo - make sure api call is not waiting for response, then have job.set_may after call
job_params = {
'job_status': 'submitted',
'submission_time': str(datetime.datetime.now()),
}
job.set_many(job_params)
# todo - this is an agent skill
# todo - handle async
try:
requests.get(f'http://{orchestrator_agent["url"]}:{orchestrator_agent["port"]}/orchestrate',
params={
'git_repo': job.get("git_repo"),
'file_name': job.get("file_name"),
'job_id': job.job_id,
'submission_time': submission_time,
'submitter_name': get_global('agent_name'),
'submitter_url': get_global('agent_url'),
'submitter_port': get_global('agent_port')
},
timeout=0.0000000001)
except requests.exceptions.ReadTimeout:
pass
agent.set('agent_status', 'connected')
except Exception as e:
agent.log(e)
return f'error submitting job {job.job_id}: {e}'
return {
'status': 'submitted',
'timestamp': submission_time,
'orchestrator': orchestrator_agent,
'job_id': job.job_id
}
def request_orchestrator(agent, required, job_id=None):
agent.log(f'requesting orchestrating agents', report=True, job_id=job_id)
orchestrator_agents = json.loads(requests.get(f'http://{get_global("tracker_host")}:3000/assign_agents',
params={'source': get_global('agent_name'),
'required': required}).content.decode("ascii"))[0]
agent.log(f'acquired orchestrating agents: {orchestrator_agents}', report=True, job_id=job_id)
return orchestrator_agents
def report_job(self, job_id, message):
requests.get(f'http://{get_global("tracker_host")}:3000/agent_log',
params={
'agent_name': get_global('agent_name'),
'role': self.role,
'job_id': job_id,
'agent_log': message
}
)
def __init__(self) -> None:
self.agent_name = get_global('agent_name')
self.agent_status = get_global('agent_status')
self.timestamp = str(datetime.now())
def get(key):
return get_global(key)
'agent_name':
os.environ['AGENT_NAME'],
'agent_url':
os.environ['AGENT_URL'],
'agent_port':
port,
'agent_status':
'connected',
'tracker_host':
os.environ['TRACKER_HOST'] if 'TRACKER_HOST' in os.environ else 'tracker'
}
db_global.update({'type': 'properties'}, global_entry, upsert=True)
try:
response = requests.get(
f'http://{get_global("tracker_host")}:3000/register_agent',
params={
'agent_name': get_global('agent_name'),
'agent_url': get_global('agent_url'),
'agent_port': get_global('agent_port')
})
except Exception as e:
logger().exception(f'unable to register agent: {e}')
CORS(app, resources={r'/*': {'origins': '*'}})
app.run(debug=True,
host='0.0.0.0',
port=port,
use_reloader=False,
threaded=True)