def post(self):
args = parser.parse_args()
for worker_id in list_integers_string(args['id']):
worker = Worker.query.get(worker_id)
worker.status = args['status']
http_rest_request(worker.manager.host, '/workers/' + worker_id, 'patch', dict(status=worker.status))
return '', 204
def stop_task(task_ids):
"""Stop a list of tasks
"""
logging.info("Stopping tasks {0}".format(task_ids))
managers = {}
for task_id in task_ids:
task = Task.query.get(task_id)
manager = Manager.query.filter_by(id = task.manager_id).first()
if not manager.id in managers:
managers[manager.id] = []
if manager.has_virtual_workers == 0:
managers[manager.id].append(task_id)
for man in managers:
params = {'tasks': managers[man]}
try:
delete_task = http_rest_request(
manager.host,
'/tasks',
'delete',
params=params)
except:
logging.info("Error deleting task from Manager")
return
pass
task.status = 'ready'
db.session.add(task)
db.session.commit()
def stop_task(task_ids):
"""Stop a list of tasks
"""
logging.info("Stopping tasks {0}".format(task_ids))
managers = {}
for task_id in task_ids:
task = Task.query.get(task_id)
manager = Manager.query.filter_by(id=task.manager_id).first()
if not manager.id in managers:
managers[manager.id] = []
if manager.has_virtual_workers == 0:
managers[manager.id].append(task_id)
for man in managers:
params = {'tasks': managers[man]}
try:
delete_task = http_rest_request(manager.host,
'/tasks',
'delete',
params=params)
except:
logging.info("Error deleting task from Manager")
return
pass
task.status = 'waiting'
db.session.add(task)
db.session.commit()
def start_task(manager, task):
"""Execute a single task
We pass manager and task as objects (and at the moment we use a bad
way to get the additional job information - should be done with join)
"""
params = {
'priority': task.priority,
'type': task.type,
'parser': task.parser,
'task_id': task.id,
'job_id': task.job_id,
'settings': task.settings
}
task.status = 'active'
task.manager_id = manager.id
db.session.add(task)
db.session.commit()
r = http_rest_request(manager.host,
'/tasks/file/{0}'.format(task.job_id), 'get')
# print ('testing file')
# print (r)
if not r['file']:
job = Job.query.get(task.job_id)
serverstorage = app.config['SERVER_STORAGE']
projectpath = os.path.join(serverstorage, str(job.project_id))
jobpath = os.path.join(projectpath, str(job.id))
zippath = os.path.join(jobpath, "jobfile_{0}.zip".format(job.id))
try:
jobfile = [('jobfile', ('jobfile.zip', open(zippath, 'rb'),
'application/zip'))]
except IOError, e:
logging.error(e)
try:
# requests.post(serverurl, files = render_file , data = job_properties)
r = http_rest_request(manager.host,
'/tasks',
'post',
params,
files=jobfile)
except ConnectionError:
print("Connection Error: {0}".format(serverurl))
def get(self):
workers = {}
for manager in Manager.query.all():
print('Precessing manager workers: {}'.format(manager.host))
r = http_rest_request(manager.host, '/workers', 'get')
for worker in r.keys():
r[worker]['manager_id'] = manager.id
workers = dict(workers.items() + r.items())
return jsonify(workers)
def post(self):
args = parser.parse_args()
workers = []
pairs = args['id'].split(',')
for par in pairs:
int_list = par.split(';')
workers.append(map(int, int_list))
for worker_id, manager_id in workers:
manager = Manager.query.get(manager_id)
r = http_rest_request(manager.host,
'/workers/status/{0}'.format(worker_id),
'patch', dict(status=args['status']))
if r['task_id'] != None:
task_id = r['task_id']
http_rest_request(manager.host, '/tasks/{0}'.format(task_id),
'delete')
return '', 204
def start_task(manager, task):
"""Execute a single task
We pass manager and task as objects (and at the moment we use a bad
way to get the additional job information - should be done with join)
"""
params = {'priority':task.priority,
'type':task.type,
'parser':task.parser,
'task_id':task.id,
'job_id':task.job_id,
'settings':task.settings}
task.status = 'active'
task.manager_id = manager.id
db.session.add(task)
db.session.commit()
r = http_rest_request(
manager.host,
'/tasks/file/{0}'.format(task.job_id),
'get')
# print ('testing file')
# print (r)
if not r['file']:
job = Job.query.get(task.job_id)
serverstorage = app.config['STORAGE_SERVER']
projectpath = os.path.join(serverstorage, str(job.project_id))
jobpath = os.path.join(projectpath, str(job.id))
zippath = os.path.join(jobpath, "jobfile_{0}.zip".format(job.id))
try:
jobfile = [('jobfile',
('jobfile.zip', open(zippath, 'rb'), 'application/zip'))]
except IOError, e:
logging.error(e)
try:
# requests.post(serverurl, files = render_file , data = job_properties)
r = http_rest_request(manager.host, '/tasks', 'post', params, files=jobfile)
except ConnectionError:
print ("Connection Error: {0}".format(serverurl))
def get(self):
workers = {}
for manager in Manager.query.all():
try:
r = http_rest_request(manager.host, '/workers', 'get')
for worker in r.keys():
r[worker]['manager_id'] = manager.id
workers = dict(workers.items() + r.items())
except:
# TODO add proper exception handling!
pass
return jsonify(workers)
def get(self):
workers={}
# TODO should use db
#manager_db = Manager.query.all()
manager_db = app.config['MANAGERS']
for manager in manager_db:
try:
r = http_rest_request(manager.host, '/workers', 'get')
workers = dict(workers.items() + r.items())
except:
pass
return jsonify(workers)
def get(self):
workers = {}
for manager in Manager.query.all():
if manager.has_virtual_workers:
continue
try:
r = http_rest_request(manager.host, '/workers', 'get')
for worker in r.keys():
r[worker]['manager_id'] = manager.id
workers = dict(workers.items() + r.items())
except:
# TODO add proper exception handling!
pass
return jsonify(workers)
def post(self):
args = parser.parse_args()
workers = []
pairs = args['id'].split(',')
for par in pairs:
int_list = par.split(';')
workers.append( map(int, int_list) )
for worker_id,manager_id in workers:
manager = Manager.query.get(manager_id)
if not manager.has_virtual_workers:
r = http_rest_request(manager.host, '/workers/status/{0}'.format(worker_id), 'patch', dict(status=args['status']))
return '', 204
def post(self):
args = parser.parse_args()
workers = []
pairs = args['id'].split(',')
for par in pairs:
int_list = par.split(';')
workers.append(map(int, int_list))
for worker_id, manager_id in workers:
manager = Manager.query.get(manager_id)
if not manager.has_virtual_workers:
r = http_rest_request(manager.host,
'/workers/status/{0}'.format(worker_id),
'patch', dict(status=args['status']))
return '', 204
def stop_task(task_id):
"""Stop a single task
"""
print('Stoping task %s' % task_id)
task = Task.query.get(task_id)
manager = Manager.query.filter_by(id=task.manager_id).first()
try:
delete_task = http_rest_request(manager.host,
'/tasks/' + str(task.id), 'delete')
except:
logging.info("Error deleting task from Manager")
return
pass
task.status = 'ready'
db.session.add(task)
db.session.commit()
print "Task %d stopped" % task_id
serverstorage = app.config['SERVER_STORAGE']
projectpath = os.path.join(serverstorage, str(job.project_id))
jobpath = os.path.join(projectpath, str(job.id))
zippath = os.path.join(jobpath, "jobfile_{0}.zip".format(job.id))
try:
jobfile = [('jobfile', ('jobfile.zip', open(zippath, 'rb'), 'application/zip'))]
except IOError, e:
logging.error(e)
try:
# requests.post(serverurl, files = render_file , data = job_properties)
r = http_rest_request(manager.host, '/tasks', 'post', params, files=jobfile)
except ConnectionError:
print ("Connection Error: {0}".format(serverurl))
else:
r = http_rest_request(manager.host, '/tasks', 'post', params)
if not u'status' in r:
task.status = 'running'
task.manager_id = manager.id
db.session.add(task)
db.session.commit()
# TODO get a reply from the worker (running, error, etc)
@staticmethod
def delete_task(task_id):
# At the moment this function is not used anywhere
try:
task = Tasks.query.get(task_id)
except Exception, e:
jobfile = [('jobfile', ('jobfile.zip', open(zippath, 'rb'),
'application/zip'))]
except IOError, e:
logging.error(e)
try:
# requests.post(serverurl, files = render_file , data = job_properties)
r = http_rest_request(manager.host,
'/tasks',
'post',
params,
files=jobfile)
except ConnectionError:
print("Connection Error: {0}".format(serverurl))
else:
r = http_rest_request(manager.host, '/tasks', 'post', params)
if not u'status' in r:
task.status = 'active'
task.manager_id = manager.id
db.session.add(task)
db.session.commit()
# TODO get a reply from the worker (running, error, etc)
@staticmethod
def delete_task(task_id):
# At the moment this function is not used anywhere
try:
task = Tasks.query.get(task_id)
except Exception, e:
print(e)