def update_engine_conf(self, engine_conf):
if self.database_name == None or self.collection_name == None:
logging.error('task list not selected.')
return
# end if
res = self.check_engine_pars(engine_conf)
if not(res['status']):
logging.error(str(res['errors']))
return
# end if
# update the database
db, db_client = utils.connect_mongoDB_server(self.db_pars)
coll = db_client[self.database_name][self.collection_name]
res = coll.update_one(
{'role': 'engine_conf'},
<<<<<<< HEAD
{'set' : engine_conf }
=======
{'$set': engine_conf },
upsert=True
>>>>>>> e009115e2b8660f291d8a3c8db71e01d39525aa5
) # end update one
return res
def deactivate_task_program(self, name):
db, db_client = utils.connect_mongoDB_server(self.db_pars)
response = db_client[self.database_name][self.collection_name].update_one(
{'role': 'task', 'name': name},
{'$set': {'status.status': 'inactive'}}
) # end update
return response
def load_task_lists():
# initiate database connection
db_pars_path = '/home/scube_backend/.keys/mongodb_pars.yaml' # note: hardcoded !
db_pars = yaml.load(open(db_pars_path))
db, db_client = utils.connect_mongoDB_server(db_pars)
# query scflex control databse
ans = db_client['Scflex_control']['task_list_monitoring'].find(
{"role": "task_list_info"})
ans = list(ans)
if len(ans) == 0: ""
# get task list name
task_list_names = [doc['db_name'] + '/' + doc['coll_name'] for doc in ans]
# make the html
soup = BeautifulSoup("", 'lxml')
for name in task_list_names:
new_tag = soup.new_tag('option')
new_tag.attrs['value'] = name
label = ' - '.join(name.split('/'))
new_tag.insert(0, label)
soup.insert(0, new_tag)
# end for
return str(soup)
def display_engine_pars(self):
db, db_client = utils.connect_mongoDB_server(self.db_pars)
ans = db_client[self.database_name][self.collection_name].find_one(
{'role': 'engine_conf'}
) # end cursor
db_client.close()
return ans
def get_task_statuses(db_name, coll_name):
# need stronger security check here -> ensure it's a task list
# initiate database connection
db_pars_path = '/home/scube_backend/.keys/mongodb_pars.yaml' # note: hardcoded !
db_pars = yaml.load(open(db_pars_path))
db, db_client = utils.connect_mongoDB_server(db_pars)
# get all tasks from database
cur_time = date_formatter(time.time())
tasks = list(db_client[db_name][coll_name].find({'role': 'task'}))
names = [_['name'] for _ in tasks]
tasks = [_['status'] for _ in tasks]
output = []
for i in range(len(tasks)):
task = deepcopy(tasks[i])
res = task.pop('last_response')
res.pop('status_code')
duration = res.pop('info')['duration']
# add extra information
task['name'] = names[i]
task['duration'] = duration
# combine with other task info
out = info = {'last_' + key: val for key, val in res.items()}
out.update(task)
output.append(out)
# end for
df = pd.DataFrame(output)
# date format
date_cols = ['last_failure', 'last_success', 'last_updated']
for _col in date_cols:
df[_col] = map(date_formatter, df[_col])
# end for
# organize columns
cols = [
'name', 'last_updated', 'last_success', 'last_failure', 'failure_n',
'last_status', 'last_output', 'last_error', 'status', 'priority_r'
]
# cheap fix for missing columns
for col in cols:
if col not in df.columns:
df[col] = 'N/A'
# end if
# end for
df = df[cols]
db_client.close()
return df
def add_task_program(self, name, program_pars, uuid_label = None, timeout = 300):
# 1. data integrity check
res = self.check_program_pars(program_pars)
if not(res['status']):
logging.error(str(res['errors']))
return
# end if
# connect to collection
db, db_client = utils.connect_mongoDB_server(self.db_pars)
# name must be unique within the task-list
logger.info('checking task list')
_ = db_client[self.database_name][self.collection_name].find_one({'name': name})
if _ != None:
msg = 'task "%s" already exists.' % (name)
return {'status': 'failed', 'errors': [msg]}
# end if
# make uuid label if it's not provided
if uuid_label == None:
uuid_label = str(uuid.uuid4())
# end if
# make document
_task = {
'name' : name,
'role' : 'task',
'uuid' : uuid_label,
'date_created': datetime.datetime.utcnow().isoformat(),
'historicals' : [],
'status': {
'status' : 'active',
'last_updated' : 0,
'last_success' : 0,
'last_failure' : 0,
'failure_n' : 0,
'priority_r' : 1.0,
'last_response': {
'status_code': -1,
'info': {
'duration': -1
}
}
},
'program_pars': program_pars
} # end _task
# insert doc
logger.info('inserting new task')
res = db_client[self.database_name][self.collection_name].insert(_task)
return res
def display_task_statuses(self, skip=0, limit=0):
db, db_client = utils.connect_mongoDB_server(self.db_pars)
cursor = db_client[self.database_name][self.collection_name].find(
{'role': 'task'}, {'name': True, 'status': True}
).skip(skip).limit(lmit)
records = list(cursor)
db_client.close()
# formatting a little
output = []
for rec in records:
status = rec.pop('status')
rec.update(status)
output.append(rec)
# end for
return output
def receive_cron_tasks():
db_pars = configs
db, db_client = utils.connect_mongoDB_server(db_pars)
content = flask.request.get_json()
# assign a uuid
for i in range(len(content)):
content[i]['uuid'] = uuid.uuid4()
# end for
# deactivate previous list
db_client['Scflex_control']['cronjob_list'].update_many({'role': 'cronjob_list'}, {'$set': {'status': 'inactive'}})
# insert new list
doc = new_doc(); doc['content'] = content
ans = db_client['Scflex_control']['cronjob_list'].insert_one(doc)
return ans
def init_task_list(self, database_name, collection_name, engine_conf): db, db_client = utils.connect_mongoDB_server(self.db_pars)
def delete_task_program(self, name):
db, db_client = utils.connect_mongoDB_server(self.db_pars)
response = db_client[self.database_name][self.collection_name].delete_one(
{'role': 'task', 'name': name},
) # end update
return response