def set_business_code_description(self, business_code, description):
"""Modify in DB the description of the Business code given in
parameter. It raises HPCStatsRuntimeError if the Business code
is not found in DB.
"""
business = Business(business_code, description)
if not business.existing(self.db):
raise HPCStatsRuntimeError( \
"unable to find business code %s in database" \
% (business_code))
logger.info("updating business code %s with new description",
business_code)
business.update(self.db)
def set_business_code_description(self, business_code, description):
"""Modify in DB the description of the Business code given in
parameter. It raises HPCStatsRuntimeError if the Business code
is not found in DB.
"""
business = Business(business_code, description)
if not business.existing(self.db):
raise HPCStatsRuntimeError( \
"unable to find business code %s in database" \
% (business_code))
logger.info("updating business code %s with new description",
business_code)
business.update(self.db)
def load(self):
"""Load BusinessCodes from CSV files in businesses attribute. Raise
HPCStatsSourceError if error in encountered.
"""
self.businesses = []
self.check()
with open(self._business_file, 'r') as csvfile:
file_reader = csv.reader(csvfile, delimiter=';', quotechar='|')
for row in file_reader:
if len(row) != 2:
raise HPCStatsSourceError( \
"business line format in CSV is invalid")
code = row[0].strip()
description = row[1].strip()
if len(code) == 0:
raise HPCStatsSourceError( \
"business code in CSV is empty")
if len(description) == 0:
description = None
business = Business(code, description)
self.businesses.append(business)
def test_update_exists(self):
"""ProjectImporterCSV.update() works when business code exists
"""
business1 = Business('code1', 'business description 1')
MockPg2.PG_REQS['existing_business'].set_assoc(
params=(business1.code, ), result=[['code1']])
self.importer.businesses = [business1]
self.importer.update()
def test_update_exists_with_mock(self, mock_update):
"""ProjectImporterCSV.update() call Business.update() when business
code exists
"""
business1 = Business('code1', 'business description 1')
MockPg2.PG_REQS['existing_business'].set_assoc(
params=(business1.code, ), result=[['code1']])
self.importer.businesses = [business1]
self.importer.update()
mock_update.assert_called_with(self.db)
def load_cluster(self, cluster):
"""Connect to cluster Slurm database to extract business codes from
jobs wckeys. Raises HPCStatsSourceError in case of error.
"""
self.log.debug("loading business codes from %s slurm database",
cluster)
self.connect_db(cluster)
if not len(self.clusters_db[cluster]['partitions']):
partitions_clause = ''
else:
partitions_clause = \
"WHERE job.partition IN (%s)" % \
','.join(['%s'] * len(self.clusters_db[cluster]['partitions']))
req = """
SELECT DISTINCT(wckey)
FROM %s_job_table job
%s
""" % (self.clusters_db[cluster]['prefix'], partitions_clause)
params = tuple(self.clusters_db[cluster]['partitions'])
self.cur.execute(req, params)
while (1):
row = self.cur.fetchone()
if row == None:
break
wckey = row[0]
if wckey == '':
continue
else:
wckey_items = wckey.split(':')
if len(wckey_items) != 2:
if wckey not in self.invalid_wckeys:
self.invalid_wckeys.append(wckey)
self.log.warn(Errors.E_B0001,
"format of wckey %s is not valid", wckey)
continue
else:
business_code = wckey_items[1]
business = Business(code=business_code, description=None)
# check for duplicate project
if not self.find(business):
self.businesses.append(business)
def __init__(self, db, config, cluster_name):
self._db = db
self._cluster_name = cluster_name
business_section = self._cluster_name + "/business"
self._business_file = config.get(business_section, "file")
if not os.path.isfile(self._business_file):
logging.error("business file %s does not exist", self._business_file)
raise RuntimeError
#In first delete all entries in business table and his dependances
logging.debug("Delete all business entries in db")
delete_contexts_with_business(self._db)
delete_business(self._db)
self._db.commit()
b_file = open(self._business_file, 'r')
# savepoint is used to considere exceptions and commit only at the end
db.get_cur().execute("SAVEPOINT my_savepoint;")
with b_file as csvfile:
file_reader = csv.reader(csvfile, delimiter=';', quotechar='|')
for row in file_reader:
code = Business( key = row[0],
description = row[1])
try:
if not code.already_exist(self._db):
code.save(self._db)
if not code.get_description():
logging.debug("add new business entry with key : %s, without description", code.get_key())
else:
logging.debug("add new business entry with key : %s, and description : %s", \
code.get_key(), \
code.get_description())
db.get_cur().execute("SAVEPOINT my_savepoint;")
else :
logging.debug("code %s not added, already exist", code.get_key())
except psycopg2.DataError:
logging.error("impossible to add BUSINESS entry in database : (%s) du to encoding error", row)
db.get_cur().execute("ROLLBACK TO SAVEPOINT my_savepoint;")
pass
self._db.commit()
b_file.close()
def get_jobs_after_batchid(self, batchid, window_size=0):
"""Fill the jobs attribute with the list of Jobs found in Slurm DB
whose id_job is over or equals to the batchid in parameter.
Returns the last found batch_id.
"""
self.jobs = []
if window_size:
limit = "LIMIT %d" % (window_size)
else:
limit = ''
last_batch_id = -1
old_schema = self._is_old_schema()
if old_schema is True:
cpu_field = 'cpus_alloc'
else:
cpu_field = 'tres_alloc'
if not len(self.partitions):
partitions_clause = ''
else:
partitions_clause = "AND job.partition IN (%s)" % \
','.join(['%s'] * len(self.partitions))
req = """
SELECT job_db_inx,
id_job,
id_user,
id_group,
time_submit,
time_start,
time_end,
timelimit,
nodes_alloc,
%s,
job.partition,
qos.name AS qos,
job.account,
state,
nodelist,
assoc.user,
job_name,
wckey
FROM %s_job_table job,
%s_assoc_table assoc,
qos_table qos
WHERE job_db_inx >= %%s
%s
AND assoc.id_assoc = job.id_assoc
AND qos.id = job.id_qos
ORDER BY job_db_inx %s
""" % (cpu_field, self.prefix, self.prefix, partitions_clause,
limit)
params = (batchid, ) + tuple(self.partitions)
self.cur.execute(req, params)
while (1):
row = self.cur.fetchone()
if row == None:
break
self.nb_loaded_jobs += 1
batch_id = last_batch_id = row[0]
sched_id = row[1]
submission_t = row[4]
if submission_t == 0:
submission = None
else:
submission = datetime.fromtimestamp(submission_t)
start_t = row[5]
if start_t == 0:
start = None
else:
start = datetime.fromtimestamp(start_t)
end_t = row[6]
if end_t == 0:
end = None
else:
end = datetime.fromtimestamp(end_t)
# Some jobs in Slurm DBD have an end but no start. Typically, this
# concernes the jobs that have been cancelled before starting. For
# these jobs, we set the start equal to the end.
if start is None and end is not None:
start = end
wall_t = row[7]
if wall_t == 0:
walltime = None
elif wall_t >= 2147483648:
walltime = "-1"
else:
walltime = str(wall_t)
name = row[16]
if old_schema is True:
nbcpu = row[9]
else:
nbcpu = extract_tres_cpu(row[9])
if nbcpu == -1:
raise HPCStatsSourceError( \
"unable to extract cpus_alloc from job tres")
state = JobImporterSlurm.get_job_state_from_slurm_state(row[13])
nodelist = row[14]
if nodelist == "(null)" or nodelist == "None assigned":
nodelist = None
partition = self.job_partition(sched_id, row[10], nodelist)
qos = row[11]
queue = "%s-%s" % (partition, qos)
job_acct = row[12]
login = row[15]
searched_user = User(login, None, None, None)
searched_account = Account(searched_user, self.cluster, None, None,
None, None)
account = self.app.users.find_account(searched_account)
if account is None:
msg = "account %s not found in loaded accounts" \
% (login)
if self.strict_job_account_binding == True:
raise HPCStatsSourceError(msg)
elif login not in self.unknown_accounts:
self.unknown_accounts.append(login)
self.log.warn(Errors.E_J0001, msg)
self.nb_excluded_jobs += 1
continue
user = self.app.users.find_user(searched_user)
if user is None:
msg = "user %s not found in loaded users" % (login)
raise HPCStatsSourceError(msg)
job_department = user.department
wckey = row[17]
# empty wckey must be considered as None
if wckey == '':
wckey = None
if wckey is None:
project = None
business = None
else:
wckey_items = wckey.split(':')
if len(wckey_items) != 2:
msg = "format of wckey %s is not valid" % (wckey)
if self.strict_job_wckey_format == True:
raise HPCStatsSourceError(msg)
elif wckey not in self.invalid_wckeys:
self.invalid_wckeys.append(wckey)
self.log.warn(Errors.E_J0002, msg)
project = None
business = None
else:
project_code = wckey_items[0]
searched_project = Project(None, project_code, None)
project = self.app.projects.find_project(searched_project)
if project is None:
msg = "project %s not found in loaded projects" \
% (project_code)
if self.strict_job_project_binding == True:
raise HPCStatsSourceError(msg)
elif project_code not in self.unknown_projects:
self.unknown_projects.append(project_code)
self.log.warn(Errors.E_J0003, msg)
business_code = wckey_items[1]
searched_business = Business(business_code, None)
business = self.app.business.find(searched_business)
if business is None:
msg = "business code %s not found in loaded " \
"business codes" % (business_code)
if self.strict_job_businesscode_binding == True:
raise HPCStatsSourceError(msg)
elif business_code not in self.unknown_businesses:
self.unknown_businesses.append(business_code)
self.log.warn(Errors.E_J0004, msg)
job = Job(account, project, business, sched_id, str(batch_id),
name, nbcpu, state, queue, job_acct, job_department,
submission, start, end, walltime)
self.jobs.append(job)
if nodelist is not None:
self.create_runs(nodelist, job)
return last_batch_id
def __init__(self, db, config, cluster_name):
self._db = db
self._cluster_name = cluster_name
context_section = self._cluster_name + "/context"
self._context_file = config.get(context_section, "file")
if not os.path.isfile(self._context_file):
logging.error("context file %s does not exist", self._context_file)
raise RuntimeError
# delete all contexts entries in databases
logging.debug("Delete all context entries in db")
delete_contexts(self._db)
self._db.commit()
p_file = open(self._context_file, 'r')
# save point is used to considere exception and commit in database only at the end
db.get_cur().execute("SAVEPOINT my_savepoint;")
with p_file as csvfile:
file_reader = csv.reader(csvfile, delimiter=',', quotechar='|')
for row in file_reader:
logging.debug("update projects and business codes for user : %s", row[0].lower())
# a new context is set in database for all projects attached AND for all business attached.
# new line is set with a project referance OR a business referance.
if row[6]:
for pareo in re.split('\|',row[6]):
project = Project()
try:
project.project_from_pareo(self._db, pareo)
context = Context(login = row[0].lower(),
job = None,
project = project.get_id(),
business = None)
try:
context.save(self._db)
logging.debug("add context : %s", context)
#self._db.commit()
db.get_cur().execute("SAVEPOINT my_savepoint;")
except psycopg2.DataError:
logging.error("impossible to add CONTEXT entry in database : (%s), du to encoding error", row)
db.get_cur().execute("ROLLBACK TO SAVEPOINT my_savepoint;")
pass
except psycopg2.IntegrityError:
logging.error("impossible to add CONTEXT entry in database : (%s), du to relations error", row)
db.get_cur().execute("ROLLBACK TO SAVEPOINT my_savepoint;")
pass
except:
logging.error("context rejected. Project %s does not exist", pareo)
db.get_cur().execute("ROLLBACK TO SAVEPOINT my_savepoint;")
pass
if row[7]:
for code in re.split('\|',row[7]):
business = Business()
try:
business.business_from_key(self._db, code)
context = Context(login = row[0].lower(),
job = None,
project = None,
business = business.get_id())
try:
context.save(self._db)
logging.debug("add context : %s", context)
#self._db.commit()
db.get_cur().execute("SAVEPOINT my_savepoint;")
except psycopg2.DataError:
logging.error("impossible to add CONTEXT entry in database : (%s), du to encoding error", row)
db.get_cur().execute("ROLLBACK TO SAVEPOINT my_savepoint;")
pass
except psycopg2.IntegrityError:
logging.error("impossible to add CONTEXT entry in database : (%s), du to relations error", row)
db.get_cur().execute("ROLLBACK TO SAVEPOINT my_savepoint;")
pass
except:
logging.error("context rejected. Business %s does not exist", code)
db.get_cur().execute("ROLLBACK TO SAVEPOINT my_savepoint;")
pass
if not row[6] and not row[7]:
logging.error("line : %s rejected - not code or project associate", row)
self._db.commit()
p_file.close()
def test_update_not_exists(self):
"""ProjectImporterCSV.update() works when business code does not exist
"""
business1 = Business('code1', 'business description 1')
self.importer.businesses = [business1]
self.importer.update()