def set_project_domain(self, project_code, domain_key):
"""Modify the Domain of the Project whose code is given in parameter.
It raises HPCStatsRuntimeError if either the Project code or the
Domain key are not found in DB.
"""
domain = Domain(domain_key, None)
if not domain.existing(self.db):
raise HPCStatsRuntimeError( \
"unable to find domain %s in database" \
% (domain_key))
project = Project(None, project_code, None)
if not project.find(self.db):
raise HPCStatsRuntimeError( \
"unable to find project %s in database" \
% (project_code))
# Load the Project in DB to get its description
project.load(self.db)
project.domain = domain
logger.info("updating project %s with new domain %s", project_code,
domain_key)
project.update(self.db)
def save(self, db):
"""Insert FSUsage in database. It first makes sure that the FSUsage
does not already exist in database yet by calling FSUsage.existing()
method if needed. If the fsusage already exists in database, it
raises HPCStatsRuntimeError.
"""
if self.exists is None: # not checked yet
self.existing(db)
if self.exists is True:
raise HPCStatsRuntimeError(
"could not insert fsusage %s since already existing in "\
"database" % (str(self)))
req = """
INSERT INTO fsusage ( filesystem_id,
cluster_id,
fsusage_time,
fsusage_usage,
fsusage_inode )
VALUES ( %s, %s, %s, %s, %s )
"""
params = (self.filesystem.fs_id, self.filesystem.cluster.cluster_id,
self.timestamp, self.usage, self.inode)
#print db.cur.mogrify(req, params)
db.execute(req, params)
def save(self, db):
"""Insert Business in database. It first makes sure that the Business
does not already exist in database yet by calling
Business.existing() method if needed. If the business already exists
in database, it raises HPCStatsRuntimeError.
"""
if self.exists is None: # not checked yet
self.existing(db)
if self.exists is True:
raise HPCStatsRuntimeError(
"could not insert business %s since already existing in "\
"database" % (str(self)))
logger.info("creating business code %s" % str(self))
req = """
INSERT INTO Business ( business_code,
business_description )
VALUES (%s, %s)
"""
params = (self.code, self.description)
#print db.cur.mogrify(req, params)
db.execute(req, params)
self.exists = True
def get_nb_accounts(self, db, creation_date):
"""Returns the total of users on the cluster whose account have been
created defore date given in parameter.
"""
if self.cluster_id is None:
raise HPCStatsRuntimeError(
"could not search for data with cluster %s since not " \
"found in database" \
% (str(self)))
req = """
SELECT COUNT (userhpc_id)
FROM Userhpc,
Account
WHERE Account.userhpc_id = Userhpc.userhpc_id
AND Account.account_creation < %s
AND Account.cluster_id = %s
"""
params = (creation_date, self.cluster_id)
#print db.cur.mogrify(req, params)
db.execute(req, params)
return db.cur.fetchone()[0]
def save(self, db):
"""Insert Run in database. It first makes sure that the Run does not
already exist in database yet by calling Run.existing() method if
needed. If the Run already exists in database, it raises
HPCStatsRuntimeError.
"""
if self.exists is None: # not checked yet
self.existing(db)
if self.exists is True:
raise HPCStatsRuntimeError(
"could not insert run %s since already existing in "\
"database" % (str(self)))
req = """
INSERT INTO Run ( job_id,
node_id,
cluster_id )
VALUES (%s, %s, %s)
"""
params = (self.job.job_id, self.node.node_id, self.cluster.cluster_id)
#print db.cur.mogrify(req, params)
db.execute(req, params)
self.exists = True
def save(self, db):
"""Insert User in database. You must make sure that the User does
not already exist in database yet (typically using User.find()
method else there is a risk of future integrity errors because of
duplicated users. If user_id attribute is set, it raises
HPCStatsRuntimeError.
"""
if self.user_id is not None:
raise HPCStatsRuntimeError(
"could not insert user %s since already existing in "\
"database" \
% (str(self)))
req = """
INSERT INTO Userhpc ( userhpc_login,
userhpc_firstname,
userhpc_name,
userhpc_department)
VALUES ( %s, %s, %s, %s)
RETURNING userhpc_id
"""
params = (self.login, self.firstname, self.lastname, self.department)
#print db.cur.mogrify(req, params)
db.execute(req, params)
self.user_id = db.cur.fetchone()[0]
def save(self, db):
"""Insert Node in database. You must make sure that the Node does not
already exist in database yet (typically using Node.find() method
else there is a risk of future integrity errors because of
duplicated nodes. If node_id attribute is set, it raises
HPCStatsRuntimeError.
"""
if self.node_id is not None:
raise HPCStatsRuntimeError(
"could not insert node %s since already existing in "\
"database" \
% (str(self)))
req = """
INSERT INTO Node ( node_name,
cluster_id,
node_model,
node_partition,
node_nbCpu,
node_memory,
node_flops )
VALUES ( %s, %s, %s, %s, %s, %s, %s )
RETURNING node_id
"""
params = (self.name, self.cluster.cluster_id, self.model,
self.partition, self.cpu, self.memory, self.flops)
#print db.cur.mogrify(req, params)
db.execute(req, params)
self.node_id = db.cur.fetchone()[0]
def update(self, db):
"""Update Job sched_id, nbcpu, name, state, queue, account, department,
submission, start, end and walltime in database.
Raises HPCStatsRuntimeError if self.job_id is None.
"""
if self.job_id is None:
raise HPCStatsRuntimeError(
"could not update job %s since not found in database" \
% (str(self)))
req = """
UPDATE Job
SET job_sched_id = %s,
job_nbCpu = %s,
job_name = %s,
job_state = %s,
job_queue = %s,
job_account = %s,
job_department = %s,
job_submission = %s,
job_start = %s,
job_end = %s,
job_walltime = %s
WHERE job_id = %s
"""
params = (self.sched_id, self.nbcpu, self.name, self.state, self.queue,
self.job_acct, self.job_department, self.submission,
self.start, self.end, self.walltime, self.job_id)
#print db.cur.mogrify(req, params)
db.execute(req, params)
def load(self, db):
"""Load the Project based on the content of the DB and set objects
attributes accordingly. The project_id attribute must have been
set previously, typically by calling find() method. It raises
raises HPCStatsRuntimeError if the Project is not found in DB.
"""
if self.project_id is None:
raise HPCStatsRuntimeError(
"could not load project %s since not found in database" \
% (str(self)))
req = """
SELECT project_code,
project_description,
domain_id
FROM Project
WHERE project_id = %s
"""
params = ( self.project_id, )
db.execute(req, params)
# We know here there is only one result thanks to existing() method
result = db.cur.fetchone()
self.code = result[0]
self.description = result[1]
# build the Domain object with the key found in DB
domain_key = result[2]
self.domain = Domain(domain_key, None)
def save(self, db):
"""Insert Project in database. You must make sure that the Project does
not already exist in database yet (typically using Project.find()
method else there is a risk of future integrity errors because of
duplicated clusters. If project_id attribute is set, it raises
HPCStatsRuntimeError.
"""
if self.project_id is not None:
raise HPCStatsRuntimeError(
"could not insert project %s since already existing in "\
"database" \
% (str(self)))
req = """
INSERT INTO Project ( project_code,
project_description,
domain_id )
VALUES ( %s, %s, %s )
RETURNING project_id
"""
params = ( self.code,
self.description,
self.domain.key )
#print db.cur.mogrify(req, params)
db.execute(req, params)
self.project_id = db.cur.fetchone()[0]
def save(self, db):
"""Insert Account in database. It first makes sure that the Account
does not already exist in database yet by calling Account.existing()
method if needed. If the account already exists in database, it
raises HPCStatsRuntimeError.
"""
if self.exists is None: # not checked yet
self.existing(db)
if self.exists is True:
raise HPCStatsRuntimeError(
"could not insert account %s since already existing in "\
"database" % (str(self)))
req = """
INSERT INTO Account ( account_uid,
account_gid,
account_creation,
account_deletion,
userhpc_id,
cluster_id )
VALUES ( %s, %s, %s, %s, %s, %s )
"""
params = (self.uid, self.gid, self.creation_date, self.deletion_date,
self.user.user_id, self.cluster.cluster_id)
#print db.cur.mogrify(req, params)
db.execute(req, params)
self.exists = True
def update(self, db):
"""Update Account uid, gid, creation date and deletion date in
database. Raises HPCStatsRuntimeError is exists is False.
"""
if self.exists is None: # not checked yet
self.existing(db)
if self.exists is False:
raise HPCStatsRuntimeError(
"could not update account %s since not found in database" \
% (str(self)))
req = """
UPDATE Account
SET account_uid = %s,
account_gid = %s,
account_creation = %s,
account_deletion = %s
WHERE userhpc_id = %s
AND cluster_id = %s
"""
params = (self.uid, self.gid, self.creation_date, self.deletion_date,
self.user.user_id, self.cluster.cluster_id)
#print db.cur.mogrify(req, params)
db.execute(req, params)
def load(self, db):
"""Load the Account based on the content of the DB and set objects
attributes accordingly. It calls existing() method if needed and
raises HPCStatsRuntimeError if the Account is not found in DB.
"""
if self.exists is None: # not checked yet
self.existing(db)
if self.exists is False:
raise HPCStatsRuntimeError(
"could not update load %s since not found in database" \
% (str(self)))
req = """
SELECT account_uid,
account_gid,
account_creation,
account_deletion
FROM Account
WHERE userhpc_id = %s
AND cluster_id = %s
"""
params = (self.user.user_id, self.cluster.cluster_id)
db.execute(req, params)
# We know here there is only one result thanks to existing() method
result = db.cur.fetchone()
self.uid = result[0]
self.gid = result[1]
self.creation_date = result[2]
self.deletion_date = result[3]
def update(self, db):
"""Update the Event in DB. The event_id attribute must be set for the
Event, either by passing this id to __init__() or by calling
Event.find() method previously. If event_id attribute is not set, it
raises HPCStatsRuntimeError.
"""
if self.event_id is None:
raise HPCStatsRuntimeError(
"could not update event %s since not found in database" \
% (str(self)))
req = """
UPDATE Event
SET event_end = %s,
event_start = %s,
event_type = %s,
event_reason = %s,
event_nbCpu = %s
WHERE event_id = %s
AND node_id = %s
AND cluster_id = %s
"""
params = (self.end_datetime, self.start_datetime, self.event_type,
self.reason, self.nb_cpu, self.event_id, self.node.node_id,
self.cluster.cluster_id)
#print db.cur.mogrify(req, params)
db.execute(req, params)
def save(self, db):
"""Insert Event in database. You must make sure that the Event does not
already exist in database yet (typically using Event.find() method
else there is a risk of future integrity errors because of
duplicated events. If event_id attribute is set, it raises
HPCStatsRuntimeError.
"""
if self.event_id is not None:
raise HPCStatsRuntimeError(
"could not insert event %s since already existing in "\
"database" \
% (str(self)))
req = """
INSERT INTO Event (
node_id,
cluster_id,
event_type,
event_reason,
event_nbCpu,
event_start,
event_end )
VALUES ( %s, %s, %s, %s, %s, %s, %s)
RETURNING event_id
"""
params = (self.node.node_id, self.cluster.cluster_id, self.event_type,
self.reason, self.nb_cpu, self.start_datetime,
self.end_datetime)
#print db.cur.mogrify(req, params)
db.execute(req, params)
self.event_id = db.cur.fetchone()[0]
def save(self, db):
"""Insert Filesystem in database. You must make sure that the
Filesystem does not already exist in database yet (typically using
Filesystem.find() method else there is a risk of future integrity
errors because of duplicated filesystems. If fs_id attribute is set,
it raises HPCStatsRuntimeError.
"""
if self.fs_id is not None:
raise HPCStatsRuntimeError(
"could not insert filesystem %s since already existing " \
"in database" \
% (str(self)))
req = """
INSERT INTO filesystem ( filesystem_name,
cluster_id )
VALUES ( %s, %s)
RETURNING filesystem_id
"""
params = (self.mountpoint, self.cluster.cluster_id)
#print db.cur.mogrify(req, params)
db.execute(req, params)
self.fs_id = db.cur.fetchone()[0]
def save(self, db):
"""Insert Job in database. You must make sure that the Job does not
already exist in database yet (typically using Job.find() method
else there is a risk of future integrity errors because of
duplicated jobs. If job_id attribute is set, it raises
HPCStatsRuntimeError.
"""
if self.job_id is not None:
raise HPCStatsRuntimeError(
"could not insert job %s since already existing in "\
"database" \
% (str(self)))
req = """
INSERT INTO job ( job_sched_id,
job_batch_id,
job_name,
job_nbCpu,
job_state,
job_queue,
job_account,
job_department,
job_submission,
job_start,
job_end,
job_walltime,
cluster_id,
userhpc_id,
project_id,
business_code )
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
RETURNING job_id
"""
if self.project is None:
project_id = None
else:
project_id = self.project.project_id
if self.business is None:
business_code = None
else:
business_code = self.business.code
params = (self.sched_id, self.batch_id, self.name, self.nbcpu,
self.state, self.queue, self.job_acct, self.job_department,
self.submission, self.start, self.end, self.walltime,
self.account.cluster.cluster_id, self.account.user.user_id,
project_id, business_code)
#print db.cur.mogrify(req, params)
db.execute(req, params)
self.job_id = db.cur.fetchone()[0]
def create_domain(self, domain_key, domain_name):
"""Creates the domain whose key and name are given in parameters. It
raises HPCStatsRuntimeError if another Domain with the same key
already exists in DB.
"""
domain = Domain(domain_key, domain_name)
if domain.existing(self.db):
raise HPCStatsRuntimeError("domain %s already exists in database" %
(domain_key))
logger.info("creating domain %s in database", domain_key)
domain.save(self.db)
def bind(self):
""" Connection to the database """
conn_str = "host = %(dbhostname)s " \
"dbname= %(dbname)s " \
"user= %(dbuser)s " \
"password= %(dbpassword)s" \
% (self.database)
try:
self._conn = psycopg2.connect(conn_str)
except psycopg2.OperationalError, err:
raise HPCStatsRuntimeError( \
"Error while trying to connect on HPCStats DB: %s" \
% (err))
def import_cluster_data(self, db, cluster_name):
"""Import from sources all data specific to a cluster."""
cluster = None
#
# Architecture Importer is both responsible for importing/updating data
# about cluster and nodes in database and creating the Cluster object
# for other importers.
#
logger.info("updating architecture for cluster %s", cluster_name)
self.arch = \
ArchitectureImporterFactory.factory(self, db, self.conf,
cluster_name)
self.arch.load()
self.arch.update()
cluster = self.arch.cluster
# check that cluster has been properly created and initialized
if cluster is None or cluster.cluster_id is None:
raise HPCStatsRuntimeError("problem in DB with cluster %s" %
(str(cluster)))
logger.info("updating users for cluster %s", cluster.name)
self.users = \
UserImporterFactory.factory(self, db, self.conf, cluster)
self.users.load()
self.users.update()
logger.info("updating filesystem usage for cluster %s", cluster.name)
self.fsusage = \
FSUsageImporterFactory.factory(self, db, self.conf, cluster)
self.fsusage.load()
self.fsusage.update()
logger.info("updating filesystem quota for cluster %s", cluster.name)
self.fsquota = \
FSQuotaImporterFactory.factory(self, db, self.conf, cluster)
self.fsquota.load()
self.fsquota.update()
logger.info("updating events for cluster %s", cluster.name)
self.events = \
EventImporterFactory.factory(self, db, self.conf, cluster)
self.events.load()
self.events.update()
logger.info("updating jobs for cluster %s", cluster.name)
self.jobs = JobImporterFactory.factory(self, db, self.conf, cluster)
self.jobs.load_update_window()
def set_business_code_description(self, business_code, description):
"""Modify in DB the description of the Business code given in
parameter. It raises HPCStatsRuntimeError if the Business code
is not found in DB.
"""
business = Business(business_code, description)
if not business.existing(self.db):
raise HPCStatsRuntimeError( \
"unable to find business code %s in database" \
% (business_code))
logger.info("updating business code %s with new description",
business_code)
business.update(self.db)
def set_project_description(self, project_code, description):
"""Modify in DB the description of the Project given in parameter. It
raises HPCStatsRuntimeError if the Project is not found in DB.
"""
project = Project(None, project_code, None)
if not project.find(self.db):
raise HPCStatsRuntimeError( \
"unable to find project %s in database" \
% (project_code))
# Load the Project from DB to get its domain key
project.load(self.db)
project.description = description
logger.info("updating project %s with new description", project_code)
project.update(self.db)
def get_nb_cpus(self, db):
"""Returns the total number of CPUs available on the cluster"""
if self.cluster_id is None:
raise HPCStatsRuntimeError(
"could not search for data with cluster %s since not " \
"found in database" \
% (str(self)))
req = """
SELECT SUM(node_nbCpu)
FROM Node
WHERE cluster_id = %s
"""
params = (self.cluster_id, )
#print db.cur.mogrify(req, params)
db.execute(req, params)
return db.cur.fetchone()[0]
def update(self, db):
"""Update Domain name in database. Raises HPCStatsRuntimeError if
exists is False.
"""
if self.exists is None: # not checked yet
self.existing(db)
if self.exists is False:
raise HPCStatsRuntimeError(
"could not update domain %s since not found in database" \
% (str(self)))
req = """
UPDATE Domain
SET domain_name = %s
WHERE domain_id = %s
"""
params = (self.name, self.key)
#print db.cur.mogrify(req, params)
db.execute(req, params)
def update(self, db):
"""Update Project description field in database. Raises
HPCStatsRuntimeError if self.project_id is None.
"""
if self.project_id is None:
raise HPCStatsRuntimeError(
"could not update project %s since not found in database" \
% (str(self)))
req = """
UPDATE Project
SET project_description = %s,
domain_id = %s
WHERE project_id = %s
"""
params = ( self.description,
self.domain.key,
self.project_id )
#print db.cur.mogrify(req, params)
db.execute(req, params)
def update(self, db):
"""Update User firstname, lastname and department fields in database.
Raises HPCStatsRuntimeError is user_id is None.
"""
if self.user_id is None:
raise HPCStatsRuntimeError(
"could not update user %s since not found in database" \
% (str(self)))
req = """
UPDATE Userhpc
SET userhpc_firstname = %s,
userhpc_name = %s,
userhpc_department = %s
WHERE userhpc_id = %s
"""
params = (self.firstname, self.lastname, self.department, self.user_id)
#print db.cur.mogrify(req, params)
db.execute(req, params)
def get_min_datetime(self, db):
"""Returns the start datetime of the oldest started and unfinished
job on the cluster.
"""
if self.cluster_id is None:
raise HPCStatsRuntimeError(
"could not search for data with cluster %s since not " \
"found in database" \
% (str(self)))
req = """
SELECT MIN(job_start)
FROM Job
WHERE cluster_id = %s
AND job_state NOT IN ('CANCELLED', 'NODE_FAIL', 'PENDING')
"""
params = (self.cluster_id, )
#print db.cur.mogrify(req, params)
db.execute(req, params)
return db.cur.fetchone()[0]
def save(self, db):
"""Insert FSQuota in database. It first makes sure that the FSQuota
does not already exist in database yet by calling FSQuota.existing()
method if needed. If the fsquota already exists in database, it
raises HPCStatsRuntimeError.
"""
if self.exists is None: # not checked yet
self.existing(db)
if self.exists is True:
raise HPCStatsRuntimeError(
"could not insert fsquota %s since already existing in "\
"database" % (str(self)))
req = """
INSERT INTO fsquota ( filesystem_id,
cluster_id,
fsquota_time,
fsquota_name,
fsquota_block_KB,
fsquota_block_quota,
fsquota_block_limit,
fsquota_block_in_doubt,
fsquota_block_grace,
fsquota_file_files,
fsquota_file_quota,
fsquota_file_limit,
fsquota_file_in_doubt,
fsquota_file_grace )
VALUES ( %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s )
"""
params = (self.filesystem.fs_id, self.filesystem.cluster.cluster_id,
self.timestamp, self.name, self.block_KB, self.block_quota,
self.block_limit, self.block_in_doubt, self.block_grace,
self.file_files, self.file_quota, self.file_limit,
self.file_in_doubt, self.file_grace)
#print db.cur.mogrify(req, params)
db.execute(req, params)
def update(self, db):
"""Update Business description in database. Raises HPCStatsRuntimeError
if exists is False.
"""
if self.exists is None: # not checked yet
self.existing(db)
if self.exists is False:
raise HPCStatsRuntimeError(
"could not update business %s since not found in " \
"database" \
% (str(self)))
logger.debug("updating business code %s" % str(self))
req = """
UPDATE Business
SET business_description = %s
WHERE business_code = %s
"""
params = (self.description, self.code)
#print db.cur.mogrify(req, params)
db.execute(req, params)
def save(self, db):
"""Insert Domain in database. It first makes sure that the Domain does
not already exist in database yet by calling Domain.existing()
method if needed. If the domain already exists in database, it
raises HPCStatsRuntimeError.
"""
if self.exists is None: # not checked yet
self.existing(db)
if self.exists is True:
raise HPCStatsRuntimeError(
"could not insert domain %s since already existing in "\
"database" % (str(self)))
req = """
INSERT INTO Domain ( domain_id,
domain_name )
VALUES ( %s, %s )
"""
params = (self.key, self.name)
#print db.cur.mogrify(req, params)
db.execute(req, params)
self.exists = True