def __enter__(self):
if self.batch_system == "XDMoD":
self.dbac = XDMoDArchiveCache(self.config)
else:
self.dbac = DbArchiveCache(self.config)
self.paths_file = tempfile.NamedTemporaryFile('wb',
delete=not self.keep_csv,
suffix=".csv",
prefix="archive_paths")
self.paths_csv = csv.writer(self.paths_file,
lineterminator="\n",
quoting=csv.QUOTE_MINIMAL,
escapechar='\\')
self.joblevel_file = tempfile.NamedTemporaryFile(
'wb',
delete=not self.keep_csv,
suffix=".csv",
prefix="archives_joblevel")
self.joblevel_csv = csv.writer(self.joblevel_file,
lineterminator="\n",
quoting=csv.QUOTE_MINIMAL,
escapechar='\\')
self.nodelevel_file = tempfile.NamedTemporaryFile(
'wb',
delete=not self.keep_csv,
suffix=".csv",
prefix="archives_nodelevel")
self.nodelevel_csv = csv.writer(self.nodelevel_file,
lineterminator="\n",
quoting=csv.QUOTE_MINIMAL,
escapechar='\\')
return self
def archive_cache_factory(resconf, config):
""" Return the implementation of the accounting class for the resource """
atype = resconf['batch_system']
if atype == "XDMoD":
return XDMoDArchiveCache(config)
else:
return DbArchiveCache(config)
class LoadFileIndexUpdater(object):
def __init__(self, config, resconf, keep_csv=False, dry_run=False):
self.config = config
self.resource_id = resconf["resource_id"]
self.batch_system = resconf['batch_system']
self.keep_csv = keep_csv
self.dry_run = dry_run
def __enter__(self):
if self.batch_system == "XDMoD":
self.dbac = XDMoDArchiveCache(self.config)
else:
self.dbac = DbArchiveCache(self.config)
self.paths_file = tempfile.NamedTemporaryFile('wb', delete=not self.keep_csv, suffix=".csv", prefix="archive_paths")
self.paths_csv = csv.writer(self.paths_file, lineterminator="\n", quoting=csv.QUOTE_MINIMAL, escapechar='\\')
self.joblevel_file = tempfile.NamedTemporaryFile('wb', delete=not self.keep_csv, suffix=".csv", prefix="archives_joblevel")
self.joblevel_csv = csv.writer(self.joblevel_file, lineterminator="\n", quoting=csv.QUOTE_MINIMAL, escapechar='\\')
self.nodelevel_file = tempfile.NamedTemporaryFile('wb', delete=not self.keep_csv, suffix=".csv", prefix="archives_nodelevel")
self.nodelevel_csv = csv.writer(self.nodelevel_file, lineterminator="\n", quoting=csv.QUOTE_MINIMAL, escapechar='\\')
return self
def __exit__(self, exc_type, exc_val, exc_tb):
if self.keep_csv:
logging.info(self.paths_file.name)
logging.info(self.joblevel_file.name)
logging.info(self.nodelevel_file.name)
self.paths_file.file.flush()
self.joblevel_file.file.flush()
self.nodelevel_file.file.flush()
if not self.dry_run:
self.dbac.insert_from_files(self.paths_file.name, self.joblevel_file.name, self.nodelevel_file.name)
self.paths_file.close()
self.joblevel_file.close()
self.nodelevel_file.close()
def insert(self, hostname, archive_path, start_timestamp, end_timestamp, jobid):
self.paths_csv.writerow((archive_path,))
if jobid is not None:
self.joblevel_csv.writerow((archive_path, hostname, jobid[0], jobid[1], jobid[2], int(math.floor(start_timestamp)), int(math.ceil(end_timestamp))))
else:
self.nodelevel_csv.writerow((archive_path, hostname, int(math.floor(start_timestamp)), int(math.ceil(end_timestamp))))
def __exit__(self, exc_type, exc_val, exc_tb):
if self.keep_csv:
logging.info(self.paths_file.name)
logging.info(self.joblevel_file.name)
logging.info(self.nodelevel_file.name)
self.paths_file.file.flush()
self.joblevel_file.file.flush()
self.nodelevel_file.file.flush()
if not self.dry_run:
if self.batch_system == "XDMoD":
dbac = XDMoDArchiveCache(self.config)
else:
dbac = DbArchiveCache(self.config)
dbac.insert_from_files(self.paths_file.name, self.joblevel_file.name, self.nodelevel_file.name)
self.paths_file.close()
self.joblevel_file.close()
self.nodelevel_file.close()