def __enter__(self): if self.batch_system == "XDMoD": self.dbac = XDMoDArchiveCache(self.config) else: self.dbac = DbArchiveCache(self.config) self.paths_file = tempfile.NamedTemporaryFile('wb', delete=not self.keep_csv, suffix=".csv", prefix="archive_paths") self.paths_csv = csv.writer(self.paths_file, lineterminator="\n", quoting=csv.QUOTE_MINIMAL, escapechar='\\') self.joblevel_file = tempfile.NamedTemporaryFile( 'wb', delete=not self.keep_csv, suffix=".csv", prefix="archives_joblevel") self.joblevel_csv = csv.writer(self.joblevel_file, lineterminator="\n", quoting=csv.QUOTE_MINIMAL, escapechar='\\') self.nodelevel_file = tempfile.NamedTemporaryFile( 'wb', delete=not self.keep_csv, suffix=".csv", prefix="archives_nodelevel") self.nodelevel_csv = csv.writer(self.nodelevel_file, lineterminator="\n", quoting=csv.QUOTE_MINIMAL, escapechar='\\') return self
def archive_cache_factory(resconf, config): """ Return the implementation of the accounting class for the resource """ atype = resconf['batch_system'] if atype == "XDMoD": return XDMoDArchiveCache(config) else: return DbArchiveCache(config)
class LoadFileIndexUpdater(object): def __init__(self, config, resconf, keep_csv=False, dry_run=False): self.config = config self.resource_id = resconf["resource_id"] self.batch_system = resconf['batch_system'] self.keep_csv = keep_csv self.dry_run = dry_run def __enter__(self): if self.batch_system == "XDMoD": self.dbac = XDMoDArchiveCache(self.config) else: self.dbac = DbArchiveCache(self.config) self.paths_file = tempfile.NamedTemporaryFile('wb', delete=not self.keep_csv, suffix=".csv", prefix="archive_paths") self.paths_csv = csv.writer(self.paths_file, lineterminator="\n", quoting=csv.QUOTE_MINIMAL, escapechar='\\') self.joblevel_file = tempfile.NamedTemporaryFile('wb', delete=not self.keep_csv, suffix=".csv", prefix="archives_joblevel") self.joblevel_csv = csv.writer(self.joblevel_file, lineterminator="\n", quoting=csv.QUOTE_MINIMAL, escapechar='\\') self.nodelevel_file = tempfile.NamedTemporaryFile('wb', delete=not self.keep_csv, suffix=".csv", prefix="archives_nodelevel") self.nodelevel_csv = csv.writer(self.nodelevel_file, lineterminator="\n", quoting=csv.QUOTE_MINIMAL, escapechar='\\') return self def __exit__(self, exc_type, exc_val, exc_tb): if self.keep_csv: logging.info(self.paths_file.name) logging.info(self.joblevel_file.name) logging.info(self.nodelevel_file.name) self.paths_file.file.flush() self.joblevel_file.file.flush() self.nodelevel_file.file.flush() if not self.dry_run: self.dbac.insert_from_files(self.paths_file.name, self.joblevel_file.name, self.nodelevel_file.name) self.paths_file.close() self.joblevel_file.close() self.nodelevel_file.close() def insert(self, hostname, archive_path, start_timestamp, end_timestamp, jobid): self.paths_csv.writerow((archive_path,)) if jobid is not None: self.joblevel_csv.writerow((archive_path, hostname, jobid[0], jobid[1], jobid[2], int(math.floor(start_timestamp)), int(math.ceil(end_timestamp)))) else: self.nodelevel_csv.writerow((archive_path, hostname, int(math.floor(start_timestamp)), int(math.ceil(end_timestamp))))
def __exit__(self, exc_type, exc_val, exc_tb): if self.keep_csv: logging.info(self.paths_file.name) logging.info(self.joblevel_file.name) logging.info(self.nodelevel_file.name) self.paths_file.file.flush() self.joblevel_file.file.flush() self.nodelevel_file.file.flush() if not self.dry_run: if self.batch_system == "XDMoD": dbac = XDMoDArchiveCache(self.config) else: dbac = DbArchiveCache(self.config) dbac.insert_from_files(self.paths_file.name, self.joblevel_file.name, self.nodelevel_file.name) self.paths_file.close() self.joblevel_file.close() self.nodelevel_file.close()