def create_id_dir(id_dir: Path, group: str, umask: int) -> (int, Path):
"""In the given directory, create the lowest numbered (positive integer)
directory that doesn't already exist.
:param id_dir: Path to the directory that contains these 'id'
directories
:param group: The group owner for this path.
:param umask: The umask to apply to this path.
:returns: The id and path to the created directory.
:raises OSError: on directory creation failure.
:raises TimeoutError: If we couldn't get the lock in time.
"""
lockfile_path = id_dir/'.lockfile'
with lockfile.LockFile(lockfile_path, timeout=1):
next_fn = id_dir/PKEY_FN
next_valid = True
if next_fn.exists():
try:
with next_fn.open() as next_file:
next_id = int(next_file.read())
next_id_path = make_id_path(id_dir, next_id)
if next_id_path.exists():
next_valid = False
except (OSError, ValueError):
# In either case, on failure, invalidate the next file.
next_valid = False
else:
next_valid = False
if not next_valid:
# If the next file's id wasn't valid, then find the next available
# id directory the hard way.
ids = list(os.listdir(str(id_dir)))
# Only return the test directories that could be integers.
ids = [id_ for id_ in ids if id_.isdigit()]
ids = [int(id_) for id_ in ids]
ids.sort()
# Find the first unused id.
next_id = 1
while next_id in ids:
next_id += 1
next_id_path = make_id_path(id_dir, next_id)
with permissions.PermissionsManager(next_id_path, group, umask), \
permissions.PermissionsManager(next_fn, group, umask):
next_id_path.mkdir()
with next_fn.open('w') as next_file:
next_file.write(str(next_id + 1))
return next_id, next_id_path
def index(id_dir: Path, idx_name: str,
transform: Callable[[Path], Dict[str, Any]],
complete_key: str = 'complete',
refresh_period: int = 1,
remove_missing: bool = False,
verbose: IO[str] = None,
fn_base: int = 10) -> Index:
"""Load and/or update an index of the given directory for the given
transform, and return it. The returned index is a dictionary by id of
the transformed data.
:param id_dir: The directory to index.
:param idx_name: The name of the index.
:param transform: A transformation function that produces a json
compatible dictionary.
:param complete_key: The key in the transformed dictionary that marks a
record as complete. If not given, the record is always assumed to be
complete. Incomplete records are recompiled every time the index is
updated (hopefully they will be complete eventually).
:param refresh_period: Only update the index if this much time (in seconds)
has passed since the last update.
:param remove_missing: Remove items that no longer exist from the index.
:param fn_base: The integer base for dir_db.
"""
idx_path = (id_dir/idx_name).with_suffix('.db')
idx = Index({})
idx_db = None
# Open and read the index if it exists. Any errors cause the index to
# regenerate from scratch.
idx_mtime = 0
if idx_path.exists():
try:
idx_db = dbm.open(idx_path.as_posix())
idx_mtime = idx_path.stat().st_mtime
idx = Index({int(k): json.loads(idx_db[k]) for k in idx_db.keys()})
except (OSError, PermissionError, json.JSONDecodeError) as err:
# In either error case, start from scratch.
LOGGER.warning(
"Error reading index at '%s'. Regenerating from "
"scratch. %s", idx_path.as_posix(), err.args[0])
new_items = {}
# If the index hasn't been updated lately (or is empty) do so.
# Small updates should happen unnoticeably fast, while full generation
# will take a bit.
if not idx or time.time() - idx_mtime > refresh_period:
seen = []
if verbose:
if idx_db is None:
output.fprint(
"No index file found for '{}'. This may take a while."
.format(id_dir.name),
file=verbose)
last_perc = None
progress = 0
files = list(os.scandir(id_dir.as_posix()))
for file in files:
try:
id_ = int(file.name, fn_base)
except ValueError:
continue
seen.append(id_)
progress += 1
complete_perc = int(100*progress/len(files))
if verbose and complete_perc != last_perc:
output.fprint(
"Indexing: {}%".format(complete_perc),
end='\r', file=verbose)
last_perc = complete_perc
# Skip entries that are known and complete.
if id_ in idx and idx[id_].get(complete_key, False):
continue
# Only directories with integer names are db entries.
if not file.is_dir():
continue
# Update incomplete or unknown entries.
try:
new = transform(Path(file.path))
except ValueError:
continue
old = idx.get(id_)
if new != old:
new_items[id_] = new
idx[id_] = new
missing = set(idx.keys()) - set(seen)
if new_items or missing:
try:
group = idx_path.parent.stat().st_gid
tmp_path = Path(tempfile.mktemp(
suffix='.dbtmp',
dir=idx_path.parent.as_posix()))
if idx_path.exists():
try:
shutil.copyfile(idx_path, tmp_path.as_posix())
except OSError:
pass
# Write our updated index atomically.
with permissions.PermissionsManager(tmp_path,
group, 0o002):
out_db = dbm.open(tmp_path.as_posix(), 'c')
for id_, value in new_items.items():
out_db[str(id_)] = json.dumps(value)
for id_ in missing:
del out_db[str(id_)]
del idx[id_]
tmp_path.rename(idx_path)
except OSError:
pass
return idx
def main():
"""Setup Pavilion and run a command."""
# Pavilion is compatible with python >= 3.4
if sys.version_info[0] != 3 or sys.version_info[1] < 5:
output.fprint("Pavilion requires python 3.5 or higher.",
color=output.RED,
file=sys.stderr)
sys.exit(-1)
# This has to be done before we initialize plugins
parser = arguments.get_parser()
# Get the config, and
try:
pav_cfg = config.find()
except Exception as err:
output.fprint(
"Error getting config, exiting: {}"
.format(err),
file=sys.stderr,
color=output.RED)
sys.exit(-1)
# Create the basic directories in the working directory and the .pavilion
# directory.
perm_man = permissions.PermissionsManager(None, pav_cfg['shared_group'],
pav_cfg['umask'])
for path in [
config.USER_HOME_PAV,
config.USER_HOME_PAV/'working_dir',
pav_cfg.working_dir/'builds',
pav_cfg.working_dir/'series',
pav_cfg.working_dir/'test_runs',
pav_cfg.working_dir/'users']:
try:
path = path.expanduser()
path.mkdir(exist_ok=True)
perm_man.set_perms(path)
except OSError as err:
output.fprint(
"Could not create base directory '{}': {}"
.format(path, err),
color=output.RED,
file=sys.stderr,
)
sys.exit(1)
# Setup all the loggers for Pavilion
if not log_setup.setup_loggers(pav_cfg):
sys.exit(1)
# Initialize all the plugins
try:
plugins.initialize_plugins(pav_cfg)
except plugins.PluginError as err:
output.fprint(
"Error initializing plugins: {}"
.format(err),
color=output.RED,
file=sys.stderr)
sys.exit(-1)
# Parse the arguments
try:
args = parser.parse_args()
except Exception:
raise
if args.command_name is None:
parser.print_help()
sys.exit(0)
pav_cfg.pav_vars = pavilion_variables.PavVars()
if not args.profile:
run_cmd(pav_cfg, args)
else:
import cProfile
import pstats
stats_path = '/tmp/{}_pav_pstats'.format(os.getlogin())
cProfile.runctx('run_cmd(pav_cfg, args)', globals(), locals(),
stats_path)
stats = pstats.Stats(stats_path)
print("Profile Table")
stats.strip_dirs().sort_stats(args.profile_sort)\
.print_stats(args.profile_count)