def mode_create(session_dir, args):
validate_session_name(args.session)
logger.debug("Init is called - Session: %s, Volume: %s" %
(args.session, args.volume))
mkdirp(session_dir, exit_on_err=True, logger=logger)
mkdirp(os.path.join(session_dir, args.volume),
exit_on_err=True,
logger=logger)
status_file = os.path.join(session_dir, args.volume, "status")
if os.path.exists(status_file) and not args.force:
fail("Session %s already created" % args.session, logger=logger)
if not os.path.exists(status_file) or args.force:
ssh_setup(args)
enable_volume_options(args)
# Add Rollover time to current time to make sure changelogs
# will be available if we use this time as start time
time_to_update = int(time.time()) + get_changelog_rollover_time(
args.volume)
run_cmd_nodes("create", args, time_to_update=str(time_to_update))
if not os.path.exists(status_file) or args.reset_session_time:
with open(status_file, "w") as f:
f.write(str(time_to_update))
sys.stdout.write("Session %s created with volume %s\n" %
(args.session, args.volume))
sys.exit(0)
def mode_pre(session_dir, args):
global gtmpfilename
"""
Read from Session file and write to session.pre file
"""
endtime_to_update = int(time.time()) - get_changelog_rollover_time(
args.volume)
status_file = os.path.join(session_dir, args.volume, "status")
status_file_pre = status_file + ".pre"
mkdirp(os.path.dirname(args.outfile), exit_on_err=True, logger=logger)
# If Pre status file exists and running pre command again
if os.path.exists(status_file_pre) and not args.regenerate_outfile:
fail("Post command is not run after last pre, "
"use --regenerate-outfile")
start = 0
try:
with open(status_file) as f:
start = int(f.read().strip())
except ValueError:
pass
except (OSError, IOError) as e:
fail("Error Opening Session file %s: %s" % (status_file, e),
logger=logger)
logger.debug("Pre is called - Session: %s, Volume: %s, "
"Start time: %s, End time: %s" %
(args.session, args.volume, start, endtime_to_update))
prefix = datetime.now().strftime("%Y%m%d-%H%M%S-%f-")
gtmpfilename = prefix + next(tempfile._get_candidate_names())
run_cmd_nodes("pre", args, start=start, end=-1, tmpfilename=gtmpfilename)
# Merger
if args.full:
cmd = ["sort", "-u"] + node_outfiles + ["-o", args.outfile]
execute(cmd,
exit_msg="Failed to merge output files "
"collected from nodes",
logger=logger)
else:
# Read each Changelogs db and generate finaldb
create_file(args.outfile, exit_on_err=True, logger=logger)
outfilemerger = OutputMerger(args.outfile + ".db", node_outfiles)
write_output(args.outfile, outfilemerger, args.field_separator)
try:
os.remove(args.outfile + ".db")
except (IOError, OSError):
pass
run_cmd_nodes("cleanup", args, tmpfilename=gtmpfilename)
with open(status_file_pre, "w", buffering=0) as f:
f.write(str(endtime_to_update))
sys.stdout.write("Generated output file %s\n" % args.outfile)
def mode_create(session_dir, args):
validate_session_name(args.session)
logger.debug("Init is called - Session: %s, Volume: %s"
% (args.session, args.volume))
mkdirp(session_dir, exit_on_err=True, logger=logger)
mkdirp(os.path.join(session_dir, args.volume), exit_on_err=True,
logger=logger)
status_file = os.path.join(session_dir, args.volume, "status")
if os.path.exists(status_file) and not args.force:
fail("Session %s already created" % args.session, logger=logger)
if not os.path.exists(status_file) or args.force:
ssh_setup(args)
enable_volume_options(args)
# Add Rollover time to current time to make sure changelogs
# will be available if we use this time as start time
time_to_update = int(time.time()) + get_changelog_rollover_time(
args.volume)
run_cmd_nodes("create", args, time_to_update=str(time_to_update))
if not os.path.exists(status_file) or args.reset_session_time:
with open(status_file, "w") as f:
f.write(str(time_to_update))
sys.stdout.write("Session %s created with volume %s\n" %
(args.session, args.volume))
sys.exit(0)
def mode_pre(session_dir, args):
global gtmpfilename
"""
Read from Session file and write to session.pre file
"""
endtime_to_update = int(time.time()) - get_changelog_rollover_time(
args.volume)
status_file = os.path.join(session_dir, args.volume, "status")
status_file_pre = status_file + ".pre"
mkdirp(os.path.dirname(args.outfile), exit_on_err=True, logger=logger)
# If Pre status file exists and running pre command again
if os.path.exists(status_file_pre) and not args.regenerate_outfile:
fail("Post command is not run after last pre, "
"use --regenerate-outfile")
start = 0
try:
with open(status_file) as f:
start = int(f.read().strip())
except ValueError:
pass
except (OSError, IOError) as e:
fail("Error Opening Session file %s: %s"
% (status_file, e), logger=logger)
logger.debug("Pre is called - Session: %s, Volume: %s, "
"Start time: %s, End time: %s"
% (args.session, args.volume, start, endtime_to_update))
prefix = datetime.now().strftime("%Y%m%d-%H%M%S-%f-")
gtmpfilename = prefix + next(tempfile._get_candidate_names())
run_cmd_nodes("pre", args, start=start, end=-1, tmpfilename=gtmpfilename)
# Merger
if args.full:
cmd = ["sort", "-u"] + node_outfiles + ["-o", args.outfile]
execute(cmd,
exit_msg="Failed to merge output files "
"collected from nodes", logger=logger)
else:
# Read each Changelogs db and generate finaldb
create_file(args.outfile, exit_on_err=True, logger=logger)
outfilemerger = OutputMerger(args.outfile + ".db", node_outfiles)
write_output(args.outfile, outfilemerger, args.field_separator)
try:
os.remove(args.outfile + ".db")
except (IOError, OSError):
pass
run_cmd_nodes("cleanup", args, tmpfilename=gtmpfilename)
with open(status_file_pre, "w", buffering=0) as f:
f.write(str(endtime_to_update))
sys.stdout.write("Generated output file %s\n" % args.outfile)
def mode_create(session_dir, args):
logger.debug("Init is called - Session: %s, Volume: %s" %
(args.session, args.volume))
execute(["gluster", "volume", "info", args.volume],
exit_msg="Unable to get volume details",
logger=logger)
mkdirp(session_dir, exit_on_err=True, logger=logger)
mkdirp(os.path.join(session_dir, args.volume),
exit_on_err=True,
logger=logger)
status_file = os.path.join(session_dir, args.volume, "status")
if os.path.exists(status_file) and not args.force:
fail("Session %s already created" % args.session, logger=logger)
if not os.path.exists(status_file) or args.force:
ssh_setup(args)
execute(["gluster", "volume", "set", args.volume, "build-pgfid", "on"],
exit_msg="Failed to set volume option build-pgfid on",
logger=logger)
logger.info("Volume option set %s, build-pgfid on" % args.volume)
execute([
"gluster", "volume", "set", args.volume, "changelog.changelog",
"on"
],
exit_msg="Failed to set volume option "
"changelog.changelog on",
logger=logger)
logger.info("Volume option set %s, changelog.changelog on" %
args.volume)
execute([
"gluster", "volume", "set", args.volume,
"changelog.capture-del-path", "on"
],
exit_msg="Failed to set volume option "
"changelog.capture-del-path on",
logger=logger)
logger.info("Volume option set %s, changelog.capture-del-path on" %
args.volume)
# Add Rollover time to current time to make sure changelogs
# will be available if we use this time as start time
time_to_update = int(time.time()) + get_changelog_rollover_time(
args.volume)
run_cmd_nodes("create", args, time_to_update=str(time_to_update))
if not os.path.exists(status_file) or args.reset_session_time:
with open(status_file, "w", buffering=0) as f:
f.write(str(time_to_update))
sys.exit(0)
def mode_create(session_dir, args):
logger.debug("Init is called - Session: %s, Volume: %s"
% (args.session, args.volume))
execute(["gluster", "volume", "info", args.volume],
exit_msg="Unable to get volume details",
logger=logger)
mkdirp(session_dir, exit_on_err=True, logger=logger)
mkdirp(os.path.join(session_dir, args.volume), exit_on_err=True,
logger=logger)
status_file = os.path.join(session_dir, args.volume, "status")
if os.path.exists(status_file) and not args.force:
fail("Session %s already created" % args.session, logger=logger)
if not os.path.exists(status_file) or args.force:
ssh_setup(args)
execute(["gluster", "volume", "set",
args.volume, "build-pgfid", "on"],
exit_msg="Failed to set volume option build-pgfid on",
logger=logger)
logger.info("Volume option set %s, build-pgfid on" % args.volume)
execute(["gluster", "volume", "set",
args.volume, "changelog.changelog", "on"],
exit_msg="Failed to set volume option "
"changelog.changelog on", logger=logger)
logger.info("Volume option set %s, changelog.changelog on"
% args.volume)
execute(["gluster", "volume", "set",
args.volume, "changelog.capture-del-path", "on"],
exit_msg="Failed to set volume option "
"changelog.capture-del-path on", logger=logger)
logger.info("Volume option set %s, changelog.capture-del-path on"
% args.volume)
# Add Rollover time to current time to make sure changelogs
# will be available if we use this time as start time
time_to_update = int(time.time()) + get_changelog_rollover_time(
args.volume)
run_cmd_nodes("create", args, time_to_update=str(time_to_update))
if not os.path.exists(status_file) or args.reset_session_time:
with open(status_file, "w", buffering=0) as f:
f.write(str(time_to_update))
sys.exit(0)
def mode_create(session_dir, args):
logger.debug("Init is called - Session: %s, Volume: %s" %
(args.session, args.volume))
cmd = ["gluster", 'volume', 'info', args.volume, "--xml"]
_, data, _ = execute(cmd,
exit_msg="Failed to Run Gluster Volume Info",
logger=logger)
try:
tree = etree.fromstring(data)
statusStr = tree.find('volInfo/volumes/volume/statusStr').text
except (ParseError, AttributeError) as e:
fail("Invalid Volume: %s" % e, logger=logger)
if statusStr != "Started":
fail("Volume %s is not online" % args.volume, logger=logger)
mkdirp(session_dir, exit_on_err=True, logger=logger)
mkdirp(os.path.join(session_dir, args.volume),
exit_on_err=True,
logger=logger)
status_file = os.path.join(session_dir, args.volume, "status")
if os.path.exists(status_file) and not args.force:
fail("Session %s already created" % args.session, logger=logger)
if not os.path.exists(status_file) or args.force:
ssh_setup(args)
enable_volume_options(args)
# Add Rollover time to current time to make sure changelogs
# will be available if we use this time as start time
time_to_update = int(time.time()) + get_changelog_rollover_time(
args.volume)
run_cmd_nodes("create", args, time_to_update=str(time_to_update))
if not os.path.exists(status_file) or args.reset_session_time:
with open(status_file, "w", buffering=0) as f:
f.write(str(time_to_update))
sys.stdout.write("Session %s created with volume %s\n" %
(args.session, args.volume))
sys.exit(0)
def mode_create(session_dir, args):
logger.debug("Init is called - Session: %s, Volume: %s"
% (args.session, args.volume))
cmd = ["gluster", 'volume', 'info', args.volume, "--xml"]
_, data, _ = execute(cmd,
exit_msg="Failed to Run Gluster Volume Info",
logger=logger)
try:
tree = etree.fromstring(data)
statusStr = tree.find('volInfo/volumes/volume/statusStr').text
except (ParseError, AttributeError) as e:
fail("Invalid Volume: %s" % e, logger=logger)
if statusStr != "Started":
fail("Volume %s is not online" % args.volume, logger=logger)
mkdirp(session_dir, exit_on_err=True, logger=logger)
mkdirp(os.path.join(session_dir, args.volume), exit_on_err=True,
logger=logger)
status_file = os.path.join(session_dir, args.volume, "status")
if os.path.exists(status_file) and not args.force:
fail("Session %s already created" % args.session, logger=logger)
if not os.path.exists(status_file) or args.force:
ssh_setup(args)
enable_volume_options(args)
# Add Rollover time to current time to make sure changelogs
# will be available if we use this time as start time
time_to_update = int(time.time()) + get_changelog_rollover_time(
args.volume)
run_cmd_nodes("create", args, time_to_update=str(time_to_update))
if not os.path.exists(status_file) or args.reset_session_time:
with open(status_file, "w", buffering=0) as f:
f.write(str(time_to_update))
sys.stdout.write("Session %s created with volume %s\n" %
(args.session, args.volume))
sys.exit(0)
mkdirp(os.path.join(session_dir, args.volume),
exit_on_err=True,
logger=logger)
end = -1
if args.only_query:
start = args.start
end = args.end
else:
try:
with open(status_file) as f:
start = int(f.read().strip())
except (ValueError, OSError, IOError):
start = args.start
# end time is optional; so a -1 may be sent to use the default method of
# identifying the end time
if end == -1:
end = int(time.time()) - get_changelog_rollover_time(args.volume)
logger.info("%s Started Changelog Crawl - Start: %s End: %s" %
(args.brick, start, end))
actual_end = changelog_crawl(args.brick, start, end, args)
if not args.only_query:
with open(status_file_pre, "w") as f:
f.write(str(actual_end))
logger.info("%s Finished Changelog Crawl - End: %s" %
(args.brick, actual_end))
sys.exit(0)
mkdirp(os.path.join(session_dir, args.volume), exit_on_err=True,
logger=logger)
end = -1
if args.only_query:
start = args.start
end = args.end
else:
try:
with open(status_file) as f:
start = int(f.read().strip())
except (ValueError, OSError, IOError):
start = args.start
# end time is optional; so a -1 may be sent to use the default method of
# identifying the end time
if end == -1:
end = int(time.time()) - get_changelog_rollover_time(args.volume)
logger.info("%s Started Changelog Crawl - Start: %s End: %s" % (args.brick,
start,
end))
actual_end = changelog_crawl(args.brick, start, end, args)
if not args.only_query:
with open(status_file_pre, "w") as f:
f.write(str(actual_end))
logger.info("%s Finished Changelog Crawl - End: %s" % (args.brick,
actual_end))
sys.exit(0)
def mode_pre(session_dir, args):
"""
Read from Session file and write to session.pre file
"""
endtime_to_update = int(time.time()) - get_changelog_rollover_time(
args.volume)
status_file = os.path.join(session_dir, args.volume, "status")
status_file_pre = status_file + ".pre"
mkdirp(os.path.dirname(args.outfile), exit_on_err=True, logger=logger)
# If Pre status file exists and running pre command again
if os.path.exists(status_file_pre) and not args.regenerate_outfile:
fail("Post command is not run after last pre, "
"use --regenerate-outfile")
start = 0
try:
with open(status_file) as f:
start = int(f.read().strip())
except ValueError:
pass
except (OSError, IOError) as e:
fail("Error Opening Session file %s: %s"
% (status_file, e), logger=logger)
logger.debug("Pre is called - Session: %s, Volume: %s, "
"Start time: %s, End time: %s"
% (args.session, args.volume, start, endtime_to_update))
run_cmd_nodes("pre", args, start=start)
# Merger
if args.full:
cmd = ["sort", "-u"] + node_outfiles + ["-o", args.outfile]
execute(cmd,
exit_msg="Failed to merge output files "
"collected from nodes", logger=logger)
else:
# Read each Changelogs db and generate finaldb
create_file(args.outfile, exit_on_err=True, logger=logger)
outfilemerger = OutputMerger(args.outfile + ".db", node_outfiles)
with open(args.outfile, "a") as f:
for row in outfilemerger.get():
# Multiple paths in case of Hardlinks
paths = row[1].split(",")
for p in paths:
if p == "":
continue
f.write("%s %s %s\n" % (row[0], p, row[2]))
try:
os.remove(args.outfile + ".db")
except (IOError, OSError):
pass
run_cmd_nodes("cleanup", args)
with open(status_file_pre, "w", buffering=0) as f:
f.write(str(endtime_to_update))
sys.stdout.write("Generated output file %s\n" % args.outfile)
def mode_pre(session_dir, args):
"""
Read from Session file and write to session.pre file
"""
endtime_to_update = int(time.time()) - get_changelog_rollover_time(
args.volume)
status_file = os.path.join(session_dir, args.volume, "status")
status_file_pre = status_file + ".pre"
mkdirp(os.path.dirname(args.outfile), exit_on_err=True, logger=logger)
# If Pre status file exists and running pre command again
if os.path.exists(status_file_pre) and not args.regenerate_outfile:
fail("Post command is not run after last pre, "
"use --regenerate-outfile")
start = 0
try:
with open(status_file) as f:
start = int(f.read().strip())
except ValueError:
pass
except (OSError, IOError) as e:
fail("Error Opening Session file %s: %s" % (status_file, e),
logger=logger)
logger.debug("Pre is called - Session: %s, Volume: %s, "
"Start time: %s, End time: %s" %
(args.session, args.volume, start, endtime_to_update))
run_cmd_nodes("pre", args, start=start)
# Merger
if args.full:
cmd = ["sort", "-u"] + node_outfiles + ["-o", args.outfile]
execute(cmd,
exit_msg="Failed to merge output files "
"collected from nodes",
logger=logger)
else:
# Read each Changelogs db and generate finaldb
create_file(args.outfile, exit_on_err=True, logger=logger)
outfilemerger = OutputMerger(args.outfile + ".db", node_outfiles)
with open(args.outfile, "a") as f:
for row in outfilemerger.get():
# Multiple paths in case of Hardlinks
paths = row[1].split(",")
for p in paths:
if p == "":
continue
f.write("%s %s %s\n" % (row[0], p, row[2]))
try:
os.remove(args.outfile + ".db")
except (IOError, OSError):
pass
run_cmd_nodes("cleanup", args)
with open(status_file_pre, "w", buffering=0) as f:
f.write(str(endtime_to_update))
sys.stdout.write("Generated output file %s\n" % args.outfile)
