def compare_loudly(release,
svn_wrkspc,
git_wrkspc,
ignored=None,
pause_on_error=False,
debug=False,
verbose=False):
# compare Git and SVN workspaces
proj_counts = compare_workspaces(release,
svn_wrkspc,
git_wrkspc,
ignored=ignored,
debug=debug,
verbose=verbose)
text = None
if proj_counts is not None:
for proj, count in sorted(proj_counts.items(), key=lambda x: x[0]):
if proj in ignored:
continue
if text is None:
text = ":"
text += " %s*%d" % (proj, count)
if text is None:
print("** %s matches" % release)
else:
print("!! MISMATCH for %s%s" % (release, text))
__print_revisions("pdaq", svn_wrkspc, debug=debug, verbose=verbose)
if pause_on_error:
print(":: repodiff %s %s" % (git_wrkspc, svn_wrkspc))
read_input("%s %% Hit Return to continue: " % svn_wrkspc)
def send_alert(self, start_ticks, stop_ticks, destdir, request_id=None,
username=None, prefix=None, extract_hits=False, hubs=None,
print_to_console=False, proceed_no_prompt=False):
'''
Send request to Sender and wait for response
'''
if print_to_console:
print_log = LogToConsole
else:
print_log = logging
# get number of seconds of data requested
secrange = (stop_ticks - start_ticks) / 1E10
# catch negative ranges
if secrange <= 0:
print_log.error("Requesting negative time range (%.2f).\n"
"Try another time window.", secrange)
return False
# catch large ranges
if secrange > self.MAX_REQUEST_SECONDS:
print_log.error("Request for %.2f seconds is too huge.\nHsWorker "
"processes request only up to %d seconds.\n"
"Try a smaller time window.", secrange,
self.MAX_REQUEST_SECONDS)
return False
if secrange > WARN_SECONDS:
print_log.error("Warning: You are requesting %.2f seconds of"
" data\nNormal requests are %d seconds or less",
secrange, WARN_SECONDS)
if print_to_console and not proceed_no_prompt:
answer = read_input("Do you want to proceed? [y/n] : ")
if not answer.lower().startswith("y"):
print_log.error("Request was not sent")
return False
logging.info("Requesting %.2f seconds of HS data [%d-%d]",
secrange, start_ticks, stop_ticks)
try:
if not HsMessage.send_initial(self.__sender, request_id,
start_ticks, stop_ticks,
destdir, prefix=prefix,
extract_hits=extract_hits,
hubs=hubs, host=self.shorthost,
username=None):
print_log.error("Initial message was not sent!")
else:
print_log.info("HsGrabber sent request")
except:
print_log.exception("Failed to send request")
return True
def __remove_empty_moat_domapp(trunk,
remove_svn=False,
pause_for_purge=False,
debug=False,
verbose=False):
"If the 'domapp' and/or 'moat' subdirectory is empty, remove it"
removed = False
for entry in ("domapp", "moat"):
path = os.path.join(trunk, entry)
if not os.path.isdir(path) or len(os.listdir(path)) > 0:
continue
if pause_for_purge:
read_input("%%%%%% Purging %s: " % path) # XXX
if remove_svn:
svn_remove(path, debug=debug, verbose=verbose)
else:
os.rmdir(path)
if pause_for_purge:
read_input("%%%%%% Done purging %s: " % path) # XXX
removed = True
return removed
def __diff_dirs(orig_dir, new_dir, debug=False, dry_run=False, verbose=False):
"Print the differences between two directories"
cmd_args = ["diff", "-ru", orig_dir, new_dir]
found_diffs = False
for line in run_generator(cmd_args,
cmdname=" ".join(cmd_args[:2]).upper(),
returncode_handler=__handle_diff_returncode,
debug=debug,
dry_run=dry_run,
verbose=verbose):
# ignore complaints about missing 'moat' and 'domapp' directories,
# we're intentionally trying to omit those directories
if line.startswith("Only in "):
idx = line.find(": ")
if idx >= 0:
filename = line[idx + 2:]
if filename == "moat" or filename == "domapp":
continue
found_diffs = True
if found_diffs:
read_input("%%%%%% Found diffs for %s: " % new_dir) # XXX
def send_alert(self,
request_id=None,
username=None,
print_to_console=False):
'''
Send request to Sender and wait for response
'''
if print_to_console:
print_log = LogToConsole
else:
print_log = logging
if print_to_console:
answer = read_input("Do you want to proceed? [y/n] : ")
if not answer.lower().startswith("y"):
return False
logging.info("Deleting request %s", request_id)
try:
if not HsMessage.send(self.__sender,
HsMessage.DELETE,
request_id,
username,
0,
0,
"/dev/null",
host=self.shorthost):
print_log.error("Delete message was not sent!")
else:
print_log.info("HsDelete sent deletion request")
except:
print_log.exception("Failed to send deletion request")
return True
def main():
"Main method"
parser = argparse.ArgumentParser()
add_arguments(parser)
args = parser.parse_args()
daq_projects_url = "http://code.icecube.wisc.edu/daq/projects"
proj_name = "domhub-tools"
# names of original and new SVN workspaces
orig_wrkspc = proj_name + ".orig"
new_wrkspc = proj_name
if os.path.exists(new_wrkspc):
shutil.rmtree(new_wrkspc)
# full URL for this project
proj_url = daq_projects_url + "/" + proj_name
__check_out_original_project(proj_url,
orig_wrkspc,
debug=args.debug,
verbose=args.verbose)
try:
rewrite_project_repo(proj_name,
proj_url,
orig_wrkspc,
new_wrkspc,
pause_for_release=args.post_rel_pause,
debug=args.debug,
verbose=args.verbose)
print("Converted SVN repo is %s" % (new_wrkspc, ))
finally:
if os.path.exists(orig_wrkspc):
shutil.rmtree(orig_wrkspc)
# set the directory where SVN project databases are created/found
PDAQManager.set_home_directory()
# load the map of SVN usernames to Git authors
AuthorDB.load_authors("svn-authors", verbose=args.verbose)
gitmgr = GitRepoManager(use_github=args.use_github,
sleep_seconds=args.sleep_seconds)
# fetch this project's info
new_url = os.path.abspath(new_wrkspc)
project = get_pdaq_project(new_url,
clear_tables=True,
preload_from_log=args.load_from_log,
debug=args.debug,
verbose=args.verbose)
# get the Github or local repo object
gitrepo = gitmgr.get_repo(proj_name,
organization=args.organization,
destroy_old_repo=True,
make_public=False,
debug=args.debug,
verbose=args.verbose)
# if uploading to GitHub and we have a Mantis SQL dump file, load issues
mantis_issues = None
if args.use_github and gitrepo.has_issue_tracker and \
args.mantis_dump is not None:
mantis_issues = MantisConverter(args.mantis_dump,
project.database,
gitrepo, (proj_name, ),
verbose=args.verbose)
mantis_issues.close_resolved = args.close_resolved
mantis_issues.preserve_all_status = args.preserve_all_status
mantis_issues.preserve_resolved_status = args.preserve_resolved_status
# execute everything in a temporary directory which will be erased on exit
with TemporaryDirectory():
print("Converting %s repo" % (proj_name, ))
try:
convert_svn_to_git(project,
gitmgr,
mantis_issues,
gitrepo.ssh_url,
debug=args.debug,
verbose=args.verbose)
except:
if args.pause:
read_input("%s %% Hit Return to abort: " % os.getcwd())
raise
def rewrite_project_repo(project,
proj_url,
orig_wrkspc,
new_wrkspc,
pause_for_release=False,
debug=False,
verbose=False):
"""
Copy the domhub-tools SVN repository to a new SVN repository which
organizes the releases in an understandable manner
"""
__create_project_repo(project, new_wrkspc, debug=debug, verbose=verbose)
startrev = 9489
endrev = 10633
__initial_commit(orig_wrkspc,
new_wrkspc,
startrev,
pause_for_release=pause_for_release,
debug=debug,
verbose=verbose)
logfile_prefix = "/projects/domhub-tools/"
# paths for original and new workspace trunk directories
new_trunk = os.path.join(new_wrkspc, "trunk")
# paths for original and new workspace rel-100 directories
orig_rel100 = "trunk/rel-100"
new_rel100 = os.path.join(new_wrkspc, "tags", "rel-100")
__add_revisions(proj_url,
orig_wrkspc,
new_trunk,
logfile_prefix,
startrev,
endrev,
orig_subrel=orig_rel100,
new_reldir=new_rel100,
pause_for_release=pause_for_release,
debug=debug,
verbose=verbose)
rel_num = 101
rel_rev = 10634
#read_input("%%%%%% Ready for rel-%s: " % (rel_num, )) # XXX
__create_release(orig_wrkspc,
new_wrkspc,
rel_num,
rel_rev,
debug=debug,
verbose=verbose)
if pause_for_release:
read_input("%%%%%% Created rel-%s from r%d: " %
(rel_num, rel_rev)) # XXX
orig_rel = os.path.join(orig_wrkspc, "releases", "rel-%s" % rel_num,
"devel")
new_rel = os.path.join(new_wrkspc, "tags", "rel-%s" % rel_num)
__duplicate_workspace(orig_rel, new_rel)
#read_input("%%%%%% Copied %s to %s: " % (orig_rel, new_rel)) # XXX
if verbose:
print("~~~~~~ Status")
for line in svn_status(sandbox_dir=new_wrkspc):
print(">> %s" % str(line))
modified = __update_workspace_from_status(new_wrkspc,
debug=debug,
verbose=verbose)
modified |= __fix_makefile(new_rel)
if modified:
if pause_for_release:
read_input("%%%%%% Commit modified rel-%d: " % (rel_num, )) # XXX
svn_commit(new_wrkspc,
"Update for rel-%d" % rel_num,
debug=debug,
verbose=verbose)
__fix_revision(orig_rel, new_wrkspc, debug=debug, verbose=verbose)
if pause_for_release:
read_input("%%%%%% Done with rel-%d: " % (rel_num, )) # XXX
current_rev = rel_rev + 1
for rel_num, rel_rev in ((200, 10799), (201, 10806), (202, 10978),
(203, 11122), (204, 11125), (205, 11276),
(206, 11289), (207, 11398), (208, 11475),
(209, 11566), (210, 11631), (211, 13200),
(212, 13891), (216, 15694)):
# path for new workspace trunk directory
new_trunk = os.path.join(new_wrkspc, "trunk")
print("XXX REL-%d revs %d-%d" % (rel_num, current_rev, rel_rev - 1))
__add_revisions(proj_url,
orig_wrkspc,
new_trunk,
logfile_prefix,
current_rev,
rel_rev - 1,
pause_for_release=pause_for_release,
debug=debug,
verbose=verbose)
#read_input("%%%%%% Ready for rel-%s: " % (rel_num, )) # XXX
__create_release(orig_wrkspc,
new_wrkspc,
rel_num,
rel_rev,
debug=debug,
verbose=verbose)
if pause_for_release:
read_input("%%%%%% Created rel-%s from r%d: " %
(rel_num, rel_rev)) # XXX
if rel_num == 200:
orig_rel = os.path.join(orig_wrkspc, "branches", "rel-2xx",
"rel-%s" % rel_num)
elif rel_num < 212:
orig_rel = os.path.join(orig_wrkspc, "releases",
"rel-%s" % rel_num, "rel-200")
else:
orig_rel = os.path.join(orig_wrkspc, "releases",
"rel-%s" % rel_num)
new_rel = os.path.join(new_wrkspc, "tags", "rel-%s" % rel_num)
__duplicate_workspace(orig_rel, new_rel)
__remove_extra(orig_rel, new_rel, debug=debug, verbose=debug)
if pause_for_release:
read_input("%%%%%% Copied %s to %s: " % (orig_rel, new_rel)) # XXX
if verbose:
print("~~~~~~ Status")
for line in svn_status(sandbox_dir=new_wrkspc):
print(">> %s" % str(line))
modified = __update_workspace_from_status(new_wrkspc,
debug=debug,
verbose=verbose)
modified |= __fix_makefile(new_rel)
if modified:
if pause_for_release:
read_input("%%%%%% Commit modified rel-%d: " %
(rel_num, )) # XXX
svn_commit(new_wrkspc,
"Update for rel-%d" % rel_num,
debug=debug,
verbose=verbose)
__fix_revision(orig_rel, new_wrkspc, debug=debug, verbose=verbose)
current_rev = rel_rev + 1
if pause_for_release:
read_input("%%%%%% Quit after rel-%d r%d: " % (rel_num, rel_rev))
def __update_for_revision(orig_wrkspc,
subdir_list,
new_wrkspc,
revision,
logmsg,
pause_for_release=False,
debug=False,
verbose=False):
# update the original repo to this revision
for line in svn_update(orig_wrkspc,
revision=revision,
debug=debug,
verbose=verbose):
pass
if verbose:
print("=== rev %d: %s" % (revision, logmsg))
__purge_nonsvn_files(new_wrkspc)
modified = False
for subdir in subdir_list:
orig_path = os.path.join(orig_wrkspc, subdir)
shutil.copytree(orig_path,
new_wrkspc,
ignore=shutil.ignore_patterns('.svn'),
dirs_exist_ok=True)
__remove_empty_moat_domapp(new_wrkspc,
remove_svn=False,
debug=debug,
verbose=verbose)
modified |= __update_workspace_from_status(new_wrkspc,
debug=debug,
verbose=verbose)
if not modified:
print("WARNING: No modifications found in %s for rev %d: %s" %
(subdir_list, revision, logmsg),
file=sys.stderr)
return
if verbose:
print("--- SVN status")
for line in svn_status(sandbox_dir=new_wrkspc):
print("%s" % line)
for subdir in subdir_list:
orig_path = os.path.join(orig_wrkspc, subdir)
print("=== OLD: %s\n~~~ NEW: %s" % (orig_path, new_wrkspc))
print("--- Diff")
__diff_dirs(orig_path, new_wrkspc, debug=debug, verbose=verbose)
if pause_for_release:
read_input("%%%%%% r%d: %s: " % (revision, logmsg))
# commit this revision
svn_commit(new_wrkspc, logmsg, debug=debug, verbose=verbose)
# update commit with original author and date
orig_path = os.path.join(orig_wrkspc, subdir_list[0])
__fix_revision(orig_path, new_wrkspc, debug=debug, verbose=verbose)
# update workspace with new release info
for line in svn_update(new_wrkspc, debug=debug, verbose=verbose):
pass
def __initial_commit(orig_wrkspc,
new_wrkspc,
initial_revision,
pause_for_release=False,
debug=False,
verbose=False):
#read_input("%%%%%% Before new repo init: ") # XXX
# update the original repo to this revision
for line in svn_update(orig_wrkspc,
revision=initial_revision,
debug=debug,
verbose=verbose):
pass
# build target path for initial files/directories
new_trunk = os.path.join(new_wrkspc, "trunk")
# copy initial release to new trunk
rel100 = os.path.join(orig_wrkspc, "releases", "rel-100", "rel-100")
__duplicate_workspace(rel100, new_trunk)
#read_input("%%%%%% Before initial release: ") # XXX
svn_commit(new_wrkspc,
"Initial commit for rel-100",
debug=debug,
verbose=verbose)
__fix_revision(rel100, new_wrkspc, debug=debug, verbose=verbose)
#read_input("%%%%%% Set properties: ") # XXX
# update new workspace with first commit info
for line in svn_update(new_wrkspc, debug=debug, verbose=verbose):
pass
#read_input("%%%%%% Ready for initial release: ") # XXX
rel_num = 100
# create rel-100 from devel code
__create_release(orig_wrkspc,
new_wrkspc,
rel_num,
9488,
debug=debug,
verbose=verbose)
if pause_for_release:
read_input("%%%%%% Created initial release: ") # XXX
orig_rel = os.path.join(orig_wrkspc, "releases", "rel-100", "rel-100")
new_rel = os.path.join(new_wrkspc, "tags", "rel-%s" % rel_num)
__duplicate_workspace(orig_rel, new_rel)
__remove_extra(orig_rel, new_rel, debug=debug, verbose=debug)
modified = __update_workspace_from_status(new_wrkspc,
debug=debug,
verbose=verbose)
modified |= __fix_makefile(new_rel)
if modified:
if pause_for_release:
read_input("%%%%%% Commit modified rel-%d: " % (rel_num, ))
svn_commit(new_wrkspc,
"Update for rel-%d" % rel_num,
debug=debug,
verbose=verbose)
__fix_revision(rel100, new_wrkspc, debug=debug, verbose=verbose)
if pause_for_release:
read_input("%%%%%% Done with rel-%d: " % (rel_num, ))
__purge_nonsvn_files(new_wrkspc)
# copy devel files to new trunk
orig_devel = os.path.join(orig_wrkspc, "trunk", "devel")
shutil.copytree(orig_devel,
os.path.join(new_wrkspc, "trunk"),
ignore=shutil.ignore_patterns('.svn'),
dirs_exist_ok=True)
modified = __update_workspace_from_status(new_wrkspc,
debug=debug,
verbose=verbose)
if not modified:
raise SystemExit("Initial repo was not updated!!!")
#read_input("%%%%%% Initial devel code: ") # XXX
# commit initial set of files/directories
svn_commit(new_wrkspc, "Initial devel code", debug=debug, verbose=verbose)
__fix_revision(orig_devel, new_wrkspc, debug=debug, verbose=verbose)
def do_all_the_things(project,
gitmgr,
mantis_issues,
test_branch,
test_revision,
checkpoint=False,
workspace=None,
debug=False,
verbose=False):
print("Loading %s database" % (project.name, ))
database = project.database
sandbox_dir = os.path.join(workspace, database.name)
if not os.path.exists(sandbox_dir):
raise Exception("Sandbox %s does not exist" % (sandbox_dir, ))
prev_checkpoint_list = None
for top_url, first_revision, first_date in database.all_urls_by_date:
_, project_name, branch_path = SVNMetadata.split_url(top_url)
if project_name != project.name:
print("WARNING: Found URL for \"%s\", not \"%s\"\n (URL %s)" %
(project_name, project.name, top_url),
file=sys.stderr)
# extract the branch name from the branch path
# (e.g. "tags/foo" -> "foo")
short_branch = branch_path.rsplit("/")[-1]
# if we're looking for a starting branch...
if test_branch is not None:
# ...and this isn't that branch...
if test_branch != short_branch:
# ..then skip this branch
continue
# we made it to the requested branch, stop looking
test_branch = None
# derive the Git remote name from the SVN branch name
if branch_path == SVNMetadata.TRUNK_NAME:
git_remote = "master"
else:
git_remote = short_branch
if git_remote in ("HEAD", "master"):
raise Exception("Questionable branch name \"%s\"" %
(git_remote, ))
print(
"%s branch %s first_rev %s (%s)\n\t%s" %
(database.name, branch_path, first_revision, first_date, top_url))
prev_saved = None
for count, entry in enumerate(database.entries(branch_path)):
if test_revision is not None:
if entry.revision < test_revision:
continue
test_revision = None
if database.name in IGNORED_REVISIONS and \
entry.revision in IGNORED_REVISIONS[database.name]:
print("Ignoring %s rev %s" % (database.name, entry.revision))
continue
if checkpoint:
tarpaths = save_checkpoint_files(sandbox_dir, database.name,
branch_path, entry.revision,
gitmgr.local_repo_path)
if prev_checkpoint_list is not None:
for path in prev_checkpoint_list:
if path is not None and os.path.exists(path):
os.unlink(path)
prev_checkpoint_list = tarpaths
print("Convert %s" % str(entry))
try:
if convert_revision(database,
gitmgr,
mantis_issues,
count,
top_url,
git_remote,
entry,
first_commit=False,
rewrite_proc=rewrite_pdaq,
sandbox_dir=sandbox_dir,
debug=debug,
verbose=verbose):
if prev_saved is not None:
entry.set_previous(prev_saved)
database.update_previous_in_database(entry)
prev_saved = entry
except:
traceback.print_exc()
print("Failed while converting %s rev %s" %
(top_url, entry.revision))
read_input("%s %% Hit Return to exit: " % os.getcwd())
return
# clean up unneeded checkpoint files
if prev_checkpoint_list is not None:
for path in prev_checkpoint_list:
if path is not None and os.path.exists(path):
os.unlink(path)