def init_logging(logfile,
logtostdout=False,
silent=False,
colorize=fancylogger.Colorize.AUTO,
tmp_logdir=None):
"""Initialize logging."""
if logtostdout:
fancylogger.logToScreen(enable=True, stdout=True, colorize=colorize)
else:
if logfile is None:
# if logdir is specified but doesn't exist yet, create it first
if tmp_logdir and not os.path.exists(tmp_logdir):
try:
os.makedirs(tmp_logdir)
except (IOError, OSError) as err:
raise EasyBuildError(
"Failed to create temporary log directory %s: %s",
tmp_logdir, err)
# mkstemp returns (fd,filename), fd is from os.open, not regular open!
fd, logfile = tempfile.mkstemp(suffix='.log',
prefix='easybuild-',
dir=tmp_logdir)
os.close(fd)
fancylogger.logToFile(logfile, max_bytes=0)
print_msg('temporary log file in case of crash %s' % (logfile),
log=None,
silent=silent)
log = fancylogger.getLogger(fname=False)
return log, logfile
def main():
"""the main function"""
fancylogger.logToScreen(enable=True, stdout=True)
fancylogger.setLogLevelInfo()
options = {
'github-user':
('Your github username to use', None, 'store', None, 'g'),
'closed-pr': ('Delete all gists from closed pull-requests', None,
'store_true', True, 'p'),
'all':
('Delete all gists from Easybuild ', None, 'store_true', False, 'a'),
'orphans': ('Delete all gists without a pull-request', None,
'store_true', False, 'o'),
'dry-run':
("Only show which gists will be deleted but don't actually delete them",
None, 'store_true', False),
}
go = simple_option(options)
log = go.log
if not (go.options.all or go.options.closed_pr or go.options.orphans):
raise EasyBuildError("Please tell me what to do?")
if go.options.github_user is None:
EasyBuildOptions.DEFAULT_LOGLEVEL = None # Don't overwrite log level
eb_go = EasyBuildOptions(envvar_prefix='EASYBUILD', go_args=[])
username = eb_go.options.github_user
log.debug("Fetch github username from easybuild, found: %s", username)
else:
username = go.options.github_user
if username is None:
raise EasyBuildError("Could not find a github username")
else:
log.info("Using username = %s", username)
token = fetch_github_token(username)
gh = RestClient(GITHUB_API_URL, username=username, token=token)
all_gists = []
cur_page = 1
while True:
status, gists = gh.gists.get(per_page=100, page=cur_page)
if status != HTTP_STATUS_OK:
raise EasyBuildError(
"Failed to get a lists of gists for user %s: error code %s, message = %s",
username, status, gists)
if gists:
all_gists.extend(gists)
cur_page += 1
else:
break
log.info("Found %s gists", len(all_gists))
re_eb_gist = re.compile(
r"(EasyBuild test report|EasyBuild log for failed build)(.*?)$")
re_pr_nr = re.compile(r"(EB )?PR #([0-9]+)")
pr_cache = {}
num_deleted = 0
for gist in all_gists:
if not gist["description"]:
continue
gist_match = re_eb_gist.search(gist["description"])
if not gist_match:
log.debug("Found a non-Easybuild gist (id=%s)", gist["id"])
continue
log.debug("Found an Easybuild gist (id=%s)", gist["id"])
pr_data = gist_match.group(2)
pr_nrs_matches = re_pr_nr.findall(pr_data)
if go.options.all:
delete_gist = True
elif not pr_nrs_matches:
log.debug("Found Easybuild test report without PR (id=%s).",
gist["id"])
delete_gist = go.options.orphans
elif go.options.closed_pr:
# All PRs must be closed
delete_gist = True
for pr_nr_match in pr_nrs_matches:
eb_str, pr_num = pr_nr_match
if eb_str or GITHUB_EASYBLOCKS_REPO in pr_data:
repo = GITHUB_EASYBLOCKS_REPO
else:
repo = GITHUB_EASYCONFIGS_REPO
cache_key = "%s-%s" % (repo, pr_num)
if cache_key not in pr_cache:
try:
status, pr = gh.repos[GITHUB_EB_MAIN][repo].pulls[
pr_num].get()
except HTTPError as e:
status, pr = e.code, e.msg
if status != HTTP_STATUS_OK:
raise EasyBuildError(
"Failed to get pull-request #%s: error code %s, message = %s",
pr_num, status, pr)
pr_cache[cache_key] = pr["state"]
if pr_cache[cache_key] == "closed":
log.debug("Found report from closed %s PR #%s (id=%s)",
repo, pr_num, gist["id"])
elif delete_gist:
if len(pr_nrs_matches) > 1:
log.debug(
"Found at least 1 PR, that is not closed yet: %s/%s (id=%s)",
repo, pr_num, gist["id"])
delete_gist = False
else:
delete_gist = True
if delete_gist:
if go.options.dry_run:
log.info("DRY-RUN: Delete gist with id=%s", gist["id"])
num_deleted += 1
continue
try:
status, del_gist = gh.gists[gist["id"]].delete()
except HTTPError as e:
status, del_gist = e.code, e.msg
except URLError as e:
status, del_gist = None, e.reason
if status != HTTP_DELETE_OK:
log.warning(
"Unable to remove gist (id=%s): error code %s, message = %s",
gist["id"], status, del_gist)
else:
log.info("Deleted gist with id=%s", gist["id"])
num_deleted += 1
if go.options.dry_run:
log.info("DRY-RUN: Would delete %s gists", num_deleted)
else:
log.info("Deleted %s gists", num_deleted)
def main():
"""the main function"""
fancylogger.logToScreen(enable=True, stdout=True)
fancylogger.setLogLevelInfo()
options = {
'github-user': ('Your github username to use', None, 'store', None, 'g'),
'closed-pr': ('Delete all gists from closed pull-requests', None, 'store_true', True, 'p'),
'all': ('Delete all gists from Easybuild ', None, 'store_true', False, 'a'),
'orphans': ('Delete all gists without a pull-request', None, 'store_true', False, 'o'),
}
go = simple_option(options)
log = go.log
if not (go.options.all or go.options.closed_pr or go.options.orphans):
raise EasyBuildError("Please tell me what to do?")
if go.options.github_user is None:
eb_go = EasyBuildOptions(envvar_prefix='EASYBUILD', go_args=[])
username = eb_go.options.github_user
log.debug("Fetch github username from easybuild, found: %s", username)
else:
username = go.options.github_user
if username is None:
raise EasyBuildError("Could not find a github username")
else:
log.info("Using username = %s", username)
token = fetch_github_token(username)
gh = RestClient(GITHUB_API_URL, username=username, token=token)
all_gists = []
cur_page = 1
while True:
status, gists = gh.gists.get(per_page=100, page=cur_page)
if status != HTTP_STATUS_OK:
raise EasyBuildError("Failed to get a lists of gists for user %s: error code %s, message = %s",
username, status, gists)
if gists:
all_gists.extend(gists)
cur_page += 1
else:
break
log.info("Found %s gists", len(all_gists))
regex = re.compile(r"(EasyBuild test report|EasyBuild log for failed build).*?(?:PR #(?P<PR>[0-9]+))?\)?$")
pr_cache = {}
num_deleted = 0
for gist in all_gists:
if not gist["description"]:
continue
re_pr_num = regex.search(gist["description"])
delete_gist = False
if re_pr_num:
log.debug("Found a Easybuild gist (id=%s)", gist["id"])
pr_num = re_pr_num.group("PR")
if go.options.all:
delete_gist = True
elif pr_num and go.options.closed_pr:
log.debug("Found Easybuild test report for PR #%s", pr_num)
if pr_num not in pr_cache:
status, pr = gh.repos[GITHUB_EB_MAIN][GITHUB_EASYCONFIGS_REPO].pulls[pr_num].get()
if status != HTTP_STATUS_OK:
raise EasyBuildError("Failed to get pull-request #%s: error code %s, message = %s",
pr_num, status, pr)
pr_cache[pr_num] = pr["state"]
if pr_cache[pr_num] == "closed":
log.debug("Found report from closed PR #%s (id=%s)", pr_num, gist["id"])
delete_gist = True
elif not pr_num and go.options.orphans:
log.debug("Found Easybuild test report without PR (id=%s)", gist["id"])
delete_gist = True
if delete_gist:
status, del_gist = gh.gists[gist["id"]].delete()
if status != HTTP_DELETE_OK:
raise EasyBuildError("Unable to remove gist (id=%s): error code %s, message = %s",
gist["id"], status, del_gist)
else:
log.info("Delete gist with id=%s", gist["id"])
num_deleted += 1
log.info("Deleted %s gists", num_deleted)
def stop_logging(logfile, logtostdout=False):
"""Stop logging."""
if logtostdout:
fancylogger.logToScreen(enable=False, stdout=True)
if logfile is not None:
fancylogger.logToFile(logfile, enable=False)