def update_program_if_needed(silent=False):
'''
See if sadm itself needs to be updated.
'''
# Don't do anything if we're running from within a sandbox.
if sandbox.find_root_from_within(APP_FOLDER):
return
# If this folder has any kind of relationship with a repo...
if vcs.folder_is_tied_to_vcs(APP_FOLDER):
try:
status = vcs.get_status(APP_FOLDER, status_filter=lambda lbl: lbl!= 'unknown', revision=-1)
except:
eprintc("Status of sadm can't be determined (is the network down?). Skipping update.", WARNING_COLOR)
return
if status:
# See if there are any items at a status that will prevent success of update.
bad_status = [x for x in status.keys() if x not in 'modified|removed|added|renamed|kind changed']
if bad_status:
eprintc('''
The master version of sadm has changed, but automatic update is impossible. Do
a manual bzr up to resolve the following issues:
''', ERROR_COLOR)
print(aggregate_vcs.format_aspect_status(None, None, "sadm", status))
return
err = vcs.update_checkout(APP_FOLDER)
if err:
print('Unable to update sadm; exit code %d from "bzr up" command. Try running "bzr up" manually.' % err)
return
return True
elif not silent:
print("Sadm is up to date.")
else:
if not silent:
print("This copy of %s doesn't run from a directory that's connected to bzr." % APP_CMD)
def apply_tag(sb, dry_run):
tag = str(sb.get_build_id())
vprint('Applying tag "%s" to entire sandbox.' % tag, verbosity=1)
if dry_run:
return
err = 0
for c in sb.get_cached_components():
for a in sb.get_component_aspects(c.name):
folder = sb.get_component_path(c.name, a)
if vcs.folder_is_tied_to_vcs(folder):
revid = None
try:
vprint('Applying tag %s %s' % (c.get_name(), a), verbosity=1)
wr = vcs.get_working_repository()
if a == 'built':
a = "built.%s" % sb.get_targeted_platform_variant()
try:
revid = wr.get_local_revid(sb.get_branch(), c.get_name(), a)
except:
revid = None
wr.tag(tag, c.get_name(), a, sb.get_branch(), revisionid=revid)
except:
txt = traceback.format_exc()
relative_path = folder[len(sb.get_root()):]
if 'TagAlreadyExists' in txt:
vprint('%s already tagged.' % relative_path, verbosity=2)
else:
vprint('Unable to tag %s.' % relative_path, verbosity=1)
if revid:
vprint('Untable to tag revid %s.' % revid, verbosity=1)
vprint('%s' % txt, verbosity=2)
return err
def get_vcs_component_aspects(comp, sb):
'''
Some aspects of a component may not exist at all; others may have been
created dynamically. Return the subset of aspects that have some
relationship to our vcs system.
'''
return [a for a in sb.get_component_aspects(comp)
if vcs.folder_is_tied_to_vcs(sb.get_component_path(comp, a))]
def get_vcs_component_aspects(comp, sb):
'''
Some aspects of a component may not exist at all; others may have been
created dynamically. Return the subset of aspects that have some
relationship to our vcs system.
'''
return [
a for a in sb.get_component_aspects(comp)
if vcs.folder_is_tied_to_vcs(sb.get_component_path(comp, a))
]
def update_program_if_needed(silent=False):
'''
See if sadm itself needs to be updated.
'''
# Don't do anything if we're running from within a sandbox.
if sandbox.find_root_from_within(APP_FOLDER):
return
# If this folder has any kind of relationship with a repo...
if vcs.folder_is_tied_to_vcs(APP_FOLDER):
try:
status = vcs.get_status(APP_FOLDER,
status_filter=lambda lbl: lbl != 'unknown',
revision=-1)
except:
eprintc(
"Status of sadm can't be determined (is the network down?). Skipping update.",
WARNING_COLOR)
return
if status:
# See if there are any items at a status that will prevent success of update.
bad_status = [
x for x in status.keys()
if x not in 'modified|removed|added|renamed|kind changed'
]
if bad_status:
eprintc(
'''
The master version of sadm has changed, but automatic update is impossible. Do
a manual bzr up to resolve the following issues:
''', ERROR_COLOR)
print(
aggregate_vcs.format_aspect_status(None, None, "sadm",
status))
return
err = vcs.update_checkout(APP_FOLDER)
if err:
print(
'Unable to update sadm; exit code %d from "bzr up" command. Try running "bzr up" manually.'
% err)
return
return True
elif not silent:
print("Sadm is up to date.")
else:
if not silent:
print(
"This copy of %s doesn't run from a directory that's connected to bzr."
% APP_CMD)
def report(sb, state):
rr = sb.get_report_root()
root = sb.get_root()
need_checkin = vcs.folder_is_tied_to_vcs(rr)
try:
# Get latest version of reports so we are less likely to cause merge
# conflicts.
wr = vcs.get_working_repository()
use_master = False
if not need_checkin:
url = os.path.join(
wr.master_reporoot,
sb.get_branch(),
sb.get_top_component(),
'report',
).replace('\\', '/')
publish.create_branch(url, False)
use_master = True
wr.create_or_update_checkout(rr,
sb.get_top_component(),
'report',
sb.get_branch(),
None,
use_master=use_master)
need_checkin = vcs.folder_is_tied_to_vcs(rr)
# Report our results.
bi = buildinfo.BuildInfo()
machineFolder = os.path.join(rr, bi.host).replace('\\', '/')
summary = EvalSummary(sb.get_build_id(),
sb.get_sandboxtype().get_style(), bi.host,
state.phase, state.reason, state.start_time,
state.timestamps,
sb.get_targeted_platform_variant(), bi.os,
bi.bitness, bi.version)
db = Dashboard(rr)
db.add_summary(summary)
if os.path.exists(os.path.join(root, 'eval-log.txt')):
shutil.copy2(os.path.join(root, 'eval-log.txt'), machineFolder)
# Check in our changes.
if need_checkin:
status = vcs.get_status(rr)
if 'unknown' in status:
vcs.add(rr)
vcs.checkin(rr, msg="update dashboard", quiet_stderr=True)
try:
vcs.push(rr)
except BzrCommandError, e:
if 'diverged' in ("%s" % e):
print "\nAttemping to resolve diverged report aspect"
print "\nNuking report dir %s" % rr
if not ioutil.nuke(rr):
print "\nAuto resolving diverged report aspect failed!"
bi = BranchInfo(branchname=sb.get_branch(),
componentname=sb.get_top_component(),
aspectname='report')
aspectdir = bi.get_branchdir(wr.local_reporoot)
print "\nNuking report repo %s" % aspectdir
if not ioutil.nuke(aspectdir):
print "\nAuto resolving diverged report aspect failed!"
# Use the master because we have a problem here.
wr.create_local_branch(sb.get_top_component(),
'report',
sb.get_branch(),
use_master=True)
wr.create_or_update_checkout(rr,
sb.get_top_component(),
'report',
sb.get_branch(),
None,
use_master=True)
db = Dashboard(rr)
db.add_summary(summary)
if os.path.exists(os.path.join(root, 'eval-log.txt')):
shutil.copy2(os.path.join(root, 'eval-log.txt'),
machineFolder)
status = vcs.get_status(rr)
if 'unknown' in status:
vcs.add(rr)
vcs.checkin(rr, msg="update dashboard", quiet_stderr=True)
vcs.push(rr)
print "\nAuto resolve diverged report aspect success!"
else:
raise e
except:
traceback.print_exc()
def report(sb, state):
rr = sb.get_report_root()
root = sb.get_root()
need_checkin = vcs.folder_is_tied_to_vcs(rr)
try:
# Get latest version of reports so we are less likely to cause merge
# conflicts.
wr = vcs.get_working_repository()
use_master = False
if not need_checkin:
url = os.path.join(wr.master_reporoot, sb.get_branch(), sb.get_top_component(), 'report', ). replace('\\', '/')
publish.create_branch(url, False)
use_master = True
wr.create_or_update_checkout(rr, sb.get_top_component(), 'report', sb.get_branch(), None, use_master=use_master)
need_checkin = vcs.folder_is_tied_to_vcs(rr)
# Report our results.
bi = buildinfo.BuildInfo()
machineFolder = os.path.join(rr, bi.host).replace('\\', '/')
summary = EvalSummary(sb.get_build_id(), sb.get_sandboxtype().get_style(), bi.host,
state.phase, state.reason, state.start_time,
state.timestamps, sb.get_targeted_platform_variant(),
bi.os, bi.bitness, bi.version)
db = Dashboard(rr)
db.add_summary(summary)
if os.path.exists(os.path.join(root, 'eval-log.txt')):
shutil.copy2(os.path.join(root, 'eval-log.txt'), machineFolder)
# Check in our changes.
if need_checkin:
status = vcs.get_status(rr)
if 'unknown' in status:
vcs.add(rr)
vcs.checkin(rr, msg="update dashboard", quiet_stderr=True)
try:
vcs.push(rr)
except BzrCommandError, e:
if 'diverged' in ("%s" % e):
print "\nAttemping to resolve diverged report aspect"
print "\nNuking report dir %s" % rr
if not ioutil.nuke(rr):
print "\nAuto resolving diverged report aspect failed!"
bi = BranchInfo(branchname=sb.get_branch(), componentname=sb.get_top_component(), aspectname='report')
aspectdir = bi.get_branchdir(wr.local_reporoot)
print "\nNuking report repo %s" % aspectdir
if not ioutil.nuke(aspectdir):
print "\nAuto resolving diverged report aspect failed!"
# Use the master because we have a problem here.
wr.create_local_branch(sb.get_top_component(), 'report', sb.get_branch(), use_master=True)
wr.create_or_update_checkout(rr, sb.get_top_component(), 'report', sb.get_branch(), None, use_master=True)
db = Dashboard(rr)
db.add_summary(summary)
if os.path.exists(os.path.join(root, 'eval-log.txt')):
shutil.copy2(os.path.join(root, 'eval-log.txt'), machineFolder)
status = vcs.get_status(rr)
if 'unknown' in status:
vcs.add(rr)
vcs.checkin(rr, msg="update dashboard", quiet_stderr=True)
vcs.push(rr)
print "\nAuto resolve diverged report aspect success!"
else:
raise e
except:
traceback.print_exc()
def _get_deps(working_repo, platform, top_component, code_root, read_deps, already_analyzed, use_master=False, check_vcs=True):
if top_component.name == 'buildscripts':
top_component.reused_aspect = component.CODE_ASPECT_NAME
##TODO julie why would we do this?
## if top_component.reused_aspect == component.BUILT_ASPECT_NAME:
## interesting_branches = [b for b in working_repo.branches if b[1] == top_component.name and b[2].startswith(component.BUILT_ASPECT_NAME) and b[0] == top_component.branch]
## if not interesting_branches:
## top_component.reused_aspect = component.CODE_ASPECT_NAME
folder = ''
if (not top_component.revision) and code_root:
fldr = os.path.join(code_root, top_component.name)
if os.path.isdir(fldr):
if check_vcs and vcs.folder_is_tied_to_vcs(fldr):
output = vcs.get_status(fldr, status_filter=lambda lbl: lbl == 'modified' or lbl == 'added')
if output:
if 'modified' in output:
if METADATA_FILE in output['modified']:
folder = fldr
if 'added' in output:
if METADATA_FILE in output['added']:
folder = fldr
else:
folder = fldr
if folder:
if folder in already_analyzed:
return top_component #sections = already_analyzed[folder]
else:
print('\nLoading %s from %s.' % (METADATA_FILE, folder))
x = get_section_info_from_disk(MISC_SECTION, folder)
if 'terminal dependency' in x and top_component.reused_aspect.startswith(component.BUILT_ASPECT_NAME):
return top_component
sections = get_section_info_from_disk(DEPENDENCIES_SECTION, folder)
already_analyzed[folder] = sections
elif check_vcs:
key = '%s:%s' % (top_component.name, top_component.reused_aspect) #str(top_component)
if key in already_analyzed:
return top_component #sections = already_analyzed[key]
else:
x = get_section_info_from_vcs(MISC_SECTION, top_component, working_repo, platform, use_master)
if 'terminal dependency' in x and top_component.reused_aspect.startswith(component.BUILT_ASPECT_NAME):
return top_component
sections = get_section_info_from_vcs(DEPENDENCIES_SECTION, top_component, working_repo, platform, use_master)
already_analyzed[key] = sections
else:
return top_component
compOldDeps = False
for componentname, info in sections.iteritems():
componentname = componentname.strip()
aspect, revision, old = component.parse_component_info(info)
if aspect == component.BUILT_ASPECT_NAME:
aspect += "." + platform
if old:
compOldDeps = True
componentname, ignored, branch, task = working_repo.normalize(componentname, aspect, top_component.branch)
if revision:
m = _TAG_PAT.match(revision)
if not m:
raise Exception('%s is not a valid tag for pinning dependencies.' % revision)
assert(aspect)
top_component.dependencies.append(component.Component(componentname, branch, revision, aspect, parent=top_component))
if compOldDeps:
print('''Component %s/%s/%s has the old format for dependencies.
Please update dependencies in metadata.txt to match format found at:
https:// ... /working-with-code/concepts/dependencies''' % (top_component.name,top_component.reused_aspect,top_component.branch)) # TODO KIM refer to doc site
top_component.rank += len(top_component.dependencies)
for dep in top_component.dependencies:
if top_component.reused_aspect.startswith(component.BUILT_ASPECT_NAME):
dep.reused_aspect = top_component.reused_aspect
# We are suspicious that this optimization isn't working
if str(dep) not in read_deps or read_deps[str(dep)] != dep:
read_deps[str(dep)] = dep
dep = _get_deps(working_repo, platform, dep, code_root, read_deps, already_analyzed, use_master, check_vcs)
top_component.rank += dep.rank
return top_component
def _get_deps(working_repo,
platform,
top_component,
code_root,
read_deps,
already_analyzed,
use_master=False,
check_vcs=True):
if top_component.name == 'buildscripts':
top_component.reused_aspect = component.CODE_ASPECT_NAME
##TODO julie why would we do this?
## if top_component.reused_aspect == component.BUILT_ASPECT_NAME:
## interesting_branches = [b for b in working_repo.branches if b[1] == top_component.name and b[2].startswith(component.BUILT_ASPECT_NAME) and b[0] == top_component.branch]
## if not interesting_branches:
## top_component.reused_aspect = component.CODE_ASPECT_NAME
folder = ''
if (not top_component.revision) and code_root:
fldr = os.path.join(code_root, top_component.name)
if os.path.isdir(fldr):
if check_vcs and vcs.folder_is_tied_to_vcs(fldr):
output = vcs.get_status(fldr,
status_filter=lambda lbl: lbl ==
'modified' or lbl == 'added')
if output:
if 'modified' in output:
if METADATA_FILE in output['modified']:
folder = fldr
if 'added' in output:
if METADATA_FILE in output['added']:
folder = fldr
else:
folder = fldr
if folder:
if folder in already_analyzed:
return top_component #sections = already_analyzed[folder]
else:
print('\nLoading %s from %s.' % (METADATA_FILE, folder))
x = get_section_info_from_disk(MISC_SECTION, folder)
if 'terminal dependency' in x and top_component.reused_aspect.startswith(
component.BUILT_ASPECT_NAME):
return top_component
sections = get_section_info_from_disk(DEPENDENCIES_SECTION, folder)
already_analyzed[folder] = sections
elif check_vcs:
key = '%s:%s' % (top_component.name, top_component.reused_aspect
) #str(top_component)
if key in already_analyzed:
return top_component #sections = already_analyzed[key]
else:
x = get_section_info_from_vcs(MISC_SECTION, top_component,
working_repo, platform, use_master)
if 'terminal dependency' in x and top_component.reused_aspect.startswith(
component.BUILT_ASPECT_NAME):
return top_component
sections = get_section_info_from_vcs(DEPENDENCIES_SECTION,
top_component, working_repo,
platform, use_master)
already_analyzed[key] = sections
else:
return top_component
compOldDeps = False
for componentname, info in sections.iteritems():
componentname = componentname.strip()
aspect, revision, old = component.parse_component_info(info)
if aspect == component.BUILT_ASPECT_NAME:
aspect += "." + platform
if old:
compOldDeps = True
componentname, ignored, branch, task = working_repo.normalize(
componentname, aspect, top_component.branch)
if revision:
m = _TAG_PAT.match(revision)
if not m:
raise Exception(
'%s is not a valid tag for pinning dependencies.' %
revision)
assert (aspect)
top_component.dependencies.append(
component.Component(componentname,
branch,
revision,
aspect,
parent=top_component))
if compOldDeps:
print('''Component %s/%s/%s has the old format for dependencies.
Please update dependencies in metadata.txt to match format found at:
https:// ... /working-with-code/concepts/dependencies''' %
(top_component.name, top_component.reused_aspect,
top_component.branch)) # TODO KIM refer to doc site
top_component.rank += len(top_component.dependencies)
for dep in top_component.dependencies:
if top_component.reused_aspect.startswith(component.BUILT_ASPECT_NAME):
dep.reused_aspect = top_component.reused_aspect
# We are suspicious that this optimization isn't working
if str(dep) not in read_deps or read_deps[str(dep)] != dep:
read_deps[str(dep)] = dep
dep = _get_deps(working_repo, platform, dep, code_root, read_deps,
already_analyzed, use_master, check_vcs)
top_component.rank += dep.rank
return top_component
