def get_last_code_date(self):
"""
Return the time (in seconds since epoch) when code was last modified in
this sandbox. If code is pristine (hasn't changed from what's checked
in), None is returned.
"""
when_since_epoch = 0
cr = self.get_code_root()
# I know I could do the list comprehensions below in a single step, but
# I want to make this debuggable and do it in stages.
code_components = os.listdir(cr)
code_components = [cr + cc for cc in code_components]
code_components = [cc for cc in code_components if os.path.isdir(cc + "/.bzr")]
def potentially_changed_file(lbl):
return lbl == "modified" or lbl == "unknown"
for cc in code_components:
status = vcs.get_status(cc, status_filter=potentially_changed_file)
if status:
for k in status.keys():
for m in status[k]:
try:
lastmod = os.stat(cc + "/" + m).st_mtime
if lastmod > when_since_epoch:
when_since_epoch = lastmod
except:
pass
if not when_since_epoch:
when_since_epoch = None
return when_since_epoch
def get_component_status(comp,
sb,
status_filter=None,
aspect_filter=None,
revision=None):
'''
Return a dictionary describing all files/folders with notable status within
the specified component.
Dictionary format: key = an aspect of the component ("code", "test",
or "built"); value = sub-dictionary of notable status items. Sub-dictionary
format: key = status label like "modified" or "unknown"; value = a list of
paths, relative to the component's folder in the <aspect> root, of files
with the specified status.
@param status_filter A function that decides whether a particular status
label is interesting. Takes a string and returns a boolean.
@param aspect_filter A function that decides whether a particular aspect is
interesting. Takes a string and returns a boolean.
'''
status = {}
# Iterate over all aspects of the component that exist and are tied to vcs.
for a in get_vcs_component_aspects(comp, sb):
path = sb.get_component_path(comp, a)
if (not aspect_filter) or aspect_filter(a):
x = vcs.get_status(path,
status_filter=status_filter,
revision=revision)
if x:
status[a] = x
return status
def update_program_if_needed(silent=False):
'''
See if sadm itself needs to be updated.
'''
# Don't do anything if we're running from within a sandbox.
if sandbox.find_root_from_within(APP_FOLDER):
return
# If this folder has any kind of relationship with a repo...
if vcs.folder_is_tied_to_vcs(APP_FOLDER):
try:
status = vcs.get_status(APP_FOLDER, status_filter=lambda lbl: lbl!= 'unknown', revision=-1)
except:
eprintc("Status of sadm can't be determined (is the network down?). Skipping update.", WARNING_COLOR)
return
if status:
# See if there are any items at a status that will prevent success of update.
bad_status = [x for x in status.keys() if x not in 'modified|removed|added|renamed|kind changed']
if bad_status:
eprintc('''
The master version of sadm has changed, but automatic update is impossible. Do
a manual bzr up to resolve the following issues:
''', ERROR_COLOR)
print(aggregate_vcs.format_aspect_status(None, None, "sadm", status))
return
err = vcs.update_checkout(APP_FOLDER)
if err:
print('Unable to update sadm; exit code %d from "bzr up" command. Try running "bzr up" manually.' % err)
return
return True
elif not silent:
print("Sadm is up to date.")
else:
if not silent:
print("This copy of %s doesn't run from a directory that's connected to bzr." % APP_CMD)
def get_component_status(comp, sb, status_filter=None, aspect_filter=None, revision=None):
'''
Return a dictionary describing all files/folders with notable status within
the specified component.
Dictionary format: key = an aspect of the component ("code", "test",
or "built"); value = sub-dictionary of notable status items. Sub-dictionary
format: key = status label like "modified" or "unknown"; value = a list of
paths, relative to the component's folder in the <aspect> root, of files
with the specified status.
@param status_filter A function that decides whether a particular status
label is interesting. Takes a string and returns a boolean.
@param aspect_filter A function that decides whether a particular aspect is
interesting. Takes a string and returns a boolean.
'''
status = {}
# Iterate over all aspects of the component that exist and are tied to vcs.
for a in get_vcs_component_aspects(comp, sb):
path = sb.get_component_path(comp, a)
if (not aspect_filter) or aspect_filter(a):
x = vcs.get_status(path, status_filter=status_filter, revision=revision)
if x:
status[a] = x
return status
def get_last_code_date(self):
'''
Return the time (in seconds since epoch) when code was last modified in
this sandbox. If code is pristine (hasn't changed from what's checked
in), None is returned.
'''
when_since_epoch = 0
cr = self.get_code_root()
# I know I could do the list comprehensions below in a single step, but
# I want to make this debuggable and do it in stages.
code_components = os.listdir(cr)
code_components = [cr + cc for cc in code_components]
code_components = [
cc for cc in code_components if os.path.isdir(cc + '/.bzr')
]
def potentially_changed_file(lbl):
return lbl == 'modified' or lbl == 'unknown'
for cc in code_components:
status = vcs.get_status(cc, status_filter=potentially_changed_file)
if status:
for k in status.keys():
for m in status[k]:
try:
lastmod = os.stat(cc + '/' + m).st_mtime
if lastmod > when_since_epoch:
when_since_epoch = lastmod
except:
pass
if not when_since_epoch:
when_since_epoch = None
return when_since_epoch
def add_new_files(sb, folder_to_publish):
status = vcs.get_status(folder_to_publish, status_filter=_bzr_status_is_interesting)
if 'conflicted' in status:
vprint('Conflicting items; publish requires manual intervention.')
vprint(' ' + '\n '.join(status['conflicted']))
return 1
if 'unknown' in status:
vprint('Adding new files.', verbosity=1)
cmd = 'bzr add'
vprint(cmd, verbosity=2)
err = os.system(cmd)
if err:
return err
return 0
def update_program_if_needed(silent=False):
'''
See if sadm itself needs to be updated.
'''
# Don't do anything if we're running from within a sandbox.
if sandbox.find_root_from_within(APP_FOLDER):
return
# If this folder has any kind of relationship with a repo...
if vcs.folder_is_tied_to_vcs(APP_FOLDER):
try:
status = vcs.get_status(APP_FOLDER,
status_filter=lambda lbl: lbl != 'unknown',
revision=-1)
except:
eprintc(
"Status of sadm can't be determined (is the network down?). Skipping update.",
WARNING_COLOR)
return
if status:
# See if there are any items at a status that will prevent success of update.
bad_status = [
x for x in status.keys()
if x not in 'modified|removed|added|renamed|kind changed'
]
if bad_status:
eprintc(
'''
The master version of sadm has changed, but automatic update is impossible. Do
a manual bzr up to resolve the following issues:
''', ERROR_COLOR)
print(
aggregate_vcs.format_aspect_status(None, None, "sadm",
status))
return
err = vcs.update_checkout(APP_FOLDER)
if err:
print(
'Unable to update sadm; exit code %d from "bzr up" command. Try running "bzr up" manually.'
% err)
return
return True
elif not silent:
print("Sadm is up to date.")
else:
if not silent:
print(
"This copy of %s doesn't run from a directory that's connected to bzr."
% APP_CMD)
def checkin_component(msg, comp, sb, validate_status=True):
if validate_status:
bad_status = get_component_status(comp, status_filter=_items_that_wont_checkin)
if bad_status:
print(format_component_status(sb, bad_status))
print('Some items are not ready for checkin.')
return
aspects = get_vcs_component_aspects(comp, sb)
for a in aspects:
if a.startswith(component.BUILT_ASPECT_NAME):
continue
path = sb.get_component_path(comp, a)
if vcs.get_status(path):
try:
x = vcs.checkin(path, msg)
except:
print(sys.exc_info()[1])
def checkin_component(msg, comp, sb, validate_status=True):
if validate_status:
bad_status = get_component_status(
comp, status_filter=_items_that_wont_checkin)
if bad_status:
print(format_component_status(sb, bad_status))
print('Some items are not ready for checkin.')
return
aspects = get_vcs_component_aspects(comp, sb)
for a in aspects:
if a.startswith(component.BUILT_ASPECT_NAME):
continue
path = sb.get_component_path(comp, a)
if vcs.get_status(path):
try:
x = vcs.checkin(path, msg)
except:
print(sys.exc_info()[1])
def report(sb, state):
rr = sb.get_report_root()
root = sb.get_root()
need_checkin = vcs.folder_is_tied_to_vcs(rr)
try:
# Get latest version of reports so we are less likely to cause merge
# conflicts.
wr = vcs.get_working_repository()
use_master = False
if not need_checkin:
url = os.path.join(
wr.master_reporoot,
sb.get_branch(),
sb.get_top_component(),
'report',
).replace('\\', '/')
publish.create_branch(url, False)
use_master = True
wr.create_or_update_checkout(rr,
sb.get_top_component(),
'report',
sb.get_branch(),
None,
use_master=use_master)
need_checkin = vcs.folder_is_tied_to_vcs(rr)
# Report our results.
bi = buildinfo.BuildInfo()
machineFolder = os.path.join(rr, bi.host).replace('\\', '/')
summary = EvalSummary(sb.get_build_id(),
sb.get_sandboxtype().get_style(), bi.host,
state.phase, state.reason, state.start_time,
state.timestamps,
sb.get_targeted_platform_variant(), bi.os,
bi.bitness, bi.version)
db = Dashboard(rr)
db.add_summary(summary)
if os.path.exists(os.path.join(root, 'eval-log.txt')):
shutil.copy2(os.path.join(root, 'eval-log.txt'), machineFolder)
# Check in our changes.
if need_checkin:
status = vcs.get_status(rr)
if 'unknown' in status:
vcs.add(rr)
vcs.checkin(rr, msg="update dashboard", quiet_stderr=True)
try:
vcs.push(rr)
except BzrCommandError, e:
if 'diverged' in ("%s" % e):
print "\nAttemping to resolve diverged report aspect"
print "\nNuking report dir %s" % rr
if not ioutil.nuke(rr):
print "\nAuto resolving diverged report aspect failed!"
bi = BranchInfo(branchname=sb.get_branch(),
componentname=sb.get_top_component(),
aspectname='report')
aspectdir = bi.get_branchdir(wr.local_reporoot)
print "\nNuking report repo %s" % aspectdir
if not ioutil.nuke(aspectdir):
print "\nAuto resolving diverged report aspect failed!"
# Use the master because we have a problem here.
wr.create_local_branch(sb.get_top_component(),
'report',
sb.get_branch(),
use_master=True)
wr.create_or_update_checkout(rr,
sb.get_top_component(),
'report',
sb.get_branch(),
None,
use_master=True)
db = Dashboard(rr)
db.add_summary(summary)
if os.path.exists(os.path.join(root, 'eval-log.txt')):
shutil.copy2(os.path.join(root, 'eval-log.txt'),
machineFolder)
status = vcs.get_status(rr)
if 'unknown' in status:
vcs.add(rr)
vcs.checkin(rr, msg="update dashboard", quiet_stderr=True)
vcs.push(rr)
print "\nAuto resolve diverged report aspect success!"
else:
raise e
except:
traceback.print_exc()
def report(sb, state):
rr = sb.get_report_root()
root = sb.get_root()
need_checkin = vcs.folder_is_tied_to_vcs(rr)
try:
# Get latest version of reports so we are less likely to cause merge
# conflicts.
wr = vcs.get_working_repository()
use_master = False
if not need_checkin:
url = os.path.join(wr.master_reporoot, sb.get_branch(), sb.get_top_component(), 'report', ). replace('\\', '/')
publish.create_branch(url, False)
use_master = True
wr.create_or_update_checkout(rr, sb.get_top_component(), 'report', sb.get_branch(), None, use_master=use_master)
need_checkin = vcs.folder_is_tied_to_vcs(rr)
# Report our results.
bi = buildinfo.BuildInfo()
machineFolder = os.path.join(rr, bi.host).replace('\\', '/')
summary = EvalSummary(sb.get_build_id(), sb.get_sandboxtype().get_style(), bi.host,
state.phase, state.reason, state.start_time,
state.timestamps, sb.get_targeted_platform_variant(),
bi.os, bi.bitness, bi.version)
db = Dashboard(rr)
db.add_summary(summary)
if os.path.exists(os.path.join(root, 'eval-log.txt')):
shutil.copy2(os.path.join(root, 'eval-log.txt'), machineFolder)
# Check in our changes.
if need_checkin:
status = vcs.get_status(rr)
if 'unknown' in status:
vcs.add(rr)
vcs.checkin(rr, msg="update dashboard", quiet_stderr=True)
try:
vcs.push(rr)
except BzrCommandError, e:
if 'diverged' in ("%s" % e):
print "\nAttemping to resolve diverged report aspect"
print "\nNuking report dir %s" % rr
if not ioutil.nuke(rr):
print "\nAuto resolving diverged report aspect failed!"
bi = BranchInfo(branchname=sb.get_branch(), componentname=sb.get_top_component(), aspectname='report')
aspectdir = bi.get_branchdir(wr.local_reporoot)
print "\nNuking report repo %s" % aspectdir
if not ioutil.nuke(aspectdir):
print "\nAuto resolving diverged report aspect failed!"
# Use the master because we have a problem here.
wr.create_local_branch(sb.get_top_component(), 'report', sb.get_branch(), use_master=True)
wr.create_or_update_checkout(rr, sb.get_top_component(), 'report', sb.get_branch(), None, use_master=True)
db = Dashboard(rr)
db.add_summary(summary)
if os.path.exists(os.path.join(root, 'eval-log.txt')):
shutil.copy2(os.path.join(root, 'eval-log.txt'), machineFolder)
status = vcs.get_status(rr)
if 'unknown' in status:
vcs.add(rr)
vcs.checkin(rr, msg="update dashboard", quiet_stderr=True)
vcs.push(rr)
print "\nAuto resolve diverged report aspect success!"
else:
raise e
except:
traceback.print_exc()
def _get_deps(working_repo, platform, top_component, code_root, read_deps, already_analyzed, use_master=False, check_vcs=True):
if top_component.name == 'buildscripts':
top_component.reused_aspect = component.CODE_ASPECT_NAME
##TODO julie why would we do this?
## if top_component.reused_aspect == component.BUILT_ASPECT_NAME:
## interesting_branches = [b for b in working_repo.branches if b[1] == top_component.name and b[2].startswith(component.BUILT_ASPECT_NAME) and b[0] == top_component.branch]
## if not interesting_branches:
## top_component.reused_aspect = component.CODE_ASPECT_NAME
folder = ''
if (not top_component.revision) and code_root:
fldr = os.path.join(code_root, top_component.name)
if os.path.isdir(fldr):
if check_vcs and vcs.folder_is_tied_to_vcs(fldr):
output = vcs.get_status(fldr, status_filter=lambda lbl: lbl == 'modified' or lbl == 'added')
if output:
if 'modified' in output:
if METADATA_FILE in output['modified']:
folder = fldr
if 'added' in output:
if METADATA_FILE in output['added']:
folder = fldr
else:
folder = fldr
if folder:
if folder in already_analyzed:
return top_component #sections = already_analyzed[folder]
else:
print('\nLoading %s from %s.' % (METADATA_FILE, folder))
x = get_section_info_from_disk(MISC_SECTION, folder)
if 'terminal dependency' in x and top_component.reused_aspect.startswith(component.BUILT_ASPECT_NAME):
return top_component
sections = get_section_info_from_disk(DEPENDENCIES_SECTION, folder)
already_analyzed[folder] = sections
elif check_vcs:
key = '%s:%s' % (top_component.name, top_component.reused_aspect) #str(top_component)
if key in already_analyzed:
return top_component #sections = already_analyzed[key]
else:
x = get_section_info_from_vcs(MISC_SECTION, top_component, working_repo, platform, use_master)
if 'terminal dependency' in x and top_component.reused_aspect.startswith(component.BUILT_ASPECT_NAME):
return top_component
sections = get_section_info_from_vcs(DEPENDENCIES_SECTION, top_component, working_repo, platform, use_master)
already_analyzed[key] = sections
else:
return top_component
compOldDeps = False
for componentname, info in sections.iteritems():
componentname = componentname.strip()
aspect, revision, old = component.parse_component_info(info)
if aspect == component.BUILT_ASPECT_NAME:
aspect += "." + platform
if old:
compOldDeps = True
componentname, ignored, branch, task = working_repo.normalize(componentname, aspect, top_component.branch)
if revision:
m = _TAG_PAT.match(revision)
if not m:
raise Exception('%s is not a valid tag for pinning dependencies.' % revision)
assert(aspect)
top_component.dependencies.append(component.Component(componentname, branch, revision, aspect, parent=top_component))
if compOldDeps:
print('''Component %s/%s/%s has the old format for dependencies.
Please update dependencies in metadata.txt to match format found at:
https:// ... /working-with-code/concepts/dependencies''' % (top_component.name,top_component.reused_aspect,top_component.branch)) # TODO KIM refer to doc site
top_component.rank += len(top_component.dependencies)
for dep in top_component.dependencies:
if top_component.reused_aspect.startswith(component.BUILT_ASPECT_NAME):
dep.reused_aspect = top_component.reused_aspect
# We are suspicious that this optimization isn't working
if str(dep) not in read_deps or read_deps[str(dep)] != dep:
read_deps[str(dep)] = dep
dep = _get_deps(working_repo, platform, dep, code_root, read_deps, already_analyzed, use_master, check_vcs)
top_component.rank += dep.rank
return top_component
def _get_deps(working_repo,
platform,
top_component,
code_root,
read_deps,
already_analyzed,
use_master=False,
check_vcs=True):
if top_component.name == 'buildscripts':
top_component.reused_aspect = component.CODE_ASPECT_NAME
##TODO julie why would we do this?
## if top_component.reused_aspect == component.BUILT_ASPECT_NAME:
## interesting_branches = [b for b in working_repo.branches if b[1] == top_component.name and b[2].startswith(component.BUILT_ASPECT_NAME) and b[0] == top_component.branch]
## if not interesting_branches:
## top_component.reused_aspect = component.CODE_ASPECT_NAME
folder = ''
if (not top_component.revision) and code_root:
fldr = os.path.join(code_root, top_component.name)
if os.path.isdir(fldr):
if check_vcs and vcs.folder_is_tied_to_vcs(fldr):
output = vcs.get_status(fldr,
status_filter=lambda lbl: lbl ==
'modified' or lbl == 'added')
if output:
if 'modified' in output:
if METADATA_FILE in output['modified']:
folder = fldr
if 'added' in output:
if METADATA_FILE in output['added']:
folder = fldr
else:
folder = fldr
if folder:
if folder in already_analyzed:
return top_component #sections = already_analyzed[folder]
else:
print('\nLoading %s from %s.' % (METADATA_FILE, folder))
x = get_section_info_from_disk(MISC_SECTION, folder)
if 'terminal dependency' in x and top_component.reused_aspect.startswith(
component.BUILT_ASPECT_NAME):
return top_component
sections = get_section_info_from_disk(DEPENDENCIES_SECTION, folder)
already_analyzed[folder] = sections
elif check_vcs:
key = '%s:%s' % (top_component.name, top_component.reused_aspect
) #str(top_component)
if key in already_analyzed:
return top_component #sections = already_analyzed[key]
else:
x = get_section_info_from_vcs(MISC_SECTION, top_component,
working_repo, platform, use_master)
if 'terminal dependency' in x and top_component.reused_aspect.startswith(
component.BUILT_ASPECT_NAME):
return top_component
sections = get_section_info_from_vcs(DEPENDENCIES_SECTION,
top_component, working_repo,
platform, use_master)
already_analyzed[key] = sections
else:
return top_component
compOldDeps = False
for componentname, info in sections.iteritems():
componentname = componentname.strip()
aspect, revision, old = component.parse_component_info(info)
if aspect == component.BUILT_ASPECT_NAME:
aspect += "." + platform
if old:
compOldDeps = True
componentname, ignored, branch, task = working_repo.normalize(
componentname, aspect, top_component.branch)
if revision:
m = _TAG_PAT.match(revision)
if not m:
raise Exception(
'%s is not a valid tag for pinning dependencies.' %
revision)
assert (aspect)
top_component.dependencies.append(
component.Component(componentname,
branch,
revision,
aspect,
parent=top_component))
if compOldDeps:
print('''Component %s/%s/%s has the old format for dependencies.
Please update dependencies in metadata.txt to match format found at:
https:// ... /working-with-code/concepts/dependencies''' %
(top_component.name, top_component.reused_aspect,
top_component.branch)) # TODO KIM refer to doc site
top_component.rank += len(top_component.dependencies)
for dep in top_component.dependencies:
if top_component.reused_aspect.startswith(component.BUILT_ASPECT_NAME):
dep.reused_aspect = top_component.reused_aspect
# We are suspicious that this optimization isn't working
if str(dep) not in read_deps or read_deps[str(dep)] != dep:
read_deps[str(dep)] = dep
dep = _get_deps(working_repo, platform, dep, code_root, read_deps,
already_analyzed, use_master, check_vcs)
top_component.rank += dep.rank
return top_component