def Load(self, filename, default=None, expand_vars={}):
with log.info("Loading %s...", filename):
if not os.path.isfile(filename):
if default is None:
log.critical('File not found, exiting!')
log.info(
'To load defaults, specify default=<dict> to Properties.Load().'
)
sys.exit(1)
else:
log.warn('File not found, loading defaults.')
with open(filename, 'w') as f:
self.dumpTo(default, f)
with open(filename, 'r') as f:
for line in f:
if line.strip() == '':
continue
# Ignore comments.
if line.strip().startswith('#'):
continue
key, value = line.strip().split('=', 1)
if key in self.properties:
log.warn('Key "%s" already exists, overwriting!', key)
value = replace_vars(value, expand_vars)
self.properties[key] = value
if default is None:
return
for k, v in default.items():
if k not in self.properties:
self.properties[k] = v
log.info('Added default property %s = %s', k, v)
def decompressFile(archive, to='.', env=None):
'''
Decompresses the file to the current working directory.
Uses 7za for .7z and .rar files. (p7zip)
'''
if env is None:
env = ENV
#print('Trying to decompress ' + archive)
def get7ZA():
if PATH_7ZA is None:
PATH_7ZA = env.which('7za')
return PATH_7ZA
lc = archive.lower()
if lc.endswith('.tar.gz') or lc.endswith('.tgz'):
with tarfile.open(archive, mode='r:gz') as arch:
arch.extractall(to)
return True
elif lc.endswith('.bz2') or lc.endswith('.tbz'):
with tarfile.open(archive, mode='r:bz2') as arch:
arch.extractall(to)
return True
elif lc.endswith('.tar.xz'):
with tarfile.open(archive, mode='r:xz') as arch:
arch.extractall(to)
return True
elif lc.endswith('.tar.7z'):
cmd([get7ZA(), 'x', '-aoa', archive, '-o', to],
echo=True,
show_output=False,
critical=True)
with tarfile.open(archive[:-3], mode='r') as arch:
arch.extractall(to)
os.remove(archive[:-3])
return True
elif lc.endswith('.gz'):
with tarfile.open(archive, mode='r:gz') as arch:
arch.extractall(to)
elif lc.endswith('.7z'):
if PLATFORM == 'Windows':
archive = cygpath(archive)
cmd([get7ZA(), 'x', '-aoa', archive, '-o', to],
echo=True,
show_output=False,
critical=True)
elif lc.endswith('.zip'):
with zipfile.ZipFile(archive) as arch:
arch.extractall(to)
return True
elif lc.endswith('.rar'):
cmd([get7ZA(), 'x', '-aoa', archive, '-o', to],
echo=True,
show_output=False,
critical=True)
else:
log.critical('decompressFile(): Unknown file extension: %s', archive)
return False
def __exit__(self, typeName, value, traceback):
try:
if os.getcwdu() != self.pwd:
os.chdir(self.pwd)
if not self.quiet:
log.info('cd ' + self.pwd)
except:
log.critical('Failed to chdir to {}.'.format(self.pwd))
sys.exit(1)
return False
def __enter__(self):
try:
if os.getcwdu() != self.chdir:
os.chdir(self.chdir)
if not self.quiet:
log.info('cd ' + self.chdir)
except:
log.critical('Failed to chdir to {}.'.format(self.chdir))
sys.exit(1)
return self
def __exit__(self, typeName, value, traceback):
try:
if os.getcwdu() != self.pwd:
os.chdir(self.pwd)
if not self.quiet:
log.info('cd ' + self.pwd)
except:
log.critical('Failed to chdir to {}.'.format(self.pwd))
sys.exit(1)
return False
def __enter__(self):
try:
if os.getcwdu() != self.chdir:
os.chdir(self.chdir)
if not self.quiet:
log.info('cd ' + self.chdir)
except:
log.critical('Failed to chdir to {}.'.format(self.chdir))
sys.exit(1)
return self
def InstallDpkgPackages(packages):
import apt # IGNORE:import-error
with log.info('Checking dpkg packages...'):
cache = apt.Cache()
num_changes = 0
with cache.actiongroup():
for pkg in packages:
if pkg not in cache:
log.critical('UNKNOWN APT PACKAGE {}!'.format(pkg))
sys.exit(1)
package = cache[pkg]
if not package.is_installed:
package.mark_install()
num_changes += 1
if num_changes == 0:
log.info('No changes required, skipping.')
return
cache.commit(apt.progress.text.AcquireProgress(),
apt.progress.base.InstallProgress())
def InstallDpkgPackages(packages):
import apt # IGNORE:import-error
with log.info('Checking dpkg packages...'):
cache = apt.Cache()
num_changes = 0
with cache.actiongroup():
for pkg in packages:
if pkg not in cache:
log.critical('UNKNOWN APT PACKAGE {}!'.format(pkg))
sys.exit(1)
package = cache[pkg]
if not package.is_installed:
package.mark_install()
num_changes += 1
if num_changes == 0:
log.info('No changes required, skipping.')
return
cache.commit(apt.progress.text.AcquireProgress(),
apt.progress.base.InstallProgress())
def checkForCycles(self):
with log.info('Checking for dependency cycles...'):
# Using Tarjan's Strongly Connected Cycles algorithm
tg = TarjanGraph()
# First, I need to convert all BuildTargets to TarjanGraphVertexes.
for bt in self.alltargets:
refs = []
for depend in bt.dependencies:
if not isinstance(depend, str):
log.critical(
'Build target %s has invalid dependency %s.',
bt.name, depend)
sys.exit(1)
providers = []
for obt in self.alltargets:
if depend in obt.provides():
#log.info('%s provides %s, which %s needs', obt.name, depend, bt.name)
providers += [obt]
if len(providers) > 1:
log.warning(
'Build target %s has %d providers for dependency %s: %r',
bt.name, len(providers), depend,
[x.name for x in providers])
elif len(providers) == 0:
log.critical(
'Build target %s has no providers for dependency %s: %r',
bt.name, depend, [x.name for x in providers])
sys.exit(1)
refs.append(providers[-1].ID)
with log.debug('Dependency tree:'):
with log.debug('[%s] (%d,[%s])', bt.name, bt.ID,
', '.join([str(x) for x in refs])):
for refID in refs:
log.debug(self.alltargets[refID].name)
tg.add_edge(bt.ID, refs)
# Run the algo
tg.SCC()
# Sort through the crap that falls out
foundCycles = False
for cycle in tg.cycles:
if len(cycle) > 1:
log.critical('CYCLE FOUND: %r', [
'#{} ({})'.format(self.alltargets[btid].ID,
self.alltargets[btid].name)
for btid in cycle
])
foundCycles = True
return foundCycles
def OutputIsFatal(cls, output):
for line in output.splitlines():
if line.startswith('fatal:'):
log.critical(output)
return True
return False
def run(self, verbose=None):
if verbose is not None:
self.verbose = verbose
new_targets = []
for t in self.targets:
if t in new_targets:
log.warn('Target %s added more than once.', t)
else:
new_targets.append(t)
if self.checkForCycles():
return
keys = []
alldeps = []
for target in self.alltargets:
keys += target.provides()
alldeps += target.dependencies
target.built = False
alldeps = list(set(alldeps))
# Redundant
#for target in self.alltargets:
# for reqfile in callLambda(target.files):
# if reqfile in keys and reqfile not in target.dependencies:
# target.dependencies.append(reqfile)
loop = 0
#progress = tqdm(total=len(self.targets), unit='target', desc='Building', leave=False)
self.targetsCompleted = []
self.targetsDirty = []
while len(self.targets) > len(self.targetsCompleted) and loop < 100:
loop += 1
for bt in self.alltargets:
bt.maestro = self
if bt.canBuild(self, keys) and any([
target not in self.targetsCompleted
for target in bt.provides()
]):
try:
bt.try_build()
# progress.update(1)
self.targetsCompleted += bt.provides()
if bt.dirty:
self.targetsDirty += bt.provides()
except Exception as e:
bt._set_failed()
self._write_targets()
log.critical('An exception occurred, build halted.')
log.exception(e)
return
except KeyboardInterrupt:
bt._set_failed()
self._write_targets()
log.critical('Cancelled via KeyboardInterrupt.')
return
bt.built = True
log.debug('%d > %d, loop = %d', len(self.targets),
len(self.targetsCompleted), loop)
log.debug('%d > %d, loop = %d', len(self.targets),
len(self.targetsCompleted), loop)
# progress.close()
self._write_targets()
if loop >= 100:
incompleteTargets = [
t for t in self.targets if t not in self.targetsCompleted
]
if len(incompleteTargets) > 0:
with log.critical(
"Failed to resolve dependencies. The following targets are left unresolved. Exiting."
):
for t in incompleteTargets:
log.critical(t)
orphanDeps = [t for t in alldeps if t not in self.targets]
if len(orphanDeps) > 0:
with log.critical(
"Failed to resolve dependencies. The following dependencies are orphaned. Exiting."
):
for t in orphanDeps:
log.critical(t)
#sys.exit(1)
with log.info('Cleaning up...'):
cachefiles = []
for bt in self.alltargets:
cachefiles.append(os.path.basename(bt.getCacheFile()))
for filename in os.listdir(os.path.join(self.builddir, 'cache')):
if filename not in cachefiles:
filename = os.path.join(self.builddir, 'cache', filename)
log.debug('<red>RM</red> %s', filename)
os.remove(filename)