def Load(self, filename, default=None, expand_vars={}):
with log.info("Loading %s...", filename):
if not os.path.isfile(filename):
if default is None:
log.critical('File not found, exiting!')
log.info(
'To load defaults, specify default=<dict> to Properties.Load().'
)
sys.exit(1)
else:
log.warn('File not found, loading defaults.')
with open(filename, 'w') as f:
self.dumpTo(default, f)
with open(filename, 'r') as f:
for line in f:
if line.strip() == '':
continue
# Ignore comments.
if line.strip().startswith('#'):
continue
key, value = line.strip().split('=', 1)
if key in self.properties:
log.warn('Key "%s" already exists, overwriting!', key)
value = replace_vars(value, expand_vars)
self.properties[key] = value
if default is None:
return
for k, v in default.items():
if k not in self.properties:
self.properties[k] = v
log.info('Added default property %s = %s', k, v)
def find_process(pid):
for proc in psutil.process_iter():
try:
if proc.pid == pid:
if proc.status() == psutil.STATUS_ZOMBIE:
log.warn('Detected zombie process #%s, skipping.', proc.pid)
continue
return proc
except psutil.AccessDenied:
continue
return None
def find_process(pid):
for proc in psutil.process_iter():
try:
if proc.pid == pid:
if proc.status() == psutil.STATUS_ZOMBIE:
log.warn('Detected zombie process #%s, skipping.',
proc.pid)
continue
return proc
except psutil.AccessDenied:
continue
return None
def GetRepoState(self, remote=None, quiet=None):
if quiet is None:
quiet = self.quiet
self.current_branch = None
self.current_commit = None
if not os.path.isdir(self.path):
log.warn('Could not find %s.', self.path)
return
with Chdir(self.path, quiet=self.quiet):
if self.UpdateRemotes(remote, quiet=quiet):
self.current_branch = Git.GetBranch(quiet=not quiet)
self.current_commit = Git.GetCommit(short=False,
quiet=not quiet)
def Load(self,
filename,
merge=False,
defaults=None,
variables={},
verbose=False):
lh = NullIndenter()
if self.actual_filename is None:
self.actual_filename = filename if self.template_dir is None else os.path.join(
self.template_dir, filename)
if verbose:
lh = log.info("Loading %s...", filename)
with lh:
if not self.skip_defaults and not os.path.isfile(filename):
if defaults is None:
if verbose: log.error('Failed to load %s.', filename)
return False
else:
if verbose: log.warn('File not found, loading defaults.')
ensureDirExists(os.path.dirname(filename))
self.dump_to_file(filename, defaults)
rendered = ''
template: jinja2.Template = None
#print(repr(self.template_dir))
try:
if self.template_dir is not None:
template = self.environment.get_template(
os.path.basename(filename)
if self.template_dir is None else filename)
else:
with open(filename) as f:
template = self.environment.from_string(f.read())
rendered = template.render(variables)
except jinja2.exceptions.TemplateNotFound as tnf:
if verbose:
log.warn(
'Jinja2 failed to load %s (TemplateNotFound). Failing over to plain string.',
filename)
log.exception(tnf)
with codecs.open(filename, 'r', encoding=self.encoding) as f:
rendered = f.read()
#print(rendered)
newcfg = self.load_from_string(rendered)
if merge:
self.cfg = dict_merge(self.cfg, newcfg)
else:
self.cfg = newcfg
return True
def Load(self, filename, merge=False, defaults=None, variables={}):
with log.info("Loading %s...", filename):
if not os.path.isfile(filename):
if defaults is None:
log.error('Failed to load %s.', filename)
return False
else:
log.warn('File not found, loading defaults.')
with open(filename, 'w') as f:
yaml.dump(defaults, f, default_flow_style=False)
template = self.environment.get_template(filename)
rendered = template.render(variables)
newcfg = yaml.load(rendered)
if merge:
self.cfg = dict_merge(self.cfg, newcfg)
else:
self.cfg = newcfg
return True
def DownloadFile(url,
filename,
log_after=True,
print_status=True,
log_before=True,
**kwargs):
'''
Download a file from url to filename.
:param url:
HTTP URL to download. (SSL/TLS will also work, assuming the cert isn't broken.)
:param filename:
Path of the file to download to.
:param log_after:
Produce a log statement after the download completes (includes URL).
:param log_before:
Produce a log statement before the download starts.
:param print_status:
Prints live-updated status of the download progress. (May not work very well for piped or redirected output.)
:param session:
Requests session.
'''
#kwargs['headers'] = dict(DEFAULT_HEADERS, **kwargs.get('headers', {}))
r = None
session = kwargs.pop('session', requests)
try:
r = session.get(url, stream=True, **kwargs)
except requests.exceptions.ConnectionError as e:
logging.warning(e)
return False
except UnicodeEncodeError as e:
logging.warning(e)
return False
if (r.status_code == 404):
logging.warn('404 - Not Found: %s', url)
return False
elif (r.status_code != 200):
logging.warn("Error code: {0}".format(r.status_code))
return False
with open(filename, 'wb') as f:
file_size = int(r.headers.get('Content-Length', '-1'))
if file_size < 0:
if 'Content-Length' not in r.headers:
log.warn(
'Content-Length header was not received, expect progress bar weirdness.'
)
else:
log.warn('Content-Length header has invalid value: %r',
r.headers['Content-Length'])
if log_before:
log.info("Downloading: %s Bytes: %s" %
(filename, file_size if file_size > -1 else 'UNKNOWN'))
file_size_dl = 0
block_sz = 8192
progress = tqdm(total=file_size,
unit_scale=True,
unit_divisor=1024,
unit='B',
leave=False) if print_status else None
for buf in r.iter_content(block_sz):
if not buf or file_size == file_size_dl:
break
buf_len = len(buf)
file_size_dl += buf_len
f.write(buf)
if print_status:
#status = r"%10d/%10d [%3.2f%%]" % (file_size_dl, file_size, file_size_dl * 100. / file_size)
progress.update(buf_len)
#status = status + chr(8) * (len(status) + 1) - pre-2.6 method
#print(status, end='\r')
if progress is not None:
progress.close()
if log_after:
log.info('Downloaded %s to %s (%dB)', url, filename, file_size_dl)
def run(self, verbose=None):
if verbose is not None:
self.verbose = verbose
new_targets = []
for t in self.targets:
if t in new_targets:
log.warn('Target %s added more than once.', t)
else:
new_targets.append(t)
if self.checkForCycles():
return
keys = []
alldeps = []
for target in self.alltargets:
keys += target.provides()
alldeps += target.dependencies
target.built = False
alldeps = list(set(alldeps))
# Redundant
#for target in self.alltargets:
# for reqfile in callLambda(target.files):
# if reqfile in keys and reqfile not in target.dependencies:
# target.dependencies.append(reqfile)
loop = 0
#progress = tqdm(total=len(self.targets), unit='target', desc='Building', leave=False)
self.targetsCompleted = []
self.targetsDirty = []
while len(self.targets) > len(self.targetsCompleted) and loop < 100:
loop += 1
for bt in self.alltargets:
bt.maestro = self
if bt.canBuild(self, keys) and any([
target not in self.targetsCompleted
for target in bt.provides()
]):
try:
bt.try_build()
# progress.update(1)
self.targetsCompleted += bt.provides()
if bt.dirty:
self.targetsDirty += bt.provides()
except Exception as e:
bt._set_failed()
self._write_targets()
log.critical('An exception occurred, build halted.')
log.exception(e)
return
except KeyboardInterrupt:
bt._set_failed()
self._write_targets()
log.critical('Cancelled via KeyboardInterrupt.')
return
bt.built = True
log.debug('%d > %d, loop = %d', len(self.targets),
len(self.targetsCompleted), loop)
log.debug('%d > %d, loop = %d', len(self.targets),
len(self.targetsCompleted), loop)
# progress.close()
self._write_targets()
if loop >= 100:
incompleteTargets = [
t for t in self.targets if t not in self.targetsCompleted
]
if len(incompleteTargets) > 0:
with log.critical(
"Failed to resolve dependencies. The following targets are left unresolved. Exiting."
):
for t in incompleteTargets:
log.critical(t)
orphanDeps = [t for t in alldeps if t not in self.targets]
if len(orphanDeps) > 0:
with log.critical(
"Failed to resolve dependencies. The following dependencies are orphaned. Exiting."
):
for t in orphanDeps:
log.critical(t)
#sys.exit(1)
with log.info('Cleaning up...'):
cachefiles = []
for bt in self.alltargets:
cachefiles.append(os.path.basename(bt.getCacheFile()))
for filename in os.listdir(os.path.join(self.builddir, 'cache')):
if filename not in cachefiles:
filename = os.path.join(self.builddir, 'cache', filename)
log.debug('<red>RM</red> %s', filename)
os.remove(filename)
def errConnectionLost(self):
log.warn('%s Lost connection to stderr.', self._logPrefix())
def inConnectionLost(self):
log.warn('%s Lost connection to stdin.', self._logPrefix())
def _set_failed(self):
self.built = False
log.warn('Cleaning build artifacts due to failure...')
self.clean()
def errConnectionLost(self):
log.warn('%s Lost connection to stderr.', self._logPrefix())
def inConnectionLost(self):
log.warn('%s Lost connection to stdin.', self._logPrefix())