def setup(args):
if len(args) not in [2, 3]:
sys.stderr.write("Usage: %s DEFINITION_FILE [ARCH]\n\n" % sys.argv[0])
sys.exit(1)
settings['target'] = args[1]
if len(args) == 3:
arch = args[2]
else:
arch = platform.machine()
if arch in ('mips', 'mips64'):
if arch == 'mips':
arch = 'mips32'
if sys.byteorder == 'big':
arch = arch + 'b'
else:
arch = arch + 'l'
settings['arch'] = arch
warnings.formatwarning = warning_handler
settings_file = './ybd.conf'
if not os.path.exists(settings_file):
settings_file = os.path.join(os.path.dirname(__file__), 'ybd.conf')
with open(settings_file) as f:
text = f.read()
for key, value in yaml.safe_load(text).items():
settings[key] = value
settings['total'] = settings['tasks'] = settings['counter'] = 0
settings['pid'] = os.getpid()
settings['program'] = os.path.basename(args[0])
settings['program-version'] = get_version(os.path.dirname(__file__))
settings['defdir'] = os.getcwd()
settings['extsdir'] = os.path.join(settings['defdir'], 'extensions')
settings['def-version'] = get_version('.')
dirs = [ 'artifacts', 'ccache_dir', 'deployment', 'gits', 'tidy', 'tmp' ]
settings['base'] = os.path.join(xdg_cache_home, settings['base'])
for directory in dirs:
try:
settings[directory] = os.path.join(settings['base'], directory)
os.makedirs(settings[directory])
except OSError:
if not os.path.isdir(settings[directory]):
exit(target, 'ERROR: Can not find or create',
settings[directory])
# git replace means we can't trust that just the sha1 of a branch
# is enough to say what it contains, so we turn it off by setting
# the right flag in an environment variable.
os.environ['GIT_NO_REPLACE_OBJECTS'] = '1'
settings['max-jobs'] = max(int(cpu_count() * 1.5 + 0.5), 1)
def setup(target, arch):
warnings.formatwarning = warning_handler
try:
settings_file = './ybd.def'
if not os.path.exists(settings_file):
settings_file = os.path.join(os.path.dirname(__file__), 'ybd.def')
with open(settings_file) as f:
text = f.read()
for key, value in yaml.safe_load(text).items():
settings[key] = value
settings['pid'] = os.getpid()
with open(os.devnull, "w") as fnull:
if call(['git', 'describe', '--all'], stdout=fnull, stderr=fnull):
exit(target, 'ERROR: not a git repo', os.getcwd())
settings['ybd-version'] = get_version(os.path.dirname(__file__))
settings['defdir'] = os.getcwd()
settings['extsdir'] = os.path.join(settings['defdir'], 'extensions')
settings['def-ver'] = get_version('.')
settings['target'] = target
settings['arch'] = arch
for directory in ['base', 'caches', 'artifacts', 'gits', 'tmp',
'ccache_dir', 'deployment']:
try:
os.makedirs(settings[directory])
except OSError:
if not os.path.isdir(settings[directory]):
exit(target, 'ERROR: Can not find or create',
settings[directory])
# git replace means we can't trust that just the sha1 of a branch
# is enough to say what it contains, so we turn it off by setting
# the right flag in an environment variable.
os.environ['GIT_NO_REPLACE_OBJECTS'] = '1'
settings['max-jobs'] = max(int(cpu_count() * 1.5 + 0.5), 1)
yield
finally:
log(target, 'Finished')
def setup(args):
os.environ['LANG'] = 'en_US.UTF-8'
config['start-time'] = datetime.datetime.now()
config['program'] = os.path.basename(args[0])
config['my-version'] = get_version(os.path.dirname(__file__))
log('SETUP', '%s version is' % config['program'], config['my-version'])
if len(args) != 3:
sys.stdout.write("\nUsage: %s DEFINITION_FILE ARCH\n\n" % sys.argv[0])
sys.exit(1)
log('SETUP', 'Running %s in' % args[0], os.getcwd())
config['target'] = os.path.basename(os.path.splitext(args[1])[0])
config['arch'] = args[2]
config['sandboxes'] = []
config['overlaps'] = []
config['new-overlaps'] = []
warnings.formatwarning = warning_handler
# Suppress multiple instances of the same warning.
warnings.simplefilter('once', append=True)
# dump any applicable environment variables into a config file
with open('./ybd.environment', 'w') as f:
for key in os.environ:
if key[:4] == "YBD_":
f.write(key[4:] + ": " + os.environ.get(key) + '\n')
# load config files in reverse order of precedence
load_configs([
os.path.join(os.getcwd(), 'ybd.environment'),
os.path.join(os.getcwd(), 'ybd.conf'),
os.path.join(os.path.dirname(__file__), '..', 'ybd.conf'),
os.path.join(os.path.dirname(__file__), 'config', 'ybd.conf')])
if config.get('kbas-url', 'http://foo.bar/') == 'http://foo.bar/':
config.pop('kbas-url')
if config.get('kbas-url'):
if not config['kbas-url'].endswith('/'):
config['kbas-url'] += '/'
config['total'] = config['tasks'] = config['counter'] = 0
config['systems'] = config['strata'] = config['chunks'] = 0
config['reproduced'] = []
config['keys'] = []
config['pid'] = os.getpid()
config['def-version'] = get_version('.')
config['defdir'] = os.getcwd()
config['extsdir'] = os.path.join(config['defdir'], 'extensions')
if config.get('manifest') is True:
config['manifest'] = os.path.join(config['defdir'],
os.path.basename(config['target']) +
'.manifest')
try:
os.remove(config['manifest'])
except OSError:
pass
base_dir = os.environ.get('XDG_CACHE_HOME') or os.path.expanduser('~')
config.setdefault('base',
os.path.join(base_dir, config['directories']['base']))
for directory, path in config.get('directories', {}).items():
try:
if config.get(directory) is None:
if path is None:
path = os.path.join(config.get('base', '/'), directory)
config[directory] = path
os.makedirs(config[directory])
except OSError:
if not os.path.isdir(config[directory]):
log('SETUP', 'Cannot find or create', config[directory],
exit=True)
log('SETUP', '%s is directory for' % config[directory], directory)
# git replace means we can't trust that just the sha1 of a branch
# is enough to say what it contains, so we turn it off by setting
# the right flag in an environment variable.
os.environ['GIT_NO_REPLACE_OBJECTS'] = '1'
if 'max-jobs' not in config:
config['max-jobs'] = cpu_count()
if 'instances' not in config:
# based on some testing (mainly on AWS), maximum effective
# max-jobs value seems to be around 8-10 if we have enough cores
# users should set values based on workload and build infrastructure
# FIXME: more testing :)
if cpu_count() >= 10:
config['instances'] = 1 + (cpu_count() / 10)
config['max-jobs'] = cpu_count() / config['instances']
config['pid'] = os.getpid()
config['counter'] = Counter()
log('SETUP', 'Max-jobs is set to', config['max-jobs'])
def setup(args, original_cwd=""):
os.environ['LANG'] = 'en_US.UTF-8'
config['start-time'] = datetime.datetime.now()
config['program'] = os.path.basename(args[0])
config['my-version'] = get_version(os.path.dirname(__file__))
log('SETUP', '%s version is' % config['program'], config['my-version'])
if len(args) != 3:
sys.stdout.write("\nUsage: %s DEFINITION_FILE ARCH\n\n" % sys.argv[0])
sys.exit(1)
log('SETUP', 'Running %s in' % args[0], os.getcwd())
config['target'] = os.path.basename(os.path.splitext(args[1])[0])
config['arch'] = args[2]
config['sandboxes'] = []
config['overlaps'] = []
config['new-overlaps'] = []
warnings.formatwarning = warning_handler
# Suppress multiple instances of the same warning.
warnings.simplefilter('once', append=True)
# dump any applicable environment variables into a config file
with open('./ybd.environment', 'w') as f:
for key in os.environ:
if key[:4] == "YBD_":
f.write(key[4:] + ": " + os.environ.get(key) + '\n')
# load config files in reverse order of precedence
load_configs([
os.path.join(os.getcwd(), 'ybd.environment'),
os.path.join(os.getcwd(), 'ybd.conf'),
os.path.join(original_cwd, 'ybd.conf'),
os.path.join(os.path.dirname(__file__), '..', 'ybd.conf'),
os.path.join(os.path.dirname(__file__), 'config', 'ybd.conf')])
if not os.geteuid() == 0 and config.get('mode') == 'normal':
log('SETUP', '%s needs root permissions' % sys.argv[0], exit=True)
if config.get('kbas-url', 'http://foo.bar/') == 'http://foo.bar/':
config.pop('kbas-url')
if config.get('kbas-url'):
if not config['kbas-url'].endswith('/'):
config['kbas-url'] += '/'
config['total'] = config['tasks'] = config['counter'] = 0
config['systems'] = config['strata'] = config['chunks'] = 0
config['reproduced'] = []
config['keys'] = []
config['pid'] = os.getpid()
config['def-version'] = get_version('.')
config['defdir'] = os.getcwd()
config['extsdir'] = os.path.join(config['defdir'], 'extensions')
if config.get('manifest') is True:
config['manifest'] = os.path.join(config['defdir'],
os.path.basename(config['target']) +
'.manifest')
try:
os.remove(config['manifest'])
except OSError:
pass
base_dir = os.environ.get('XDG_CACHE_HOME') or os.path.expanduser('~')
config.setdefault('base',
os.path.join(base_dir, config['directories']['base']))
for directory, path in config.get('directories', {}).items():
try:
if config.get(directory) is None:
if path is None:
path = os.path.join(config.get('base', '/'), directory)
config[directory] = path
os.makedirs(config[directory])
except OSError:
if not os.path.isdir(config[directory]):
log('SETUP', 'Cannot find or create', config[directory],
exit=True)
log('SETUP', '%s is directory for' % config[directory], directory)
# git replace means we can't trust that just the sha1 of a branch
# is enough to say what it contains, so we turn it off by setting
# the right flag in an environment variable.
os.environ['GIT_NO_REPLACE_OBJECTS'] = '1'
if 'max-jobs' not in config:
config['max-jobs'] = cpu_count()
if 'instances' not in config:
# based on some testing (mainly on AWS), maximum effective
# max-jobs value seems to be around 8-10 if we have enough cores
# users should set values based on workload and build infrastructure
# FIXME: more testing :)
if cpu_count() >= 10:
config['instances'] = 1 + (cpu_count() / 10)
config['max-jobs'] = cpu_count() / config['instances']
config['pid'] = os.getpid()
config['counter'] = Counter()
log('SETUP', 'Max-jobs is set to', config['max-jobs'])
def setup(args):
config['start-time'] = datetime.datetime.now()
config['program'] = os.path.basename(args[0])
config['my-version'] = get_version(os.path.dirname(__file__))
if len(args) != 3:
sys.stdout.write("%s version is %s\n\n" % (config['program'],
config['my-version']))
sys.stdout.write("Usage: %s DEFINITION_FILE ARCH\n\n" % sys.argv[0])
sys.exit(0)
log('SETUP', 'Running %s in' % args[0], os.getcwd())
config['target'] = os.path.basename(os.path.splitext(args[1])[0])
config['arch'] = args[2]
config['sandboxes'] = []
config['overlaps'] = []
config['new-overlaps'] = []
warnings.formatwarning = warning_handler
# Suppress multiple instances of the same warning.
warnings.simplefilter('once', append=True)
# dump any applicable environment variables into a config file
with open('./ybd.environment', 'w') as f:
for key in os.environ:
if key[:4] == "YBD_":
f.write(key[4:] + ": " + os.environ.get(key) + '\n')
# load config files in reverse order of precedence
load_configs([
os.path.join(os.getcwd(), 'ybd.environment'),
os.path.join(os.getcwd(), 'ybd.conf'),
os.path.join(os.path.dirname(__file__), '..', 'ybd.conf'),
os.path.join(os.path.dirname(__file__), 'config', 'ybd.conf')])
if config.get('kbas-url', 'http://foo.bar/') == 'http://foo.bar/':
config.pop('kbas-url')
if config.get('kbas-url'):
if not config['kbas-url'].endswith('/'):
config['kbas-url'] += '/'
config['total'] = config['tasks'] = config['counter'] = 0
config['reproduced'] = []
config['keys'] = []
config['pid'] = os.getpid()
config['def-version'] = get_version('.')
config['defdir'] = os.getcwd()
config['extsdir'] = os.path.join(config['defdir'], 'extensions')
base_dir = os.environ.get('XDG_CACHE_HOME') or os.path.expanduser('~')
config.setdefault('base',
os.path.join(base_dir, config['directories']['base']))
for directory, path in config.get('directories', {}).items():
try:
if config.get(directory) is None:
if path is None:
path = os.path.join(config.get('base', '/'), directory)
config[directory] = path
os.makedirs(config[directory])
except OSError:
if not os.path.isdir(config[directory]):
exit('SETUP', 'ERROR: Can not find or create',
config[directory])
log('SETUP', '%s is directory for' % config[directory], directory)
# git replace means we can't trust that just the sha1 of a branch
# is enough to say what it contains, so we turn it off by setting
# the right flag in an environment variable.
os.environ['GIT_NO_REPLACE_OBJECTS'] = '1'
if not config.get('max-jobs'):
config['max-jobs'] = cpu_count() / config.get('instances', 1)
config['pid'] = os.getpid()
config['counter'] = Counter(config['pid'])
log('SETUP', '%s version is' % config['program'], config['my-version'])
log('SETUP', 'Max-jobs is set to', config['max-jobs'])
def setup(args):
if len(args) != 3:
sys.stderr.write("Usage: %s DEFINITION_FILE ARCH\n\n" % sys.argv[0])
sys.exit(1)
log('SETUP', 'Running %s in' % args[0], os.getcwd())
config['start-time'] = datetime.datetime.now()
config['target'] = os.path.basename(os.path.splitext(args[1])[0])
config['arch'] = args[2]
config['sandboxes'] = []
warnings.formatwarning = warning_handler
# Suppress multiple instances of the same warning.
warnings.simplefilter('once', append=True)
# load config files in reverse order of precedence
load_configs([
os.path.join(os.getcwd(), 'ybd.conf'),
os.path.join(os.path.dirname(__file__), '..', 'ybd.conf'),
os.path.join(os.path.dirname(__file__), 'config', 'ybd.conf')
])
if config.get('kbas-url', 'http://foo.bar/') == 'http://foo.bar/':
config.pop('kbas-url')
if config.get('kbas-url'):
if not config['kbas-url'].endswith('/'):
config['kbas-url'] += '/'
config['total'] = config['tasks'] = config['counter'] = 0
config['pid'] = os.getpid()
config['program'] = os.path.basename(args[0])
config['my-version'] = get_version(os.path.dirname(__file__))
config['def-version'] = get_version('.')
config['defdir'] = os.getcwd()
config['extsdir'] = os.path.join(config['defdir'], 'extensions')
base_dir = os.environ.get('XDG_CACHE_HOME') or os.path.expanduser('~')
if config.get('base', None) is None:
config['base'] = os.path.join(base_dir, config['directories']['base'])
for directory, path in config.get('directories', {}).items():
try:
if config.get(directory) is None:
if path is None:
path = os.path.join(config.get('base', '/'), directory)
config[directory] = path
os.makedirs(config[directory])
except OSError:
if not os.path.isdir(config[directory]):
exit('SETUP', 'ERROR: Can not find or create',
config[directory])
log('SETUP', '%s is directory for' % config[directory], directory)
# git replace means we can't trust that just the sha1 of a branch
# is enough to say what it contains, so we turn it off by setting
# the right flag in an environment variable.
os.environ['GIT_NO_REPLACE_OBJECTS'] = '1'
if not config.get('max-jobs'):
config['max-jobs'] = cpu_count() / config.get('instances', 1)
config['pid'] = os.getpid()
config['counter'] = Counter(config['pid'])
log('SETUP', '%s version is' % config['program'], config['my-version'])
log('SETUP', 'Max-jobs is set to', config['max-jobs'])
def setup(args):
if len(args) not in [2, 3]:
sys.stderr.write("Usage: %s DEFINITION_FILE [ARCH]\n\n" % sys.argv[0])
sys.exit(1)
config['target'] = os.path.basename(os.path.splitext(args[1])[0])
if len(args) == 3:
arch = args[2]
else:
arch = platform.machine()
if arch in ('mips', 'mips64'):
if arch == 'mips':
arch = 'mips32'
if sys.byteorder == 'big':
arch = arch + 'b'
else:
arch = arch + 'l'
config['arch'] = arch
warnings.formatwarning = warning_handler
# Suppress multiple instances of the same warning.
warnings.simplefilter('once', append=True)
conf_file = './ybd.conf'
if not os.path.exists(conf_file):
conf_file = os.path.join(os.path.dirname(__file__), 'ybd.conf')
with open(conf_file) as f:
text = f.read()
for key, value in yaml.safe_load(text).items():
config[key] = value
config['total'] = config['tasks'] = config['counter'] = 0
config['pid'] = os.getpid()
config['program'] = os.path.basename(args[0])
config['my-version'] = get_version(os.path.dirname(__file__))
config['defdir'] = os.getcwd()
config['extsdir'] = os.path.join(config['defdir'], 'extensions')
config['def-version'] = get_version('.')
dirs = ['artifacts', 'ccache_dir', 'deployment', 'gits', 'tidy', 'tmp']
config['base'] = os.path.join(xdg_cache_home, config['base'])
for directory in dirs:
try:
config[directory] = os.path.join(config['base'], directory)
os.makedirs(config[directory])
except OSError:
if not os.path.isdir(config[directory]):
exit(config['target'], 'ERROR: Can not find or create',
config[directory])
# git replace means we can't trust that just the sha1 of a branch
# is enough to say what it contains, so we turn it off by setting
# the right flag in an environment variable.
os.environ['GIT_NO_REPLACE_OBJECTS'] = '1'
cores = cpu_count() / config.get('instances', 1)
if not config.get('max-jobs'):
config['max-jobs'] = max(int(cores * 1.5 + 0.5), 1)
log('SETUP', '%s version is' % config['program'], config['my-version'])
log('SETUP', 'Default configuration is:\n\n%s' % text)
log('SETUP', 'Max-jobs is set to', config['max-jobs'])
def setup(program, target, arch, mode, original_cwd=""):
os.environ['LANG'] = 'en_US.UTF-8'
# Installation of git-lfs on a system can pollute /etc/gitconfig. Setting
# GIT_CONFIG_NOSYSTEM so ybd will ignore the system config.
os.environ['GIT_CONFIG_NOSYSTEM'] = "1"
config['start-time'] = datetime.datetime.now()
config['program'] = os.path.basename(program)
config['my-version'] = get_version(os.path.dirname(__file__))
log('SETUP', '%s version is' % config['program'], config['my-version'])
log('SETUP', 'Running %s in' % program, os.getcwd())
config['filename'] = os.path.basename(target)
config['target'] = os.path.basename(os.path.splitext(target)[0])
config['arch'] = arch
config['sandboxes'] = []
config['overlaps'] = []
config['new-overlaps'] = []
warnings.formatwarning = warning_handler
# Suppress multiple instances of the same warning.
warnings.simplefilter('once', append=True)
# dump any applicable environment variables into a config file
with open('./ybd.environment', 'w') as f:
for key in os.environ:
if key[:4] == "YBD_":
f.write(key[4:] + ": " + os.environ.get(key) + '\n')
# load config files in reverse order of precedence
load_configs([
os.path.join(os.getcwd(), 'ybd.environment'),
os.path.join(os.getcwd(), 'ybd.conf'),
os.path.join(original_cwd, 'ybd.conf'),
os.path.join(os.path.dirname(__file__), '..', 'ybd.conf'),
os.path.join(os.path.dirname(__file__), 'config', 'ybd.conf')])
# After loading configuration, override the 'mode' configuration only
# if it was specified on the command line
if mode is not None:
config['mode'] = mode
if not os.geteuid() == 0 and config.get('mode') == 'normal':
log('SETUP', '%s needs root permissions' % program, exit=True)
if config.get('kbas-url', 'http://foo.bar/') == 'http://foo.bar/':
config.pop('kbas-url')
if config.get('kbas-url'):
if not config['kbas-url'].endswith('/'):
config['kbas-url'] += '/'
config['total'] = config['tasks'] = config['counter'] = 0
config['systems'] = config['strata'] = config['chunks'] = 0
config['reproduced'] = []
config['keys'] = []
config['pid'] = os.getpid()
config['def-version'] = get_version('.')
config['defdir'] = os.getcwd()
config['extsdir'] = os.path.join(config['defdir'], 'extensions')
if config.get('manifest') is True:
config['manifest'] = os.path.join(config['defdir'],
os.path.basename(config['target']) +
'.manifest')
try:
os.remove(config['manifest'])
except OSError:
pass
base_dir = os.environ.get('XDG_CACHE_HOME') or os.path.expanduser('~')
config.setdefault('base',
os.path.join(base_dir, config['directories']['base']))
for directory, path in config.get('directories', {}).items():
try:
if config.get(directory) is None:
if path is None:
path = os.path.join(config.get('base', '/'), directory)
config[directory] = path
os.makedirs(config[directory])
except OSError:
if not os.path.isdir(config[directory]):
log('SETUP', 'Cannot find or create', config[directory],
exit=True)
log('SETUP', '%s is directory for' % config[directory], directory)
# git replace means we can't trust that just the sha1 of a branch
# is enough to say what it contains, so we turn it off by setting
# the right flag in an environment variable.
os.environ['GIT_NO_REPLACE_OBJECTS'] = '1'
if 'max-jobs' not in config:
config['max-jobs'] = cpu_count()
if 'instances' not in config:
# based on some testing (mainly on AWS), maximum effective
# max-jobs value seems to be around 8-10 if we have enough cores
# users should set values based on workload and build infrastructure
# FIXME: more testing :)
if cpu_count() >= 10:
config['instances'] = 1 + (cpu_count() / 10)
config['max-jobs'] = cpu_count() / config['instances']
config['pid'] = os.getpid()
config['counter'] = Counter()
log('SETUP', 'Max-jobs is set to', config['max-jobs'])