Example #1
0
def setup(args):
    if len(args) not in [2, 3]:
        sys.stderr.write("Usage: %s DEFINITION_FILE [ARCH]\n\n" % sys.argv[0])
        sys.exit(1)

    settings['target'] = args[1]
    if len(args) == 3:
        arch = args[2]
    else:
        arch = platform.machine()
        if arch in ('mips', 'mips64'):
            if arch == 'mips':
                arch = 'mips32'
            if sys.byteorder == 'big':
                arch = arch + 'b'
            else:
                arch = arch + 'l'
    settings['arch'] = arch

    warnings.formatwarning = warning_handler
    settings_file = './ybd.conf'
    if not os.path.exists(settings_file):
        settings_file = os.path.join(os.path.dirname(__file__), 'ybd.conf')
    with open(settings_file) as f:
        text = f.read()
    for key, value in yaml.safe_load(text).items():
        settings[key] = value
    settings['total'] = settings['tasks'] = settings['counter'] = 0
    settings['pid'] = os.getpid()
    settings['program'] = os.path.basename(args[0])
    settings['program-version'] = get_version(os.path.dirname(__file__))
    settings['defdir'] = os.getcwd()
    settings['extsdir'] = os.path.join(settings['defdir'], 'extensions')
    settings['def-version'] = get_version('.')

    dirs = [ 'artifacts', 'ccache_dir', 'deployment', 'gits', 'tidy', 'tmp' ]
    settings['base'] = os.path.join(xdg_cache_home, settings['base'])
    for directory in dirs:
        try:
            settings[directory] = os.path.join(settings['base'], directory)
            os.makedirs(settings[directory])
        except OSError:
            if not os.path.isdir(settings[directory]):
                exit(target, 'ERROR: Can not find or create',
                     settings[directory])

    # git replace means we can't trust that just the sha1 of a branch
    # is enough to say what it contains, so we turn it off by setting
    # the right flag in an environment variable.
    os.environ['GIT_NO_REPLACE_OBJECTS'] = '1'

    settings['max-jobs'] = max(int(cpu_count() * 1.5 + 0.5), 1)
Example #2
0
def setup(target, arch):
    warnings.formatwarning = warning_handler

    try:
        settings_file = './ybd.def'
        if not os.path.exists(settings_file):
            settings_file = os.path.join(os.path.dirname(__file__), 'ybd.def')
        with open(settings_file) as f:
            text = f.read()
        for key, value in yaml.safe_load(text).items():
            settings[key] = value
        settings['pid'] = os.getpid()
        with open(os.devnull, "w") as fnull:
            if call(['git', 'describe', '--all'], stdout=fnull, stderr=fnull):
                exit(target, 'ERROR: not a git repo', os.getcwd())

        settings['ybd-version'] = get_version(os.path.dirname(__file__))
        settings['defdir'] = os.getcwd()
        settings['extsdir'] = os.path.join(settings['defdir'], 'extensions')
        settings['def-ver'] = get_version('.')
        settings['target'] = target
        settings['arch'] = arch

        for directory in ['base', 'caches', 'artifacts', 'gits', 'tmp',
                          'ccache_dir', 'deployment']:
            try:
                os.makedirs(settings[directory])
            except OSError:
                if not os.path.isdir(settings[directory]):
                    exit(target, 'ERROR: Can not find or create',
                         settings[directory])

        # git replace means we can't trust that just the sha1 of a branch
        # is enough to say what it contains, so we turn it off by setting
        # the right flag in an environment variable.
        os.environ['GIT_NO_REPLACE_OBJECTS'] = '1'

        settings['max-jobs'] = max(int(cpu_count() * 1.5 + 0.5), 1)
        yield

    finally:
        log(target, 'Finished')
Example #3
0
File: app.py Project: jjardon/ybd
def setup(args):
    os.environ['LANG'] = 'en_US.UTF-8'
    config['start-time'] = datetime.datetime.now()
    config['program'] = os.path.basename(args[0])
    config['my-version'] = get_version(os.path.dirname(__file__))
    log('SETUP', '%s version is' % config['program'], config['my-version'])
    if len(args) != 3:
        sys.stdout.write("\nUsage: %s DEFINITION_FILE ARCH\n\n" % sys.argv[0])
        sys.exit(1)

    log('SETUP', 'Running %s in' % args[0], os.getcwd())
    config['target'] = os.path.basename(os.path.splitext(args[1])[0])
    config['arch'] = args[2]
    config['sandboxes'] = []
    config['overlaps'] = []
    config['new-overlaps'] = []

    warnings.formatwarning = warning_handler
    # Suppress multiple instances of the same warning.
    warnings.simplefilter('once', append=True)

    # dump any applicable environment variables into a config file
    with open('./ybd.environment', 'w') as f:
        for key in os.environ:
            if key[:4] == "YBD_":
                f.write(key[4:] + ": " + os.environ.get(key) + '\n')

    # load config files in reverse order of precedence
    load_configs([
        os.path.join(os.getcwd(), 'ybd.environment'),
        os.path.join(os.getcwd(), 'ybd.conf'),
        os.path.join(os.path.dirname(__file__), '..', 'ybd.conf'),
        os.path.join(os.path.dirname(__file__), 'config', 'ybd.conf')])

    if config.get('kbas-url', 'http://foo.bar/') == 'http://foo.bar/':
        config.pop('kbas-url')
    if config.get('kbas-url'):
        if not config['kbas-url'].endswith('/'):
            config['kbas-url'] += '/'

    config['total'] = config['tasks'] = config['counter'] = 0
    config['systems'] = config['strata'] = config['chunks'] = 0
    config['reproduced'] = []
    config['keys'] = []
    config['pid'] = os.getpid()
    config['def-version'] = get_version('.')

    config['defdir'] = os.getcwd()
    config['extsdir'] = os.path.join(config['defdir'], 'extensions')
    if config.get('manifest') is True:
        config['manifest'] = os.path.join(config['defdir'],
                                          os.path.basename(config['target']) +
                                          '.manifest')
        try:
            os.remove(config['manifest'])
        except OSError:
            pass

    base_dir = os.environ.get('XDG_CACHE_HOME') or os.path.expanduser('~')
    config.setdefault('base',
                      os.path.join(base_dir, config['directories']['base']))
    for directory, path in config.get('directories', {}).items():
        try:
            if config.get(directory) is None:
                if path is None:
                    path = os.path.join(config.get('base', '/'), directory)
                config[directory] = path
            os.makedirs(config[directory])
        except OSError:
            if not os.path.isdir(config[directory]):
                log('SETUP', 'Cannot find or create', config[directory],
                    exit=True)

        log('SETUP', '%s is directory for' % config[directory], directory)

    # git replace means we can't trust that just the sha1 of a branch
    # is enough to say what it contains, so we turn it off by setting
    # the right flag in an environment variable.
    os.environ['GIT_NO_REPLACE_OBJECTS'] = '1'

    if 'max-jobs' not in config:
        config['max-jobs'] = cpu_count()

    if 'instances' not in config:
        # based on some testing (mainly on AWS), maximum effective
        # max-jobs value seems to be around 8-10 if we have enough cores
        # users should set values based on workload and build infrastructure
        # FIXME: more testing :)
        if cpu_count() >= 10:
            config['instances'] = 1 + (cpu_count() / 10)
            config['max-jobs'] = cpu_count() / config['instances']

    config['pid'] = os.getpid()
    config['counter'] = Counter()
    log('SETUP', 'Max-jobs is set to', config['max-jobs'])
Example #4
0
def setup(args, original_cwd=""):
    os.environ['LANG'] = 'en_US.UTF-8'
    config['start-time'] = datetime.datetime.now()
    config['program'] = os.path.basename(args[0])
    config['my-version'] = get_version(os.path.dirname(__file__))
    log('SETUP', '%s version is' % config['program'], config['my-version'])
    if len(args) != 3:
        sys.stdout.write("\nUsage: %s DEFINITION_FILE ARCH\n\n" % sys.argv[0])
        sys.exit(1)

    log('SETUP', 'Running %s in' % args[0], os.getcwd())
    config['target'] = os.path.basename(os.path.splitext(args[1])[0])
    config['arch'] = args[2]
    config['sandboxes'] = []
    config['overlaps'] = []
    config['new-overlaps'] = []

    warnings.formatwarning = warning_handler
    # Suppress multiple instances of the same warning.
    warnings.simplefilter('once', append=True)

    # dump any applicable environment variables into a config file
    with open('./ybd.environment', 'w') as f:
        for key in os.environ:
            if key[:4] == "YBD_":
                f.write(key[4:] + ": " + os.environ.get(key) + '\n')

    # load config files in reverse order of precedence
    load_configs([
        os.path.join(os.getcwd(), 'ybd.environment'),
        os.path.join(os.getcwd(), 'ybd.conf'),
        os.path.join(original_cwd, 'ybd.conf'),
        os.path.join(os.path.dirname(__file__), '..', 'ybd.conf'),
        os.path.join(os.path.dirname(__file__), 'config', 'ybd.conf')])

    if not os.geteuid() == 0 and config.get('mode') == 'normal':
        log('SETUP', '%s needs root permissions' % sys.argv[0], exit=True)

    if config.get('kbas-url', 'http://foo.bar/') == 'http://foo.bar/':
        config.pop('kbas-url')
    if config.get('kbas-url'):
        if not config['kbas-url'].endswith('/'):
            config['kbas-url'] += '/'

    config['total'] = config['tasks'] = config['counter'] = 0
    config['systems'] = config['strata'] = config['chunks'] = 0
    config['reproduced'] = []
    config['keys'] = []
    config['pid'] = os.getpid()
    config['def-version'] = get_version('.')

    config['defdir'] = os.getcwd()
    config['extsdir'] = os.path.join(config['defdir'], 'extensions')
    if config.get('manifest') is True:
        config['manifest'] = os.path.join(config['defdir'],
                                          os.path.basename(config['target']) +
                                          '.manifest')
        try:
            os.remove(config['manifest'])
        except OSError:
            pass

    base_dir = os.environ.get('XDG_CACHE_HOME') or os.path.expanduser('~')
    config.setdefault('base',
                      os.path.join(base_dir, config['directories']['base']))
    for directory, path in config.get('directories', {}).items():
        try:
            if config.get(directory) is None:
                if path is None:
                    path = os.path.join(config.get('base', '/'), directory)
                config[directory] = path
            os.makedirs(config[directory])
        except OSError:
            if not os.path.isdir(config[directory]):
                log('SETUP', 'Cannot find or create', config[directory],
                    exit=True)

        log('SETUP', '%s is directory for' % config[directory], directory)

    # git replace means we can't trust that just the sha1 of a branch
    # is enough to say what it contains, so we turn it off by setting
    # the right flag in an environment variable.
    os.environ['GIT_NO_REPLACE_OBJECTS'] = '1'

    if 'max-jobs' not in config:
        config['max-jobs'] = cpu_count()

    if 'instances' not in config:
        # based on some testing (mainly on AWS), maximum effective
        # max-jobs value seems to be around 8-10 if we have enough cores
        # users should set values based on workload and build infrastructure
        # FIXME: more testing :)
        if cpu_count() >= 10:
            config['instances'] = 1 + (cpu_count() / 10)
            config['max-jobs'] = cpu_count() / config['instances']

    config['pid'] = os.getpid()
    config['counter'] = Counter()
    log('SETUP', 'Max-jobs is set to', config['max-jobs'])
Example #5
0
File: app.py Project: nuxeh/ybd
def setup(args):
    config['start-time'] = datetime.datetime.now()
    config['program'] = os.path.basename(args[0])
    config['my-version'] = get_version(os.path.dirname(__file__))
    if len(args) != 3:
        sys.stdout.write("%s version is %s\n\n" % (config['program'],
                         config['my-version']))
        sys.stdout.write("Usage: %s DEFINITION_FILE ARCH\n\n" % sys.argv[0])
        sys.exit(0)

    log('SETUP', 'Running %s in' % args[0], os.getcwd())
    config['target'] = os.path.basename(os.path.splitext(args[1])[0])
    config['arch'] = args[2]
    config['sandboxes'] = []
    config['overlaps'] = []
    config['new-overlaps'] = []

    warnings.formatwarning = warning_handler
    # Suppress multiple instances of the same warning.
    warnings.simplefilter('once', append=True)

    # dump any applicable environment variables into a config file
    with open('./ybd.environment', 'w') as f:
        for key in os.environ:
            if key[:4] == "YBD_":
                f.write(key[4:] + ": " + os.environ.get(key) + '\n')

    # load config files in reverse order of precedence
    load_configs([
        os.path.join(os.getcwd(), 'ybd.environment'),
        os.path.join(os.getcwd(), 'ybd.conf'),
        os.path.join(os.path.dirname(__file__), '..', 'ybd.conf'),
        os.path.join(os.path.dirname(__file__), 'config', 'ybd.conf')])

    if config.get('kbas-url', 'http://foo.bar/') == 'http://foo.bar/':
        config.pop('kbas-url')
    if config.get('kbas-url'):
        if not config['kbas-url'].endswith('/'):
            config['kbas-url'] += '/'

    config['total'] = config['tasks'] = config['counter'] = 0
    config['reproduced'] = []
    config['keys'] = []
    config['pid'] = os.getpid()
    config['def-version'] = get_version('.')

    config['defdir'] = os.getcwd()
    config['extsdir'] = os.path.join(config['defdir'], 'extensions')
    base_dir = os.environ.get('XDG_CACHE_HOME') or os.path.expanduser('~')
    config.setdefault('base',
                      os.path.join(base_dir, config['directories']['base']))
    for directory, path in config.get('directories', {}).items():
        try:
            if config.get(directory) is None:
                if path is None:
                    path = os.path.join(config.get('base', '/'), directory)
                config[directory] = path
            os.makedirs(config[directory])
        except OSError:
            if not os.path.isdir(config[directory]):
                exit('SETUP', 'ERROR: Can not find or create',
                     config[directory])

        log('SETUP', '%s is directory for' % config[directory], directory)

    # git replace means we can't trust that just the sha1 of a branch
    # is enough to say what it contains, so we turn it off by setting
    # the right flag in an environment variable.
    os.environ['GIT_NO_REPLACE_OBJECTS'] = '1'

    if not config.get('max-jobs'):
        config['max-jobs'] = cpu_count() / config.get('instances', 1)

    config['pid'] = os.getpid()
    config['counter'] = Counter(config['pid'])
    log('SETUP', '%s version is' % config['program'], config['my-version'])
    log('SETUP', 'Max-jobs is set to', config['max-jobs'])
Example #6
0
def setup(args):
    if len(args) != 3:
        sys.stderr.write("Usage: %s DEFINITION_FILE ARCH\n\n" % sys.argv[0])
        sys.exit(1)

    log('SETUP', 'Running %s in' % args[0], os.getcwd())
    config['start-time'] = datetime.datetime.now()
    config['target'] = os.path.basename(os.path.splitext(args[1])[0])
    config['arch'] = args[2]
    config['sandboxes'] = []

    warnings.formatwarning = warning_handler
    # Suppress multiple instances of the same warning.
    warnings.simplefilter('once', append=True)

    # load config files in reverse order of precedence
    load_configs([
        os.path.join(os.getcwd(), 'ybd.conf'),
        os.path.join(os.path.dirname(__file__), '..', 'ybd.conf'),
        os.path.join(os.path.dirname(__file__), 'config', 'ybd.conf')
    ])

    if config.get('kbas-url', 'http://foo.bar/') == 'http://foo.bar/':
        config.pop('kbas-url')
    if config.get('kbas-url'):
        if not config['kbas-url'].endswith('/'):
            config['kbas-url'] += '/'

    config['total'] = config['tasks'] = config['counter'] = 0
    config['pid'] = os.getpid()
    config['program'] = os.path.basename(args[0])
    config['my-version'] = get_version(os.path.dirname(__file__))
    config['def-version'] = get_version('.')

    config['defdir'] = os.getcwd()
    config['extsdir'] = os.path.join(config['defdir'], 'extensions')
    base_dir = os.environ.get('XDG_CACHE_HOME') or os.path.expanduser('~')
    if config.get('base', None) is None:
        config['base'] = os.path.join(base_dir, config['directories']['base'])
    for directory, path in config.get('directories', {}).items():
        try:
            if config.get(directory) is None:
                if path is None:
                    path = os.path.join(config.get('base', '/'), directory)
                config[directory] = path
            os.makedirs(config[directory])
        except OSError:
            if not os.path.isdir(config[directory]):
                exit('SETUP', 'ERROR: Can not find or create',
                     config[directory])

        log('SETUP', '%s is directory for' % config[directory], directory)

    # git replace means we can't trust that just the sha1 of a branch
    # is enough to say what it contains, so we turn it off by setting
    # the right flag in an environment variable.
    os.environ['GIT_NO_REPLACE_OBJECTS'] = '1'

    if not config.get('max-jobs'):
        config['max-jobs'] = cpu_count() / config.get('instances', 1)

    config['pid'] = os.getpid()
    config['counter'] = Counter(config['pid'])
    log('SETUP', '%s version is' % config['program'], config['my-version'])
    log('SETUP', 'Max-jobs is set to', config['max-jobs'])
Example #7
0
File: app.py Project: nowster/ybd
def setup(args):
    if len(args) not in [2, 3]:
        sys.stderr.write("Usage: %s DEFINITION_FILE [ARCH]\n\n" % sys.argv[0])
        sys.exit(1)

    config['target'] = os.path.basename(os.path.splitext(args[1])[0])
    if len(args) == 3:
        arch = args[2]
    else:
        arch = platform.machine()
        if arch in ('mips', 'mips64'):
            if arch == 'mips':
                arch = 'mips32'
            if sys.byteorder == 'big':
                arch = arch + 'b'
            else:
                arch = arch + 'l'
    config['arch'] = arch

    warnings.formatwarning = warning_handler
    # Suppress multiple instances of the same warning.
    warnings.simplefilter('once', append=True)

    conf_file = './ybd.conf'
    if not os.path.exists(conf_file):
        conf_file = os.path.join(os.path.dirname(__file__), 'ybd.conf')
    with open(conf_file) as f:
        text = f.read()
    for key, value in yaml.safe_load(text).items():
        config[key] = value
    config['total'] = config['tasks'] = config['counter'] = 0
    config['pid'] = os.getpid()
    config['program'] = os.path.basename(args[0])
    config['my-version'] = get_version(os.path.dirname(__file__))
    config['defdir'] = os.getcwd()
    config['extsdir'] = os.path.join(config['defdir'], 'extensions')
    config['def-version'] = get_version('.')

    dirs = ['artifacts', 'ccache_dir', 'deployment', 'gits', 'tidy', 'tmp']
    config['base'] = os.path.join(xdg_cache_home, config['base'])
    for directory in dirs:
        try:
            config[directory] = os.path.join(config['base'], directory)
            os.makedirs(config[directory])
        except OSError:
            if not os.path.isdir(config[directory]):
                exit(config['target'], 'ERROR: Can not find or create',
                     config[directory])

    # git replace means we can't trust that just the sha1 of a branch
    # is enough to say what it contains, so we turn it off by setting
    # the right flag in an environment variable.
    os.environ['GIT_NO_REPLACE_OBJECTS'] = '1'

    cores = cpu_count() / config.get('instances', 1)
    if not config.get('max-jobs'):
        config['max-jobs'] = max(int(cores * 1.5 + 0.5), 1)

    log('SETUP', '%s version is' % config['program'], config['my-version'])
    log('SETUP', 'Default configuration is:\n\n%s' % text)
    log('SETUP', 'Max-jobs is set to', config['max-jobs'])
Example #8
0
def setup(program, target, arch, mode, original_cwd=""):
    os.environ['LANG'] = 'en_US.UTF-8'
    # Installation of git-lfs on a system can pollute /etc/gitconfig. Setting
    # GIT_CONFIG_NOSYSTEM so ybd will ignore the system config.
    os.environ['GIT_CONFIG_NOSYSTEM'] = "1"
    config['start-time'] = datetime.datetime.now()
    config['program'] = os.path.basename(program)
    config['my-version'] = get_version(os.path.dirname(__file__))
    log('SETUP', '%s version is' % config['program'], config['my-version'])

    log('SETUP', 'Running %s in' % program, os.getcwd())
    config['filename'] = os.path.basename(target)
    config['target'] = os.path.basename(os.path.splitext(target)[0])
    config['arch'] = arch
    config['sandboxes'] = []
    config['overlaps'] = []
    config['new-overlaps'] = []

    warnings.formatwarning = warning_handler
    # Suppress multiple instances of the same warning.
    warnings.simplefilter('once', append=True)

    # dump any applicable environment variables into a config file
    with open('./ybd.environment', 'w') as f:
        for key in os.environ:
            if key[:4] == "YBD_":
                f.write(key[4:] + ": " + os.environ.get(key) + '\n')

    # load config files in reverse order of precedence
    load_configs([
        os.path.join(os.getcwd(), 'ybd.environment'),
        os.path.join(os.getcwd(), 'ybd.conf'),
        os.path.join(original_cwd, 'ybd.conf'),
        os.path.join(os.path.dirname(__file__), '..', 'ybd.conf'),
        os.path.join(os.path.dirname(__file__), 'config', 'ybd.conf')])

    # After loading configuration, override the 'mode' configuration only
    # if it was specified on the command line
    if mode is not None:
        config['mode'] = mode

    if not os.geteuid() == 0 and config.get('mode') == 'normal':
        log('SETUP', '%s needs root permissions' % program, exit=True)

    if config.get('kbas-url', 'http://foo.bar/') == 'http://foo.bar/':
        config.pop('kbas-url')
    if config.get('kbas-url'):
        if not config['kbas-url'].endswith('/'):
            config['kbas-url'] += '/'

    config['total'] = config['tasks'] = config['counter'] = 0
    config['systems'] = config['strata'] = config['chunks'] = 0
    config['reproduced'] = []
    config['keys'] = []
    config['pid'] = os.getpid()
    config['def-version'] = get_version('.')

    config['defdir'] = os.getcwd()
    config['extsdir'] = os.path.join(config['defdir'], 'extensions')
    if config.get('manifest') is True:
        config['manifest'] = os.path.join(config['defdir'],
                                          os.path.basename(config['target']) +
                                          '.manifest')
        try:
            os.remove(config['manifest'])
        except OSError:
            pass

    base_dir = os.environ.get('XDG_CACHE_HOME') or os.path.expanduser('~')
    config.setdefault('base',
                      os.path.join(base_dir, config['directories']['base']))
    for directory, path in config.get('directories', {}).items():
        try:
            if config.get(directory) is None:
                if path is None:
                    path = os.path.join(config.get('base', '/'), directory)
                config[directory] = path
            os.makedirs(config[directory])
        except OSError:
            if not os.path.isdir(config[directory]):
                log('SETUP', 'Cannot find or create', config[directory],
                    exit=True)

        log('SETUP', '%s is directory for' % config[directory], directory)

    # git replace means we can't trust that just the sha1 of a branch
    # is enough to say what it contains, so we turn it off by setting
    # the right flag in an environment variable.
    os.environ['GIT_NO_REPLACE_OBJECTS'] = '1'

    if 'max-jobs' not in config:
        config['max-jobs'] = cpu_count()

    if 'instances' not in config:
        # based on some testing (mainly on AWS), maximum effective
        # max-jobs value seems to be around 8-10 if we have enough cores
        # users should set values based on workload and build infrastructure
        # FIXME: more testing :)
        if cpu_count() >= 10:
            config['instances'] = 1 + (cpu_count() / 10)
            config['max-jobs'] = cpu_count() / config['instances']

    config['pid'] = os.getpid()
    config['counter'] = Counter()
    log('SETUP', 'Max-jobs is set to', config['max-jobs'])