示例#1
0
文件: assembly.py 项目: rdale/ybd
def compose(defs, target):
    '''Work through defs tree, building and assembling until target exists'''

    component = defs.get(target)
    if app.config.get('log-verbose'):
        app.log(target, "Composing", component['name'])

    # if we can't calculate cache key, we can't create this component
    if cache_key(defs, component) is False:
        return False

    # if this component is already cached, we're done
    if get_cache(defs, component):
        return cache_key(defs, component)

    # if we have a kbas, look there to see if this component exists
    if app.config.get('kbas-url'):
        with claim(defs, component):
            if get_remote(defs, component):
                app.config['counter'].increment()
                return cache_key(defs, component)

    if component.get('arch') and component['arch'] != app.config['arch']:
        return None

    with sandbox.setup(component):
        assemble(defs, component)
        if 'systems' not in component and not get_cache(defs, component):
            install_dependencies(defs, component)
            build(defs, component)

    return cache_key(defs, component)
示例#2
0
文件: assembly.py 项目: nowster/ybd
def assemble(defs, target):
    '''Assemble dependencies and contents recursively until target exists.'''

    if cache.get_cache(defs, target):
        # needed for artifact splitting
        load_manifest(defs, target)
        return cache.cache_key(defs, target)

    random.seed(datetime.datetime.now())
    component = defs.get(target)

    if component.get('arch') and component['arch'] != app.config['arch']:
        app.log(target, 'Skipping assembly for', component.get('arch'))
        return None

    def assemble_system_recursively(system):
        assemble(defs, system['path'])

        for subsystem in system.get('subsystems', []):
            assemble_system_recursively(subsystem)

    with app.timer(component, 'assembly'):
        sandbox.setup(component)

        systems = component.get('systems', [])
        random.shuffle(systems)
        for system in systems:
            assemble_system_recursively(system)

        dependencies = component.get('build-depends', [])
        random.shuffle(dependencies)
        for it in dependencies:
            dependency = defs.get(it)
            assemble(defs, dependency)
            sandbox.install(defs, component, dependency)

        contents = component.get('contents', [])
        random.shuffle(contents)
        for it in contents:
            subcomponent = defs.get(it)
            if subcomponent.get('build-mode') != 'bootstrap':
                assemble(defs, subcomponent)
                splits = None
                if component.get('kind') == 'system':
                    splits = subcomponent.get('artifacts')
                sandbox.install(defs, component, subcomponent, splits)

        app.config['counter'] += 1
        if 'systems' not in component:
            with app.timer(component, 'build'):
                build(defs, component)
        with app.timer(component, 'artifact creation'):
            do_manifest(defs, component)
            cache.cache(defs, component,
                        full_root=component.get('kind') == "system")
        sandbox.remove(component)

    return cache.cache_key(defs, component)
示例#3
0
def assemble(defs, target):
    '''Assemble dependencies and contents recursively until target exists.'''

    component = defs.get(target)

    if cache_key(defs, component) is False:
        return False

    if get_cache(defs, component):
        return cache_key(defs, component)

    if app.config.get('kbas-url'):
        with claim(defs, component):
            if get_remote(defs, component):
                app.config['counter'].increment()
                return cache_key(defs, component)

    random.seed(datetime.datetime.now())

    if component.get('arch') and component['arch'] != app.config['arch']:
        return None

    sandbox.setup(component)

    systems = component.get('systems', [])
    random.shuffle(systems)
    for system in systems:
        assemble(defs, system['path'])
        for subsystem in system.get('subsystems', []):
            assemble(defs, subsystem)

    dependencies = component.get('build-depends', [])
    for it in dependencies:
        preinstall(defs, component, it)

    contents = component.get('contents', [])
    random.shuffle(contents)
    for it in contents:
        subcomponent = defs.get(it)
        if subcomponent.get('build-mode', 'staging') != 'bootstrap':
            preinstall(defs, component, subcomponent)

    if 'systems' not in component and not get_cache(defs, component):
        if app.config.get('instances', 1) > 1:
            with claim(defs, component):
                # in here, exceptions get eaten
                do_build(defs, component)
        else:
            # in here, exceptions do not get eaten
            do_build(defs, component)

    app.remove_dir(component['sandbox'])

    return cache_key(defs, component)
示例#4
0
def assemble(defs, target):
    '''Assemble dependencies and contents recursively until target exists.'''

    component = defs.get(target)

    if cache_key(defs, component) is False:
        return False

    if get_cache(defs, component):
        return cache_key(defs, component)

    if app.config.get('kbas-url'):
        with claim(defs, component):
            if get_remote(defs, component):
                app.config['counter'].increment()
                return cache_key(defs, component)

    random.seed(datetime.datetime.now())

    if component.get('arch') and component['arch'] != app.config['arch']:
        return None

    sandbox.setup(component)

    systems = component.get('systems', [])
    random.shuffle(systems)
    for system in systems:
        assemble(defs, system['path'])
        for subsystem in system.get('subsystems', []):
            assemble(defs, subsystem)

    dependencies = component.get('build-depends', [])
    for it in dependencies:
        preinstall(defs, component, it)

    contents = component.get('contents', [])
    random.shuffle(contents)
    for it in contents:
        subcomponent = defs.get(it)
        if subcomponent.get('build-mode', 'staging') != 'bootstrap':
            preinstall(defs, component, subcomponent)

    if 'systems' not in component and not get_cache(defs, component):
        if app.config.get('instances', 1) > 1:
            with claim(defs, component):
                # in here, exceptions get eaten
                do_build(defs, component)
        else:
            # in here, exceptions do not get eaten
            do_build(defs, component)

    app.remove_dir(component['sandbox'])

    return cache_key(defs, component)
示例#5
0
文件: assembly.py 项目: padrigali/ybd
def assemble(defs, target):
    '''Assemble dependencies and contents recursively until target exists.'''

    component = defs.get(target)
    if get_cache(defs, component) or get_remote(defs, component):
        return cache_key(defs, component)

    random.seed(datetime.datetime.now())

    if component.get('arch') and component['arch'] != app.config['arch']:
        app.log(target, 'Skipping assembly for', component.get('arch'))
        return None

    sandbox.setup(component)

    systems = component.get('systems', [])
    random.shuffle(systems)
    for system in systems:
        assemble(defs, system['path'])
        for subsystem in system.get('subsystems', []):
            assemble(defs, subsystem)

    dependencies = component.get('build-depends', [])
    for it in dependencies:
        preinstall(defs, component, it)

    contents = component.get('contents', [])
    random.shuffle(contents)
    for it in contents:
        subcomponent = defs.get(it)
        if subcomponent.get('build-mode', 'staging') != 'bootstrap':
            preinstall(defs, component, subcomponent)

    if 'systems' not in component:
        if is_building(defs, component):
            import time
            time.sleep(10)
            raise Exception

        app.config['counter'] += 1
        if not get_cache(defs, component):
            with app.timer(component, 'build of %s' % component['cache']):
                with claim(defs, component):
                    build(defs, component)

    with app.timer(component, 'artifact creation'):
        do_manifest(component)
        cache(defs, component)
    sandbox.remove(component)

    return cache_key(defs, component)
示例#6
0
def assemble(defs, target):
    '''Assemble dependencies and contents recursively until target exists.'''

    if cache.get_cache(defs, target):
        return cache.cache_key(defs, target)

    component = defs.get(target)

    if component.get('arch') and component['arch'] != app.settings['arch']:
        app.log(target, 'Skipping assembly for', component.get('arch'))
        return None

    def assemble_system_recursively(system):
        assemble(defs, system['path'])
        for subsystem in system.get('subsystems', []):
            assemble_system_recursively(subsystem)

    with app.timer(component, 'Starting assembly'):
        sandbox.setup(component)
        for system_spec in component.get('systems', []):
            assemble_system_recursively(system_spec)

        dependencies = component.get('build-depends', [])
        random.shuffle(dependencies)
        for it in dependencies:
            dependency = defs.get(it)
            assemble(defs, dependency)
            sandbox.install(defs, component, dependency)

        contents = component.get('contents', [])
        random.shuffle(contents)
        for it in contents:
            subcomponent = defs.get(it)
            if subcomponent.get('build-mode') != 'bootstrap':
                assemble(defs, subcomponent)
                sandbox.install(defs, component, subcomponent)

        app.settings['counter'] += 1
        if 'systems' not in component:
            build(defs, component)
        do_manifest(component)
        cache.cache(defs, component,
                    full_root=component.get('kind') == "system")
        sandbox.remove(component)

    return cache.cache_key(defs, component)
示例#7
0
文件: assembly.py 项目: JanderJLR/ybd
def compose(dn):
    '''Work through defs tree, building and assembling until target exists'''

    if type(dn) is not dict:
        dn = app.defs.get(dn)

    # if we can't calculate cache key, we can't create this component
    if cache_key(dn) is False:
        if 'tried' not in dn:
            log(dn, 'No cache_key, so skipping compose')
            dn['tried'] = True
        return False

    # if dn is already cached, we're done
    if get_cache(dn):
        return cache_key(dn)

    log(dn, "Composing", dn['name'], verbose=True)

    # if we have a kbas, look there to see if this component exists
    if config.get('kbas-url') and not config.get('reproduce'):
        with claim(dn):
            if get_remote(dn):
                config['counter'].increment()
                return cache_key(dn)

    # we only work with user-specified arch
    if 'arch' in dn and dn['arch'] != config['arch']:
        return None

    # Create composite components (strata, systems, clusters)
    systems = dn.get('systems', [])
    shuffle(systems)
    for system in systems:
        for s in system.get('subsystems', []):
            subsystem = app.defs.get(s['path'])
            compose(subsystem)
        compose(system['path'])

    with sandbox.setup(dn):
        install_contents(dn)
        build(dn)  # bring in 'build-depends', and run make

    return cache_key(dn)
示例#8
0
文件: assembly.py 项目: jjardon/ybd
def compose(dn):
    '''Work through defs tree, building and assembling until target exists'''

    if type(dn) is not dict:
        dn = app.defs.get(dn)

    # if we can't calculate cache key, we can't create this component
    if cache_key(dn) is False:
        if 'tried' not in dn:
            log(dn, 'No cache_key, so skipping compose')
            dn['tried'] = True
        return False

    # if dn is already cached, we're done
    if get_cache(dn):
        return cache_key(dn)

    log(dn, "Composing", dn['name'], verbose=True)

    # if we have a kbas, look there to see if this component exists
    if config.get('kbas-url') and not config.get('reproduce'):
        with claim(dn):
            if get_remote(dn):
                config['counter'].increment()
                return cache_key(dn)

    # we only work with user-specified arch
    if 'arch' in dn and dn['arch'] != config['arch']:
        return None

    # Create composite components (strata, systems, clusters)
    systems = dn.get('systems', [])
    shuffle(systems)
    for system in systems:
        for s in system.get('subsystems', []):
            subsystem = app.defs.get(s['path'])
            compose(subsystem)
        compose(system['path'])

    with sandbox.setup(dn):
        install_contents(dn)
        build(dn)     # bring in 'build-depends', and run make

    return cache_key(dn)
示例#9
0
def assemble(target):
    '''Assemble dependencies and contents recursively until target exists.'''

    if cache.get_cache(target):
        return cache.cache_key(target)

    defs = Definitions()
    this = defs.get(target)

    if this.get('arch') and this['arch'] != app.settings['arch']:
        app.log(target, 'Skipping assembly for', this['arch'])
        return None

    with app.timer(this, 'Starting assembly'):
        sandbox.setup(this)
        for it in this.get('systems', []):
            system = defs.get(it)
            assemble(system)
            for subsystem in this.get('subsystems', []):
                assemble(subsystem)

        dependencies = this.get('build-depends', [])
        random.shuffle(dependencies)
        for it in dependencies:
            dependency = defs.get(it)
            assemble(dependency)
            sandbox.install(this, dependency)

        contents = this.get('contents', [])
        random.shuffle(contents)
        for it in contents:
            component = defs.get(it)
            if component.get('build-mode') != 'bootstrap':
                assemble(component)
                sandbox.install(this, component)

        build(this)
        do_manifest(this)
        cache.cache(this, full_root=this.get('kind', None) == "system")
        sandbox.remove(this)

    return cache.cache_key(this)
示例#10
0
文件: __main__.py 项目: leeming/ybd
            if os.path.isdir(os.path.join(os.getcwd(), '..', 'definitions')):
                os.chdir(os.path.join(os.getcwd(), '..', 'definitions'))

app.setup(sys.argv)
app.cleanup(app.config['tmp'])

with app.timer('TOTAL'):
    tmp_lock = open(os.path.join(app.config['tmp'], 'lock'), 'r')
    fcntl.flock(tmp_lock, fcntl.LOCK_SH | fcntl.LOCK_NB)

    target = os.path.join(app.config['defdir'], app.config['target'])
    app.log('TARGET', 'Target is %s' % target, app.config['arch'])
    with app.timer('DEFINITIONS', 'parsing %s' % app.config['def-version']):
        defs = Definitions()
    with app.timer('CACHE-KEYS', 'cache-key calculations'):
        cache.cache_key(defs, app.config['target'])

    cache.cull(app.config['artifacts'])
    target = defs.get(app.config['target'])
    if app.config['total'] == 0 or (app.config['total'] == 1
                                    and target.get('kind') == 'cluster'):
        app.exit('ARCH', 'ERROR: no definitions found for', app.config['arch'])

    defs.save_trees()
    if app.config.get('mode', 'normal') == 'keys-only':
        with open(app.config['result-file'], 'w') as f:
            f.write(target['cache'] + '\n')
        app.log('RESULT', 'Cache-key for target is at',
                app.config['result-file'])
        os._exit(0)
示例#11
0
文件: __main__.py 项目: nuxeh/ybd
            if os.path.isdir(os.path.join(os.getcwd(), '..', 'definitions')):
                os.chdir(os.path.join(os.getcwd(), '..', 'definitions'))

app.setup(sys.argv)
app.cleanup(app.config['tmp'])

with app.timer('TOTAL'):
    tmp_lock = open(os.path.join(app.config['tmp'], 'lock'), 'r')
    fcntl.flock(tmp_lock, fcntl.LOCK_SH | fcntl.LOCK_NB)

    target = os.path.join(app.config['defdir'], app.config['target'])
    app.log('TARGET', 'Target is %s' % target, app.config['arch'])
    with app.timer('DEFINITIONS', 'parsing %s' % app.config['def-version']):
        defs = Definitions()
    with app.timer('CACHE-KEYS', 'cache-key calculations'):
        cache.cache_key(defs, app.config['target'])

    cache.cull(app.config['artifacts'])
    target = defs.get(app.config['target'])
    if app.config['total'] == 0 or (app.config['total'] == 1 and
                                    target.get('kind') == 'cluster'):
        app.exit('ARCH', 'ERROR: no definitions found for', app.config['arch'])

    defs.save_trees()
    if app.config.get('mode', 'normal') == 'keys-only':
        with open('./ybd.result', 'w') as f:
            f.write(target['cache'] + '\n')
        os._exit(0)

    sandbox.executor = sandboxlib.executor_for_platform()
    app.log(app.config['target'], 'Sandbox using %s' % sandbox.executor)
示例#12
0
def lockfile(defs, this):
    return os.path.join(app.config['tmp'], cache_key(defs, this) + '.lock')
    def do_search_in_folder(self, folder):
        """List/search contents of a specific Smart Folder.

        Sends results to Alfred.

        :param folder: name or path of Smart Folder
        :type folder: ``unicode``

        """
        log.info(u'searching folder "%s" for "%s" ...', folder, self.query)
        files = []
        folder_path = None
        for name, path in self.folders:
            if path == folder:
                folder_path = path
                break
            elif name == folder:
                folder_path = path
                break

        else:
            return self._terminate_with_error(
                u"Unknown folder '{}'".format(folder),
                'Check your configuration with `smartfolders`')

        # Get contents of folder; update if necessary
        key = cache_key(folder_path)
        files = self.wf.cached_data(key, max_age=0)
        if files is None:
            files = []

        if not self.wf.cached_data_fresh(key, CACHE_AGE_CONTENTS):
            run_in_background(key,
                              ['/usr/bin/python',
                               self.wf.workflowfile('cache.py'),
                               '--folder', folder_path])
        if is_running(key):
            self.wf.rerun = 0.5

        if self.query:
            files = self.wf.filter(self.query, files,
                                   key=os.path.basename,
                                   min_score=10)

        if not files:
            if not self.query:
                self._add_message('Empty Smart Folder', icon=ICON_WARNING)
            else:
                self._add_message('No matching results',
                                  'Try a different query',
                                  icon=ICON_WARNING)
        else:
            for i, path in enumerate(files):
                title = os.path.basename(path)
                subtitle = path.replace(os.getenv('HOME'), '~')
                self.wf.add_item(title, subtitle,
                                 uid=path,
                                 arg=path,
                                 valid=True,
                                 icon=path,
                                 icontype='fileicon',
                                 type='file')

                if (i + 1) == MAX_RESULTS:
                    break

        self.wf.send_feedback()
示例#14
0
with timer('TOTAL'):
    tmp_lock = open(os.path.join(config['tmp'], 'lock'), 'r')
    fcntl.flock(tmp_lock, fcntl.LOCK_SH | fcntl.LOCK_NB)

    target = os.path.join(config['defdir'], config['target'])
    log('TARGET', 'Target is %s' % target, config['arch'])
    with timer('DEFINITIONS', 'parsing %s' % config['def-version']):
        app.defs = Pots()

    target = app.defs.get(config['target'])
    if config.get('mode', 'normal') == 'parse-only':
        Pipeline(target)
        os._exit(0)

    with timer('CACHE-KEYS', 'cache-key calculations'):
        cache.cache_key(target)

    if 'release-note' in config:
        do_release_note(config['release-note'])

    if config['total'] == 0 or (config['total'] == 1
                                and target.get('kind') == 'cluster'):
        log('ARCH', 'No definitions for', config['arch'], exit=True)

    app.defs.save_trees()
    if config.get('mode', 'normal') == 'keys-only':
        write_cache_key()
        os._exit(0)

    cache.cull(config['artifacts'])
示例#15
0
def lockfile(dn):
    return os.path.join(config['tmp'], cache_key(dn) + '.lock')
示例#16
0
文件: assembly.py 项目: rdale/ybd
def lockfile(defs, this):
    return os.path.join(app.config['tmp'], cache_key(defs, this) + '.lock')
    def do_search_in_folder(self, folder):
        """List/search contents of a specific Smart Folder.

        Sends results to Alfred.

        :param folder: name or path of Smart Folder
        :type folder: ``unicode``

        """
        log.info(u'searching folder "%s" for "%s" ...', folder, self.query)
        files = []
        folder_path = None
        for name, path in self.folders:
            if path == folder:
                folder_path = path
                break
            elif name == folder:
                folder_path = path
                break

        else:
            return self._terminate_with_error(
                u"Unknown folder '{}'".format(folder),
                'Check your configuration with `smartfolders`')

        # Get contents of folder; update if necessary
        key = cache_key(folder_path)
        files = self.wf.cached_data(key, max_age=0)
        if files is None:
            files = []

        if not self.wf.cached_data_fresh(key, CACHE_AGE_CONTENTS):
            run_in_background(key, [
                '/usr/bin/python',
                self.wf.workflowfile('cache.py'), '--folder', folder_path
            ])
        if is_running(key):
            self.wf.rerun = 0.5

        if self.query:
            files = self.wf.filter(self.query,
                                   files,
                                   key=os.path.basename,
                                   min_score=10)

        if not files:
            if not self.query:
                self._add_message('Empty Smart Folder', icon=ICON_WARNING)
            else:
                self._add_message('No matching results',
                                  'Try a different query',
                                  icon=ICON_WARNING)
        else:
            for i, path in enumerate(files):
                title = os.path.basename(path)
                subtitle = path.replace(os.getenv('HOME'), '~')
                self.wf.add_item(title,
                                 subtitle,
                                 uid=path,
                                 arg=path,
                                 valid=True,
                                 icon=path,
                                 icontype='fileicon',
                                 type='file')

                if (i + 1) == MAX_RESULTS:
                    break

        self.wf.send_feedback()
示例#18
0
文件: app.py 项目: jjardon/ybd
def lockfile(dn):
    return os.path.join(config['tmp'], cache_key(dn) + '.lock')
示例#19
0
文件: __main__.py 项目: jjardon/ybd
with timer('TOTAL'):
    tmp_lock = open(os.path.join(config['tmp'], 'lock'), 'r')
    fcntl.flock(tmp_lock, fcntl.LOCK_SH | fcntl.LOCK_NB)

    target = os.path.join(config['defdir'], config['target'])
    log('TARGET', 'Target is %s' % target, config['arch'])
    with timer('DEFINITIONS', 'parsing %s' % config['def-version']):
        app.defs = Pots()

    target = app.defs.get(config['target'])
    if config.get('mode', 'normal') == 'parse-only':
        Pipeline(target)
        os._exit(0)

    with timer('CACHE-KEYS', 'cache-key calculations'):
        cache.cache_key(target)

    if 'release-note' in config:
        do_release_note(config['release-note'])

    if config['total'] == 0 or (config['total'] == 1 and
                                target.get('kind') == 'cluster'):
        log('ARCH', 'No definitions for', config['arch'], exit=True)

    app.defs.save_trees()
    if config.get('mode', 'normal') == 'keys-only':
        write_cache_key()
        os._exit(0)

    cache.cull(config['artifacts'])