Пример #1
0
def example_tasks(conf, app):
    include_dir = os.path.join(conf.paths.projectroot, conf.paths.includes)
    fn_prefix = os.path.join(include_dir, 'example')

    example_sources = [ fn for fn in
                        expand_tree(include_dir, 'yaml')
                        if fn.startswith(fn_prefix) ]

    d = ExampleDataCache(example_sources, conf)

    if not os.path.isdir(fn_prefix):
        os.makedirs(fn_prefix)

    for fn in d.cache.keys():
        exmpf = d.cache[fn]
        basename = fn[len(fn_prefix)+1:-5]

        out_fn = os.path.join(conf.paths.projectroot,
                              conf.paths.branch_source,
                              'includes', 'examples', basename) + '.rst'

        t = app.add('task')
        t.target = out_fn
        t.dependency = fn
        t.job = write_full_example
        t.args = (exmpf.collection, exmpf.examples, out_fn)
        t.description = 'generate an example for ' + basename

    logger.debug('added all tasks for example generation')
Пример #2
0
def manpage_url_tasks(builder, conf, app):
    project_source = os.path.join(conf.paths.projectroot,
                                  conf.paths.source)

    top_level_items = set()
    for fs_obj in os.listdir(project_source):
        if fs_obj.startswith('.static') or fs_obj == 'index.txt':
            continue
        if os.path.isdir(os.path.join(project_source, fs_obj)):
            top_level_items.add(fs_obj)
        if fs_obj.endswith('.txt'):
            top_level_items.add(fs_obj[:-4])

    top_level_items = '/'+ r'[^\s]*|/'.join(top_level_items) + r'[^\s]*'

    re_string = r'(\\fB({0})\\fP)'.format(top_level_items).replace(r'-', r'\-')
    subst = conf.project.url + '/' + conf.project.tag + r'\2'

    regex_obj = (re.compile(re_string), subst)

    for manpage in expand_tree(os.path.join(conf.paths.projectroot,
                                            conf.paths.output,
                                            conf.git.branches.current,
                                            builder), ['1', '5']):
        task = app.add('task')
        task.target = manpage
        task.job = manpage_url
        task.args = [regex_obj, manpage]
        task.description = 'processing urls in manpage file: {0}'.format(manpage)
Пример #3
0
def dump_file_hashes(conf):
    output = conf.system.dependency_cache

    o = { 'conf': conf.dict(),
          'time': datetime.datetime.utcnow().strftime("%s"),
          'files': { }
        }

    files = expand_tree(os.path.join(conf.paths.projectroot, conf.paths.source), None)

    fmap = o['files']

    for fn in files:
        if os.path.exists(fn):
            fmap[fn] = md5_file(fn)

    output_dir = os.path.dirname(output)

    if not os.path.exists(output_dir):
        os.makedirs(output_dir)

    with open(output, 'w') as f:
        json.dump(o, f)

    logger.info('wrote dependency cache to: {0}'.format(output))
Пример #4
0
def toc_tasks(conf, app):
    paths = conf.paths

    for fn in expand_tree(paths.includes, 'yaml'):
        if not (fn.startswith(os.path.join(paths.includes, 'toc')) or
                fn.startswith(os.path.join(paths.includes, 'ref-toc')) or
                fn.startswith(os.path.join(paths.includes, 'ref-spec'))):
            continue
        elif len(fn) >= 24:
            task = app.add('task')
            base_name = _get_toc_base_name(fn)
            target = []

            fmt = fn[20:24]
            if fmt != 'spec':
                fmt = fn[16:19]

            task.dependency = os.path.join(paths.projectroot, fn)
            task.job = _generate_toc_tree
            task.args = [fn, fmt, base_name, paths, conf]
            task.description = 'generating {0} from {1}'.format(fmt, fn)

            if fmt != 'spec':
                target.append(_get_toc_output_name(base_name, 'toc', paths))

            is_ref_spec = fn.startswith(os.path.join(os.path.dirname(fn), 'ref-spec'))

            if not is_ref_spec and (fmt == 'toc' or fmt == 'spec'):
                target.append(_get_toc_output_name(base_name, 'dfn-list', paths))
            elif fmt == 'ref' or is_ref_spec:
                target.append(_get_toc_output_name(base_name, 'table', paths))

            task.target = target

            logger.debug('added task for generating toc from {0}'.format(fn))
Пример #5
0
def finalize_single_html_jobs(builder, conf):
    pjoin = os.path.join

    single_html_dir = get_single_html_dir(conf)

    if not os.path.exists(single_html_dir):
        os.makedirs(single_html_dir)

    try:
        manual_single_html(input_file=pjoin(conf.paths.branch_output,
                                                    builder, 'contents.html'),
                                   output_file=pjoin(single_html_dir, 'index.html'))
    except (IOError, OSError):
        manual_single_html(input_file=pjoin(conf.paths.branch_output,
                                                    builder, 'index.html'),
                                   output_file=pjoin(single_html_dir, 'index.html'))
    copy_if_needed(source_file=pjoin(conf.paths.branch_output,
                                     builder, 'objects.inv'),
                   target_file=pjoin(single_html_dir, 'objects.inv'))

    single_path = pjoin(single_html_dir, '_static')

    for fn in expand_tree(pjoin(conf.paths.branch_output,
                                builder, '_static'), None):

        yield {
            'job': copy_if_needed,
            'args': [fn, pjoin(single_path, os.path.basename(fn))],
            'target': None,
            'dependency': None
        }
Пример #6
0
def include_files_unused(conf, inc_files=None):
    inc_files = [ fn[6:] for fn in expand_tree(os.path.join(conf.paths.includes), None) ]
    mapping = include_files(conf=conf)

    results = []
    for fn in inc_files:
        if fn.endswith('yaml') or fn.endswith('~'):
            continue
        if fn not in mapping.keys():
            results.append(fn)

    return results
Пример #7
0
def steps_tasks(conf, app):
    paths = conf.paths

    for fn in expand_tree(os.path.join(paths.projectroot, paths.includes), 'yaml'):
        if fn.startswith(os.path.join(paths.projectroot, paths.includes, 'step')):
            out_fn = _get_steps_output_fn(fn, paths)

            task = app.add('task')
            task.dependency = fn
            task.target = out_fn
            task.job = render_step_file
            task.args = [fn, out_fn, conf]
            logger.debug('added task to generate step: {0}'.format(out_fn))
Пример #8
0
def json_output_tasks(conf, app):
    regexes = [
        (re.compile(r'<a class=\"headerlink\"'), '<a'),
        (re.compile(r'<[^>]*>'), ''),
        (re.compile(r'&#8220;'), '"'),
        (re.compile(r'&#8221;'), '"'),
        (re.compile(r'&#8216;'), "'"),
        (re.compile(r'&#8217;'), "'"),
        (re.compile(r'&#\d{4};'), ''),
        (re.compile(r'&nbsp;'), ''),
        (re.compile(r'&gt;'), '>'),
        (re.compile(r'&lt;'), '<')
    ]

    outputs = []
    for fn in expand_tree('source', 'txt'):
        # path = build/<branch>/json/<filename>

        path = os.path.join(conf.paths.branch_output,
                            'json', os.path.splitext(fn.split(os.path.sep, 1)[1])[0])
        fjson = dot_concat(path, 'fjson')
        json = dot_concat(path, 'json')

        if conf.project.name == 'mms':
            if not os.path.exists(fjson):
                continue

        task = app.add('task')
        task.target = json
        task.dependency = fjson
        task.job = process_json_file
        task.description = "processing json file".format(json)
        task.args = [fjson, json, regexes, conf]

        outputs.append(json)

    list_file = os.path.join(conf.paths.branch_output, 'json-file-list')

    list_task = app.add('task')
    list_task.target = list_file
    list_task.job = generate_list_file
    list_task.args = [outputs, list_file, conf]

    output = app.add('app')
    out_task = output.add('task')
    out_task.job = json_output
    out_task.args = [conf]
    out_task.description = 'transfer json output to public directory'
Пример #9
0
def gettext_tasks(conf, app):
    locale_dirs = os.path.join(conf.paths.projectroot,
                               conf.paths.locale, 'pot')

    builder_name = resolve_builder_path('gettext', conf.project.edition, None, conf)

    branch_output = os.path.join(conf.paths.projectroot,
                                 conf.paths.branch_output,
                                 builder_name)

    path_offset = len(branch_output) + 1

    for fn in expand_tree(branch_output, None):
        task = app.add('task')
        task.target = fn
        task.job = copy_if_needed
        task.args = [ fn, os.path.join(locale_dirs, fn[path_offset:]), None]
        task.description = "migrating po file {0} if needed".format(fn)
Пример #10
0
def get_file_list(path, input_extension):
    ''' This function wraps around expand tree to return a list of only 1 file
    if the user gives a path to a file and not a directory. Otherwise it has
    the same functionality
    :param string path: path to the file
    :param list input_extension: a list (or a single) of extensions that is acceptable
    '''
    if os.path.isfile(path):
        if input_extension is not None:
            if isinstance(input_extension, list):
                if os.path.splitext(path)[1][1:] not in input_extension:
                    return []
            else:
                if not path.endswith(input_extension):
                    return []
        return [path]
    else:
        return expand_tree(path, input_extension)
Пример #11
0
def generated_includes(conf):
    toc_spec_files = []
    step_files = []
    for fn in expand_tree(os.path.join(conf.paths.includes), input_extension='yaml'):
        base = os.path.basename(fn)

        if base.startswith('toc-spec'):
            toc_spec_files.append(fn)
        elif base.startswith('ref-spec'):
            toc_spec_files.append(fn)
        elif base.startswith('steps'):
            step_files.append(fn)

    maskl = len(conf.paths.source)
    path_prefix = conf.paths.includes[len(conf.paths.source):]
    mapping = {}
    for spec_file in toc_spec_files:
        if os.path.exists(spec_file):
            data = ingest_yaml_doc(spec_file)
        else:
            continue

        deps = [ os.path.join(path_prefix, i ) for i in data['sources']]

        mapping[spec_file[maskl:]] = deps

    for step_def in step_files:
        data = ingest_yaml_list(step_def)

        deps = []
        for step in data:
            if 'source' in step:
                deps.append(step['source']['file'])

        if len(deps) != 0:
            deps = [ os.path.join(path_prefix, i ) for i in deps ]

            mapping[step_def[maskl:]] = deps

    return mapping
Пример #12
0
def get_migration_specifications(conf):
    return [ fn for fn in expand_tree(os.path.join(conf.paths.projectroot,
                                                   conf.paths.builddata))
             if 'primer' in fn and 'migrations' in fn ]
Пример #13
0
def api_sources(conf):
    return expand_tree(os.path.join(conf.paths.projectroot, conf.paths.source, 'reference'), 'yaml')
Пример #14
0
def option_sources(conf):
    output_path = get_option_path(conf)

    for fn in expand_tree(get_base_path(conf), 'yaml'):
        if fn.startswith(output_path):
            yield fn
Пример #15
0
def step_sources(paths):
    for fn in expand_tree(os.path.join(paths.projectroot, paths.includes), ' yaml'):
        if fn.startswith(os.path.join(paths.projectroot, paths.includes, 'step')):
            yield fn
Пример #16
0
def get_migration_specifications(conf):
    return [ fn for fn in expand_tree(os.path.join(conf.paths.projectroot,
                                                   conf.paths.builddata))
             if  conf.project.name in os.path.basename(fn) and 'migrations' in fn ]