Exemplo n.º 1
0
def load_schemas_from_git(ref):
    tree = ecs_helpers.get_tree_by_ref(ref)
    fields_nested = {}
    for blob in tree['schemas'].blobs:
        if blob.name.endswith('.yml'):
            new_fields = read_schema_blob(blob, ref)
            fields_nested = ecs_helpers.safe_merge_dicts(
                fields_nested, new_fields)
    return fields_nested
Exemplo n.º 2
0
def read_version(ref=None):
    if ref:
        print('Loading schemas from git ref ' + ref)
        tree = ecs_helpers.get_tree_by_ref(ref)
        return tree['version'].data_stream.read().decode('utf-8').rstrip()
    else:
        print('Loading schemas from local files')
        with open('version', 'r') as infile:
            return infile.read().rstrip()
Exemplo n.º 3
0
Arquivo: loader.py Projeto: djptek/ecs
def load_schemas_from_git(ref, target_dir='schemas'):
    tree = ecs_helpers.get_tree_by_ref(ref)
    fields_nested = {}

    # Handles case if target dir doesn't exists in git ref
    if ecs_helpers.path_exists_in_git_tree(tree, target_dir):
        for blob in tree[target_dir].blobs:
            if blob.name.endswith('.yml'):
                new_fields = read_schema_blob(blob, ref)
                fields_nested = ecs_helpers.safe_merge_dicts(
                    fields_nested, new_fields)
    else:
        raise KeyError(
            f"Target directory './{target_dir}' not present in git ref '{ref}'!"
        )
    return fields_nested
Exemplo n.º 4
0
def main():
    args = argument_parser()
    # Get rid of empty include
    if args.include and [''] == args.include:
        args.include.clear()

    if args.ref:
        # Load ECS schemas from a specific git ref
        print('Loading schemas from git ref ' + args.ref)
        tree = ecs_helpers.get_tree_by_ref(args.ref)
        ecs_version = read_version_from_tree(tree)
        ecs_schemas = schema_reader.load_schemas_from_git(tree)
    else:
        # Load the default schemas
        print('Loading default schemas')
        ecs_version = read_version()
        ecs_schemas = schema_reader.load_schemas_from_files()

    print('Running generator. ECS version ' + ecs_version)
    intermediate_fields = schema_reader.create_schema_dicts(ecs_schemas)

    # Maybe load user specified directory of schemas
    if args.include:
        include_glob = ecs_helpers.get_glob_files(args.include,
                                                  ecs_helpers.YAML_EXT)

        print('Loading user defined schemas: {0}'.format(include_glob))

        custom_schemas = schema_reader.load_schemas_from_files(include_glob)
        intermediate_custom = schema_reader.create_schema_dicts(custom_schemas)
        schema_reader.merge_schema_fields(intermediate_fields,
                                          intermediate_custom)

    schema_reader.assemble_reusables(intermediate_fields)

    if args.subset:
        subset = {}
        for arg in args.subset:
            for file in glob.glob(arg):
                with open(file) as f:
                    raw = yaml.safe_load(f.read())
                    ecs_helpers.recursive_merge_subset_dicts(subset, raw)
        if not subset:
            raise ValueError('Subset option specified but no subsets found')
        intermediate_fields = ecs_helpers.fields_subset(
            subset, intermediate_fields)

    (nested, flat) = schema_reader.generate_nested_flat(intermediate_fields)

    # default location to save files
    out_dir = 'generated'
    docs_dir = 'docs'
    if args.out:
        out_dir = os.path.join(args.out, out_dir)
        docs_dir = os.path.join(args.out, docs_dir)

    ecs_helpers.make_dirs(out_dir)
    ecs_helpers.make_dirs(docs_dir)

    intermediate_files.generate(nested, flat, out_dir)
    if args.intermediate_only:
        exit()

    csv_generator.generate(flat, ecs_version, out_dir)
    es_template.generate(flat, ecs_version, out_dir, args.template_settings,
                         args.mapping_settings)
    beats.generate(nested, ecs_version, out_dir)
    if args.include or args.subset:
        exit()
    asciidoc_fields.generate(intermediate_fields, ecs_version, docs_dir)