def generate_template_version(elasticsearch_version, mappings_section, out_dir, template_settings_file): ecs_helpers.make_dirs( join(out_dir, 'elasticsearch', str(elasticsearch_version))) if template_settings_file: with open(template_settings_file) as f: template = json.load(f) else: template = default_template_settings() if elasticsearch_version == 6: es6_mappings_section = copy.deepcopy(mappings_section) es6_type_fallback(es6_mappings_section['properties']) # error.stack_trace needs special handling to set # index: false and doc_values: false error_stack_trace_mappings = es6_mappings_section['properties'][ 'error']['properties']['stack_trace'] error_stack_trace_mappings.setdefault('index', False) error_stack_trace_mappings.setdefault('doc_values', False) template['mappings'] = {'_doc': es6_mappings_section} else: template['mappings'] = mappings_section filename = join( out_dir, "elasticsearch/{}/template.json".format(elasticsearch_version)) save_json(filename, template)
def main(): args = argument_parser() # Get rid of empty include if args.include and [''] == args.include: args.include.clear() ecs_version = read_version() print('Running generator. ECS version ' + ecs_version) # Load the default schemas print('Loading default schemas') intermediate_fields = schema_reader.load_schemas() # Maybe load user specified directory of schemas if args.include: include_glob = ecs_helpers.get_glob_files(args.include, ecs_helpers.YAML_EXT) print('Loading user defined schemas: {0}'.format(include_glob)) intermediate_custom = schema_reader.load_schemas(include_glob) schema_reader.merge_schema_fields(intermediate_fields, intermediate_custom) schema_reader.assemble_reusables(intermediate_fields) if args.subset: subset = {} for arg in args.subset: for file in glob.glob(arg): with open(file) as f: raw = yaml.safe_load(f.read()) ecs_helpers.recursive_merge_subset_dicts(subset, raw) if not subset: raise ValueError('Subset option specified but no subsets found') intermediate_fields = ecs_helpers.fields_subset( subset, intermediate_fields) (nested, flat) = schema_reader.generate_nested_flat(intermediate_fields) # default location to save files out_dir = 'generated' docs_dir = 'docs' if args.out: out_dir = os.path.join(args.out, out_dir) docs_dir = os.path.join(args.out, docs_dir) ecs_helpers.make_dirs(out_dir) ecs_helpers.make_dirs(docs_dir) intermediate_files.generate(nested, flat, out_dir) if args.intermediate_only: exit() csv_generator.generate(flat, ecs_version, out_dir) es_template.generate(flat, ecs_version, out_dir) beats.generate(nested, ecs_version, out_dir) if args.include or args.subset: exit() asciidoc_fields.generate(intermediate_fields, ecs_version, docs_dir)
def generate_legacy_template_version(es_version, ecs_version, mappings_section, out_dir, template_settings_file): ecs_helpers.make_dirs(join(out_dir, 'elasticsearch', str(es_version))) template = template_settings(es_version, ecs_version, mappings_section, template_settings_file) filename = join(out_dir, "elasticsearch/{}/template.json".format(es_version)) save_json(filename, template)
def main() -> None: args = argument_parser() ecs_generated_version: str = read_version(args.ref) print('Running generator. ECS version ' + ecs_generated_version) # default location to save files out_dir = 'generated' docs_dir = 'docs' if args.out: default_dirs = False out_dir = os.path.join(args.out, out_dir) docs_dir = os.path.join(args.out, docs_dir) else: default_dirs = True ecs_helpers.make_dirs(out_dir) # To debug issues in the gradual building up of the nested structure, insert # statements like this after any step of interest. # ecs_helpers.yaml_dump('ecs.yml', fields) # Detect usage of experimental changes to tweak artifact version label if args.include and loader.EXPERIMENTAL_SCHEMA_DIR in args.include: ecs_generated_version += "+exp" print('Experimental ECS version ' + ecs_generated_version) fields: dict[str, FieldEntry] = loader.load_schemas(ref=args.ref, included_files=args.include) cleaner.clean(fields, strict=args.strict) finalizer.finalize(fields) fields, docs_only_fields = subset_filter.filter(fields, args.subset, out_dir) fields = exclude_filter.exclude(fields, args.exclude) nested, flat = intermediate_files.generate(fields, os.path.join(out_dir, 'ecs'), default_dirs) if args.intermediate_only: exit() csv_generator.generate(flat, ecs_generated_version, out_dir) es_template.generate(nested, ecs_generated_version, out_dir, args.mapping_settings, args.template_settings) es_template.generate_legacy(flat, ecs_generated_version, out_dir, args.mapping_settings, args.template_settings_legacy) beats.generate(nested, ecs_generated_version, out_dir) if (args.include or args.subset or args.exclude) and not args.force_docs: exit() ecs_helpers.make_dirs(docs_dir) docs_only_nested = intermediate_files.generate_nested_fields( docs_only_fields) asciidoc_fields.generate(nested, docs_only_nested, ecs_generated_version, docs_dir)
def generate_legacy_template_version(ecs_version, mappings_section, out_dir, template_settings_file): ecs_helpers.make_dirs(join(out_dir, 'elasticsearch', "legacy")) template = template_settings(ecs_version, mappings_section, template_settings_file, isLegacy=True) filename = join(out_dir, "elasticsearch/legacy/template.json") save_json(filename, template)
def all_component_templates(ecs_nested, ecs_version, out_dir): """Generate one component template per field set""" component_dir = join(out_dir, 'elasticsearch/component') ecs_helpers.make_dirs(component_dir) for (fieldset_name, fieldset) in candidate_components(ecs_nested).items(): field_mappings = {} for (flat_name, field) in fieldset['fields'].items(): name_parts = flat_name.split('.') dict_add_nested(field_mappings, name_parts, entry_for(field)) save_component_template(fieldset_name, ecs_version, component_dir, field_mappings)
def generate(fields, out_dir, default_dirs): ecs_helpers.make_dirs(join(out_dir)) # Should only be used for debugging ECS development if default_dirs: ecs_helpers.yaml_dump(join(out_dir, 'ecs.yml'), fields) flat = generate_flat_fields(fields) nested = generate_nested_fields(fields) ecs_helpers.yaml_dump(join(out_dir, 'ecs_flat.yml'), flat) ecs_helpers.yaml_dump(join(out_dir, 'ecs_nested.yml'), nested) return nested, flat
def generate_template_version(elasticsearch_version, mappings_section, out_dir, template_settings_file): ecs_helpers.make_dirs(join(out_dir, 'elasticsearch', str(elasticsearch_version))) if template_settings_file: with open(template_settings_file) as f: template = json.load(f) else: template = default_template_settings() if elasticsearch_version == 6: template['mappings'] = {'_doc': mappings_section} else: template['mappings'] = mappings_section filename = join(out_dir, "elasticsearch/{}/template.json".format(elasticsearch_version)) save_json(filename, template)
def generate_template_version(elasticsearch_version, mappings_section, out_dir): ecs_helpers.make_dirs( join(out_dir, 'elasticsearch', str(elasticsearch_version))) template = template_settings() if elasticsearch_version == 6: template['mappings'] = {'_doc': mappings_section} else: template['mappings'] = mappings_section filename = join( out_dir, "elasticsearch/{}/template.json".format(elasticsearch_version)) save_json(filename, template)
def main(): args = argument_parser() ecs_version = read_version(args.ref) print('Running generator. ECS version ' + ecs_version) # default location to save files out_dir = 'generated' docs_dir = 'docs' if args.out: default_dirs = False out_dir = os.path.join(args.out, out_dir) docs_dir = os.path.join(args.out, docs_dir) else: default_dirs = True ecs_helpers.make_dirs(out_dir) ecs_helpers.make_dirs(docs_dir) # To debug issues in the gradual building up of the nested structure, insert # statements like this after any step of interest. # ecs_helpers.yaml_dump('ecs.yml', fields) fields = loader.load_schemas(ref=args.ref, included_files=args.include) if args.oss: oss.fallback(fields) cleaner.clean(fields, strict=args.strict) finalizer.finalize(fields) fields = subset_filter.filter(fields, args.subset, out_dir) nested, flat = intermediate_files.generate(fields, os.path.join(out_dir, 'ecs'), default_dirs) if args.intermediate_only: exit() csv_generator.generate(flat, ecs_version, out_dir) es_template.generate(flat, ecs_version, out_dir, args.template_settings, args.mapping_settings) beats.generate(nested, ecs_version, out_dir) if args.include or args.subset: exit() asciidoc_fields.generate(nested, ecs_version, docs_dir)
def write_beats_yaml(beats_file, ecs_version, out_dir): ecs_helpers.make_dirs(join(out_dir, 'beats')) warning = file_header().format(version=ecs_version) ecs_helpers.yaml_dump(join(out_dir, 'beats/fields.ecs.yml'), [beats_file], preamble=warning)
def generate(ecs_nested, ecs_flat, out_dir): ecs_helpers.make_dirs(join(out_dir, 'ecs')) ecs_helpers.yaml_dump(join(out_dir, 'ecs/ecs_flat.yml'), ecs_flat) ecs_helpers.yaml_dump(join(out_dir, 'ecs/ecs_nested.yml'), ecs_nested)