def generate_template_version(elasticsearch_version, mappings_section, out_dir,
template_settings_file):
ecs_helpers.make_dirs(
join(out_dir, 'elasticsearch', str(elasticsearch_version)))
if template_settings_file:
with open(template_settings_file) as f:
template = json.load(f)
else:
template = default_template_settings()
if elasticsearch_version == 6:
es6_mappings_section = copy.deepcopy(mappings_section)
es6_type_fallback(es6_mappings_section['properties'])
# error.stack_trace needs special handling to set
# index: false and doc_values: false
error_stack_trace_mappings = es6_mappings_section['properties'][
'error']['properties']['stack_trace']
error_stack_trace_mappings.setdefault('index', False)
error_stack_trace_mappings.setdefault('doc_values', False)
template['mappings'] = {'_doc': es6_mappings_section}
else:
template['mappings'] = mappings_section
filename = join(
out_dir,
"elasticsearch/{}/template.json".format(elasticsearch_version))
save_json(filename, template)
def main():
args = argument_parser()
# Get rid of empty include
if args.include and [''] == args.include:
args.include.clear()
ecs_version = read_version()
print('Running generator. ECS version ' + ecs_version)
# Load the default schemas
print('Loading default schemas')
intermediate_fields = schema_reader.load_schemas()
# Maybe load user specified directory of schemas
if args.include:
include_glob = ecs_helpers.get_glob_files(args.include,
ecs_helpers.YAML_EXT)
print('Loading user defined schemas: {0}'.format(include_glob))
intermediate_custom = schema_reader.load_schemas(include_glob)
schema_reader.merge_schema_fields(intermediate_fields,
intermediate_custom)
schema_reader.assemble_reusables(intermediate_fields)
if args.subset:
subset = {}
for arg in args.subset:
for file in glob.glob(arg):
with open(file) as f:
raw = yaml.safe_load(f.read())
ecs_helpers.recursive_merge_subset_dicts(subset, raw)
if not subset:
raise ValueError('Subset option specified but no subsets found')
intermediate_fields = ecs_helpers.fields_subset(
subset, intermediate_fields)
(nested, flat) = schema_reader.generate_nested_flat(intermediate_fields)
# default location to save files
out_dir = 'generated'
docs_dir = 'docs'
if args.out:
out_dir = os.path.join(args.out, out_dir)
docs_dir = os.path.join(args.out, docs_dir)
ecs_helpers.make_dirs(out_dir)
ecs_helpers.make_dirs(docs_dir)
intermediate_files.generate(nested, flat, out_dir)
if args.intermediate_only:
exit()
csv_generator.generate(flat, ecs_version, out_dir)
es_template.generate(flat, ecs_version, out_dir)
beats.generate(nested, ecs_version, out_dir)
if args.include or args.subset:
exit()
asciidoc_fields.generate(intermediate_fields, ecs_version, docs_dir)
def generate_legacy_template_version(es_version, ecs_version, mappings_section,
out_dir, template_settings_file):
ecs_helpers.make_dirs(join(out_dir, 'elasticsearch', str(es_version)))
template = template_settings(es_version, ecs_version, mappings_section,
template_settings_file)
filename = join(out_dir,
"elasticsearch/{}/template.json".format(es_version))
save_json(filename, template)
def main() -> None:
args = argument_parser()
ecs_generated_version: str = read_version(args.ref)
print('Running generator. ECS version ' + ecs_generated_version)
# default location to save files
out_dir = 'generated'
docs_dir = 'docs'
if args.out:
default_dirs = False
out_dir = os.path.join(args.out, out_dir)
docs_dir = os.path.join(args.out, docs_dir)
else:
default_dirs = True
ecs_helpers.make_dirs(out_dir)
# To debug issues in the gradual building up of the nested structure, insert
# statements like this after any step of interest.
# ecs_helpers.yaml_dump('ecs.yml', fields)
# Detect usage of experimental changes to tweak artifact version label
if args.include and loader.EXPERIMENTAL_SCHEMA_DIR in args.include:
ecs_generated_version += "+exp"
print('Experimental ECS version ' + ecs_generated_version)
fields: dict[str,
FieldEntry] = loader.load_schemas(ref=args.ref,
included_files=args.include)
cleaner.clean(fields, strict=args.strict)
finalizer.finalize(fields)
fields, docs_only_fields = subset_filter.filter(fields, args.subset,
out_dir)
fields = exclude_filter.exclude(fields, args.exclude)
nested, flat = intermediate_files.generate(fields,
os.path.join(out_dir, 'ecs'),
default_dirs)
if args.intermediate_only:
exit()
csv_generator.generate(flat, ecs_generated_version, out_dir)
es_template.generate(nested, ecs_generated_version, out_dir,
args.mapping_settings, args.template_settings)
es_template.generate_legacy(flat, ecs_generated_version, out_dir,
args.mapping_settings,
args.template_settings_legacy)
beats.generate(nested, ecs_generated_version, out_dir)
if (args.include or args.subset or args.exclude) and not args.force_docs:
exit()
ecs_helpers.make_dirs(docs_dir)
docs_only_nested = intermediate_files.generate_nested_fields(
docs_only_fields)
asciidoc_fields.generate(nested, docs_only_nested, ecs_generated_version,
docs_dir)
def generate_legacy_template_version(ecs_version, mappings_section, out_dir,
template_settings_file):
ecs_helpers.make_dirs(join(out_dir, 'elasticsearch', "legacy"))
template = template_settings(ecs_version,
mappings_section,
template_settings_file,
isLegacy=True)
filename = join(out_dir, "elasticsearch/legacy/template.json")
save_json(filename, template)
def all_component_templates(ecs_nested, ecs_version, out_dir):
"""Generate one component template per field set"""
component_dir = join(out_dir, 'elasticsearch/component')
ecs_helpers.make_dirs(component_dir)
for (fieldset_name, fieldset) in candidate_components(ecs_nested).items():
field_mappings = {}
for (flat_name, field) in fieldset['fields'].items():
name_parts = flat_name.split('.')
dict_add_nested(field_mappings, name_parts, entry_for(field))
save_component_template(fieldset_name, ecs_version, component_dir, field_mappings)
def generate(fields, out_dir, default_dirs):
ecs_helpers.make_dirs(join(out_dir))
# Should only be used for debugging ECS development
if default_dirs:
ecs_helpers.yaml_dump(join(out_dir, 'ecs.yml'), fields)
flat = generate_flat_fields(fields)
nested = generate_nested_fields(fields)
ecs_helpers.yaml_dump(join(out_dir, 'ecs_flat.yml'), flat)
ecs_helpers.yaml_dump(join(out_dir, 'ecs_nested.yml'), nested)
return nested, flat
def generate_template_version(elasticsearch_version, mappings_section, out_dir, template_settings_file):
ecs_helpers.make_dirs(join(out_dir, 'elasticsearch', str(elasticsearch_version)))
if template_settings_file:
with open(template_settings_file) as f:
template = json.load(f)
else:
template = default_template_settings()
if elasticsearch_version == 6:
template['mappings'] = {'_doc': mappings_section}
else:
template['mappings'] = mappings_section
filename = join(out_dir, "elasticsearch/{}/template.json".format(elasticsearch_version))
save_json(filename, template)
def generate_template_version(elasticsearch_version, mappings_section,
out_dir):
ecs_helpers.make_dirs(
join(out_dir, 'elasticsearch', str(elasticsearch_version)))
template = template_settings()
if elasticsearch_version == 6:
template['mappings'] = {'_doc': mappings_section}
else:
template['mappings'] = mappings_section
filename = join(
out_dir,
"elasticsearch/{}/template.json".format(elasticsearch_version))
save_json(filename, template)
def main():
args = argument_parser()
ecs_version = read_version(args.ref)
print('Running generator. ECS version ' + ecs_version)
# default location to save files
out_dir = 'generated'
docs_dir = 'docs'
if args.out:
default_dirs = False
out_dir = os.path.join(args.out, out_dir)
docs_dir = os.path.join(args.out, docs_dir)
else:
default_dirs = True
ecs_helpers.make_dirs(out_dir)
ecs_helpers.make_dirs(docs_dir)
# To debug issues in the gradual building up of the nested structure, insert
# statements like this after any step of interest.
# ecs_helpers.yaml_dump('ecs.yml', fields)
fields = loader.load_schemas(ref=args.ref, included_files=args.include)
if args.oss:
oss.fallback(fields)
cleaner.clean(fields, strict=args.strict)
finalizer.finalize(fields)
fields = subset_filter.filter(fields, args.subset, out_dir)
nested, flat = intermediate_files.generate(fields,
os.path.join(out_dir, 'ecs'),
default_dirs)
if args.intermediate_only:
exit()
csv_generator.generate(flat, ecs_version, out_dir)
es_template.generate(flat, ecs_version, out_dir, args.template_settings,
args.mapping_settings)
beats.generate(nested, ecs_version, out_dir)
if args.include or args.subset:
exit()
asciidoc_fields.generate(nested, ecs_version, docs_dir)
def write_beats_yaml(beats_file, ecs_version, out_dir):
ecs_helpers.make_dirs(join(out_dir, 'beats'))
warning = file_header().format(version=ecs_version)
ecs_helpers.yaml_dump(join(out_dir, 'beats/fields.ecs.yml'), [beats_file],
preamble=warning)
def generate(ecs_nested, ecs_flat, out_dir):
ecs_helpers.make_dirs(join(out_dir, 'ecs'))
ecs_helpers.yaml_dump(join(out_dir, 'ecs/ecs_flat.yml'), ecs_flat)
ecs_helpers.yaml_dump(join(out_dir, 'ecs/ecs_nested.yml'), ecs_nested)
