def main():
args = argument_parser()
ecs_version = read_version()
print 'Running generator. ECS version ' + ecs_version
# Load the default schemas
print 'Loading default schemas'
(nested, flat) = schema_reader.load_schemas()
# Maybe load user specified directory of schemas
if args.include:
include_glob = os.path.join(args.include, '*.yml')
print 'Loading user defined schemas: {0}'.format(include_glob)
(custom_nested, custom_flat) = schema_reader.load_schemas(
sorted(glob.glob(include_glob)))
# Merge without allowing user schemas to overwrite default schemas
nested = ecs_helpers.safe_merge_dicts(nested, custom_nested)
flat = ecs_helpers.safe_merge_dicts(flat, custom_flat)
intermediate_files.generate(nested, flat)
if args.intermediate_only:
exit
csv_generator.generate(flat, ecs_version)
es_template.generate(flat, ecs_version)
beats.generate(nested, ecs_version)
asciidoc_fields.generate(nested, ecs_version)
def load_schemas_from_git(ref):
tree = ecs_helpers.get_tree_by_ref(ref)
fields_nested = {}
for blob in tree['schemas'].blobs:
if blob.name.endswith('.yml'):
new_fields = read_schema_blob(blob, ref)
fields_nested = ecs_helpers.safe_merge_dicts(
fields_nested, new_fields)
return fields_nested
def load_schemas_from_git(ref, target_dir='schemas'):
tree = ecs_helpers.get_tree_by_ref(ref)
fields_nested = {}
# Handles case if target dir doesn't exists in git ref
if ecs_helpers.path_exists_in_git_tree(tree, target_dir):
for blob in tree[target_dir].blobs:
if blob.name.endswith('.yml'):
new_fields = read_schema_blob(blob, ref)
fields_nested = ecs_helpers.safe_merge_dicts(
fields_nested, new_fields)
else:
raise KeyError(
f"Target directory './{target_dir}' not present in git ref '{ref}'!"
)
return fields_nested
def load_schema_files(files):
fields_nested = {}
for f in files:
new_fields = read_schema_file(f)
fields_nested = ecs_helpers.safe_merge_dicts(fields_nested, new_fields)
return fields_nested
def load_schema_files(files: List[str]) -> Dict[str, FieldNestedEntry]:
fields_nested: Dict[str, FieldNestedEntry] = {}
for f in files:
new_fields: Dict[str, FieldNestedEntry] = read_schema_file(f)
fields_nested = ecs_helpers.safe_merge_dicts(fields_nested, new_fields)
return fields_nested