def resolve_flat(self, info, **kwargs):
value = reduce(assign,
read_json(join(Args.logdir, self.id[1:])) or [{}])
return dot_flatten(value)
def resolve_raw(self, info, **kwargs):
return read_json(join(Args.logdir, self.id[1:]))
def resolve_keys(self, info):
value = reduce(assign,
read_json(join(Args.logdir, self.id[1:])) or [{}])
return dot_keys(value)
def resolve_value(self, info, **kwargs):
return reduce(assign,
read_json(join(Args.logdir, self.id[1:])) or [{}])
}
},
"boolean": {"type": "boolean"},
"date": {"type": "date",
"format": "yyyy-MM-dd'T'HH:mm:ss.SSSSSS"},
"long": {"type": "long"},
"float": {"type": "float"},
"null": {"type": "boolean", "null_value": False}
}
}
}
)))
for chunk in tqdm(chunked(parameter_files, 1000), desc="Uploading..."):
parameters = [reduce(assign, [
*(read_json(join(cwd, f['path'])) or []), {"dir": f['dir']}
]) for f in chunk]
actions = [{"index": dict(_id=p['dir'], )} for p in parameters]
documents = [dict(index=[dict(key=k, **v) for k, v in typify(dot_flatten(p)).items()], **p)
for p in parameters]
# documents[0]
# https://stackoverflow.com/questions/20288770/how-to-use-bulk-api-to-store-the-keywords-in-es-by-using-python
response = es.bulk(index='ml-dash', body=interleave(actions, documents))
if response['errors']:
for i, item in enumerate(response['items']):
if item['index']['status'] >= 300:
print(item['index'])
print(documents[i])