def handle(self, *args, **options):
json = JSONSerializer()
if "datasource" not in options:
self.stderr.write("You need to specify datasource to export")
return
config = django_apps.app_configs[options["datasource"]]
ElasticModel = config.elastic_model
all_docs = ElasticModel.search()
if options["to"] is not None:
all_docs = all_docs.query(
"match_all")[options["from"]:options["to"]]
total_count = all_docs.count()
all_docs = all_docs.execute()
elif options["from"]:
all_docs = all_docs.query("match_all")[options["from"]:].execute()
total_count = all_docs.count()
all_docs = all_docs.execute()
else:
total_count = all_docs.count()
all_docs = all_docs.scan()
for doc in tqdm.tqdm(all_docs, total=total_count):
doc_json = doc.to_dict()
if not options["keep_service_fields"]:
for f in self.service_fields:
if f in doc_json:
del doc_json[f]
options["outfile"].write(json.dumps(doc_json) + "\n")
def test_serializes_numpy_floats(self):
ser = JSONSerializer()
for np_type in (
np.float_,
np.float32,
np.float64,
):
self.assertRegexpMatches(ser.dumps({"d": np_type(1.2)}),
r'^\{"d":1\.2[\d]*}$')
def test_serializes_numpy_integers(self):
ser = JSONSerializer()
for np_type in (
np.int_,
np.int8,
np.int16,
np.int32,
np.int64,
):
self.assertEqual(ser.dumps({"d": np_type(-1)}), '{"d":-1}')
for np_type in (
np.uint8,
np.uint16,
np.uint32,
np.uint64,
):
self.assertEqual(ser.dumps({"d": np_type(1)}), '{"d":1}')
def handle(self, *args, **options):
json = JSONSerializer()
all_decls = Search(index=options["indexes"]).doc_type(
NACPDeclaration, Declaration)
if options["to"] is not None:
all_decls = all_decls.query('match_all')[options["from"]:options["to"]].execute()
elif options["from"]:
all_decls = all_decls.query('match_all')[options["from"]:].execute()
else:
all_decls = all_decls.scan()
for i, decl in enumerate(all_decls):
decl_json = decl.api_response(options["sections"])
options["outfile"].write(json.dumps(decl_json) + "\n")
if i and i % 1000 == 0:
self.stderr.write("Exported %s declarations" % i)
