def create_batch_geocodes(self):
mandi_prices = self.mongo_helper.db["mandi_prices"]
mandi_locations = self.mongo_helper.db["mandi_locations"]
cursor = mandi_prices.find()
for mp in cursor:
state = (mp["state"]).lower()
district = (mp["district"]).lower()
market = (mp["market"]).lower()
query = {"state": state, "district": district, "market": market}
logger.info("geocoding mandi %s" % mp["_id"])
ml = mandi_locations.find_one(query)
if ml is None:
logger.warn("no mandi location for mandi %s, %s, %s, %s" %
(mp["_id"], state, district, market))
else:
#if not "cics_loc" in mp:
if "cics_geocode" in ml:
if isinstance(ml["cics_geocode"], list):
score = -1
nm_max = None
for cg in ml["cics_geocode"]:
if int(cg["@score"]) > score:
nm_max = cg
score = int(cg["@score"])
if nm_max:
cics_loc = {
"type": "Point",
"coordinates": [float(nm_max["@lng"]), float(nm_max["@lat"])]
}
mandi_prices.update({"_id": mp["_id"]}, {"$set": {"cics_loc": cics_loc}})
if isinstance(ml["cics_geocode"], dict):
cics_loc = {
"type": "Point",
"coordinates": [float(ml["cics_geocode"]["@lng"]), float(ml["cics_geocode"]["@lat"])]
}
mandi_prices.update({"_id": mp["_id"]}, {"$set": {"cics_loc": cics_loc}})
#if not "nm_loc" in mp:
if "nm_geocode" in ml:
score = -1
nm_max = None
for nm in ml["nm_geocode"]:
if ("importance" in nm) and (float(nm["importance"]) > score):
nm_max = nm
score = float(nm["importance"])
if nm_max:
nm_loc = {
"type": "Point",
"coordinates": [nm_max["lon"], nm_max["lat"]]
}
mandi_prices.update({"_id": mp["_id"]}, {"$set": {"nm_loc": nm_loc}})
def create_batch_geocodes(self):
mandi_prices = self.mongo_helper.db["mandi_prices"]
mandi_locations = self.mongo_helper.db["mandi_locations"]
cursor = mandi_prices.find()
for mp in cursor:
state = (mp["state"]).lower()
district = (mp["district"]).lower()
market = (mp["market"]).lower()
query = {"state": state, "district": district, "market": market}
logger.info("geocoding mandi %s" % mp["_id"])
ml = mandi_locations.find_one(query)
if ml is None:
logger.warn("no mandi location for mandi %s, %s, %s, %s" %
(mp["_id"], state, district, market))
else:
#if not "cics_loc" in mp:
if "cics_geocode" in ml:
if isinstance(ml["cics_geocode"], list):
score = -1
nm_max = None
for cg in ml["cics_geocode"]:
if int(cg["@score"]) > score:
nm_max = cg
score = int(cg["@score"])
if nm_max:
cics_loc = {
"type":
"Point",
"coordinates":
[float(nm_max["@lng"]),
float(nm_max["@lat"])]
}
mandi_prices.update(
{"_id": mp["_id"]},
{"$set": {
"cics_loc": cics_loc
}})
if isinstance(ml["cics_geocode"], dict):
cics_loc = {
"type":
"Point",
"coordinates": [
float(ml["cics_geocode"]["@lng"]),
float(ml["cics_geocode"]["@lat"])
]
}
mandi_prices.update({"_id": mp["_id"]},
{"$set": {
"cics_loc": cics_loc
}})
#if not "nm_loc" in mp:
if "nm_geocode" in ml:
score = -1
nm_max = None
for nm in ml["nm_geocode"]:
if ("importance"
in nm) and (float(nm["importance"]) > score):
nm_max = nm
score = float(nm["importance"])
if nm_max:
nm_loc = {
"type": "Point",
"coordinates": [nm_max["lon"], nm_max["lat"]]
}
mandi_prices.update({"_id": mp["_id"]},
{"$set": {
"nm_loc": nm_loc
}})
"min_price": t["Min_x0020_Price"],
"modal_price": t["Modal_x0020_Price"],
"state": t["State"],
"variety": t["Variety"]
}
if "Commodity" in t:
obj["commodity"] = t["Commodity"]
elif "Column1" in t:
obj["commodity"] = t["Column1"]
else:
logger.warn("Commodity not found in %s" % t["@diffgr:id"])
continue
logger.debug("Inserted %s %s %s" %
(obj["commodity"], obj["market"], obj["arrival_date"]))
data.append(obj)
mongo_helper = MongoHelper(config)
mongo_helper.rename_collection("mandi_prices")
mongo_helper.save("mandi_prices", docs=data)
if __name__ == "__main__":
config = get_config(env=get_env())
xmlfile = download_file()
if xmlfile:
logger.info("Downloaded %s" % xmlfile)
run_ingest(config, xmlfile)
else:
logger.warn("Latest file %s already ingested" % xmlfile)
"max_price": t["Max_x0020_Price"],
"min_price": t["Min_x0020_Price"],
"modal_price": t["Modal_x0020_Price"],
"state": t["State"],
"variety": t["Variety"]
}
if "Commodity" in t:
obj["commodity"] = t["Commodity"]
elif "Column1" in t:
obj["commodity"] = t["Column1"]
else:
logger.warn("Commodity not found in %s" % t["@diffgr:id"])
continue
logger.debug("Inserted %s %s %s" % (obj["commodity"], obj["market"], obj["arrival_date"]))
data.append(obj)
mongo_helper = MongoHelper(config)
mongo_helper.rename_collection("mandi_prices")
mongo_helper.save("mandi_prices", docs=data)
if __name__ == "__main__":
config = get_config(env=get_env())
xmlfile = download_file()
if xmlfile:
logger.info("Downloaded %s" % xmlfile)
run_ingest(config, xmlfile)
else:
logger.warn("Latest file %s already ingested" % xmlfile)