def run_ingest(config, xmlfile): with open(xmlfile, 'rb') as f: xmlstr = f.read() xmldict = xmltodict.parse(xmlstr) tables = xmldict["soap:Envelope"]["soap:Body"]["showResponse"]["showResult"]["diffgr:diffgram"]["NewDataSet"]["Table"] data = [] for t in tables: obj = { "arrival_date": t["Arrival_Date"], "district": t["District"], "market": t["Market"], "max_price": t["Max_x0020_Price"], "min_price": t["Min_x0020_Price"], "modal_price": t["Modal_x0020_Price"], "state": t["State"], "variety": t["Variety"] } if "Commodity" in t: obj["commodity"] = t["Commodity"] elif "Column1" in t: obj["commodity"] = t["Column1"] else: logger.warn("Commodity not found in %s" % t["@diffgr:id"]) continue logger.debug("Inserted %s %s %s" % (obj["commodity"], obj["market"], obj["arrival_date"])) data.append(obj) mongo_helper = MongoHelper(config) mongo_helper.rename_collection("mandi_prices") mongo_helper.save("mandi_prices", docs=data)
def run_ingest(config): ogd_request = OGDRequest(resource_id=MANDI_PRICES_RESOURCE_ID) mongo_helper = MongoHelper(config) offset = 0 limit = 100 max_count = 0 resp = ogd_request.get(offset, limit) mongo_helper.save(resp["records"]) offset += limit while resp["count"] == limit: resp = ogd_request.get(offset, limit) mongo_helper.save(resp["records"]) offset += limit if (max_count > 0) and (offset + limit > max_count): break
def run_ingest(config): ogd_request = OGDRequest(resource_id=MANDI_PRICES_RESOURCE_ID) mongo_helper = MongoHelper(config) offset = 0 limit = 100 max_count = 0 resp = ogd_request.get(offset, limit) mongo_helper.save(resp["records"]) offset += limit while resp["count"] == limit: resp = ogd_request.get(offset, limit) mongo_helper.save(resp["records"]) offset += limit if (max_count > 0) and (offset+limit > max_count): break
def run_ingest(config, xmlfile): with open(xmlfile, 'rb') as f: xmlstr = f.read() xmldict = xmltodict.parse(xmlstr) tables = xmldict["soap:Envelope"]["soap:Body"]["showResponse"][ "showResult"]["diffgr:diffgram"]["NewDataSet"]["Table"] data = [] for t in tables: obj = { "arrival_date": t["Arrival_Date"], "district": t["District"], "market": t["Market"], "max_price": t["Max_x0020_Price"], "min_price": t["Min_x0020_Price"], "modal_price": t["Modal_x0020_Price"], "state": t["State"], "variety": t["Variety"] } if "Commodity" in t: obj["commodity"] = t["Commodity"] elif "Column1" in t: obj["commodity"] = t["Column1"] else: logger.warn("Commodity not found in %s" % t["@diffgr:id"]) continue logger.debug("Inserted %s %s %s" % (obj["commodity"], obj["market"], obj["arrival_date"])) data.append(obj) mongo_helper = MongoHelper(config) mongo_helper.rename_collection("mandi_prices") mongo_helper.save("mandi_prices", docs=data)