def run_ingest(config, xmlfile):
with open(xmlfile, 'rb') as f:
xmlstr = f.read()
xmldict = xmltodict.parse(xmlstr)
tables = xmldict["soap:Envelope"]["soap:Body"]["showResponse"]["showResult"]["diffgr:diffgram"]["NewDataSet"]["Table"]
data = []
for t in tables:
obj = {
"arrival_date": t["Arrival_Date"],
"district": t["District"],
"market": t["Market"],
"max_price": t["Max_x0020_Price"],
"min_price": t["Min_x0020_Price"],
"modal_price": t["Modal_x0020_Price"],
"state": t["State"],
"variety": t["Variety"]
}
if "Commodity" in t:
obj["commodity"] = t["Commodity"]
elif "Column1" in t:
obj["commodity"] = t["Column1"]
else:
logger.warn("Commodity not found in %s" % t["@diffgr:id"])
continue
logger.debug("Inserted %s %s %s" % (obj["commodity"], obj["market"], obj["arrival_date"]))
data.append(obj)
mongo_helper = MongoHelper(config)
mongo_helper.rename_collection("mandi_prices")
mongo_helper.save("mandi_prices", docs=data)
def run_ingest(config):
ogd_request = OGDRequest(resource_id=MANDI_PRICES_RESOURCE_ID)
mongo_helper = MongoHelper(config)
offset = 0
limit = 100
max_count = 0
resp = ogd_request.get(offset, limit)
mongo_helper.save(resp["records"])
offset += limit
while resp["count"] == limit:
resp = ogd_request.get(offset, limit)
mongo_helper.save(resp["records"])
offset += limit
if (max_count > 0) and (offset+limit > max_count):
break
def __init__(self, config):
self.mongo_helper = MongoHelper(config=config)
if not os.path.exists(CSIS["scraped_file_name"]):
scrape_cics_data()
if os.path.exists(CSIS["scraped_file_name"]):
with open(CSIS["scraped_file_name"], 'r') as sf:
data = sf.read()
self.states = json.loads(data)
else:
raise Exception("Could not download CSIS file")
def run_ingest(config):
ogd_request = OGDRequest(resource_id=MANDI_PRICES_RESOURCE_ID)
mongo_helper = MongoHelper(config)
offset = 0
limit = 100
max_count = 0
resp = ogd_request.get(offset, limit)
mongo_helper.save(resp["records"])
offset += limit
while resp["count"] == limit:
resp = ogd_request.get(offset, limit)
mongo_helper.save(resp["records"])
offset += limit
if (max_count > 0) and (offset + limit > max_count):
break
def run_ingest(config, xmlfile):
with open(xmlfile, 'rb') as f:
xmlstr = f.read()
xmldict = xmltodict.parse(xmlstr)
tables = xmldict["soap:Envelope"]["soap:Body"]["showResponse"][
"showResult"]["diffgr:diffgram"]["NewDataSet"]["Table"]
data = []
for t in tables:
obj = {
"arrival_date": t["Arrival_Date"],
"district": t["District"],
"market": t["Market"],
"max_price": t["Max_x0020_Price"],
"min_price": t["Min_x0020_Price"],
"modal_price": t["Modal_x0020_Price"],
"state": t["State"],
"variety": t["Variety"]
}
if "Commodity" in t:
obj["commodity"] = t["Commodity"]
elif "Column1" in t:
obj["commodity"] = t["Column1"]
else:
logger.warn("Commodity not found in %s" % t["@diffgr:id"])
continue
logger.debug("Inserted %s %s %s" %
(obj["commodity"], obj["market"], obj["arrival_date"]))
data.append(obj)
mongo_helper = MongoHelper(config)
mongo_helper.rename_collection("mandi_prices")
mongo_helper.save("mandi_prices", docs=data)
