def main(input,delete,resume,config): config=json.loads(config.read()) config.update(json.loads(input.read())) client = IODClient("http://api.idolondemand.com/", config["iodkey"]) if delete: deleteIndex(config["iodindex"]) category=False description=False flavor="standard" if config["type"]=="categories": category=True flavor="categorization" if config.get("description",False): description=True try: index=client.createIndex(config["iodindex"],flavor="categorization",index_fields=config.get("indexfields",[]), parametric_fields=config.get("parametricfields",[])) except: index=client.getIndex(config["iodindex"]) cursor="" if resume: cursor= open('cursor','rb').read() query= config["freebasequery"] freebaseUtil = FreebaseUtil(config["freebasekey"]) freebaseUtil.aliases=config["aliasfields"] freebaseUtil.runQuery(index,query,category=category,description=description,cursor=cursor)
client = IODClient("http://api.idolondemand.com/", iod_key) # client.deleteIndex('quotesdb') # try: # index=client.deleteIndex('comic_series') # except: # pass try: fields=["created_by_name","publisher_name","source_type_name","genre_name","continues_name","continued_by_name"] index=client.createIndex('comic_series',index_fields=fields, parametric_fields=fields) # index=client.createIndex('quotesdb',index_fields=["spoken_by_character_*","author_*","source_*","incorrectly_attributed_to_*","addressee_*"],parametric_fields=["spoken_by_character_*","author_*","source_*","incorrectly_attributed_to_*","addressee_*"]) print "sleeping" time.sleep(5) print "resuming" except: index=client.getIndex('comic_series') #except: ## print "getting instead" #index= client.getIndex('comic_book_series') #cursor = freebaseUtil.do_query(index) cursor = sync.get("cursor","") if not cursor: cursor = freebaseUtil.do_query(index)
#print json.dumps(flatten(result),indent=4) #print result["continues"] print index.name try: print "trying to index" print index.commit(async=True).jobID except: print "indexing failed" return response.get("cursor") client = IODClient("http://api.idolondemand.com/", iod_key) #try: client.deleteIndex('comic_series') index=client.createIndex('comic_series') #except: ## print "getting instead" #index= client.getIndex('comic_book_series') cursor = do_query(index) while(cursor): cursor = do_query(index,cursor)
syncdb=shelve.open("sync.db") # Initiate Html stripper sourcelist= json.loads(open('sources.json').read()) schedulesecs=config["schedulesecs"] database=config["database"] keyconf = shelve.open('../config.db') iod_key=keyconf['iod-key'] client = IODClient("http://api.idolondemand.com/", iod_key) #try: try: index=client.createIndex(database) print index time.sleep(3) except: print "getting instead" index=client.getIndex(database) def main(): start() if __name__ == "__main__": main() # time.sleep(5)