l.append(d); try: print "about to add "+str(len(l)) solrCon.add_many(l) solrCon.commit() print "success" except: print "failure" exc_type, exc_value, exc_traceback = sys.exc_info() traceback.print_exception(exc_type, exc_value, exc_traceback, limit=2, file=sys.stderr) logger.error("don't know what went wrong here") def loadSolr(filename,transactions): global idcnt chunkedTransactions = list(chunks(transactions, 1000)) for chunk in chunkedTransactions: loadChunk(filename,chunk) # Before we load, we need to delete! # This seems a little dangerous, but there is not much we can do. # We really want to make this a command-line argument so # that we can load one data file at a time. # Default param for SolrDeleteExistingData in ppGuiConfig is F if SolrDeleteExistingData=='T': response = solrCon.delete_query('*:*') solrCon.commit() SearchApi.applyToLoadedFiles(PathToDataFiles,None,loadSolr,MAXIMUM_NUMBER_TO_LOAD)
def executeMain(): response = solrCon.delete_query('*:*') solrCon.commit() SearchApi.applyToLoadedFiles(PathToDataFiles, None, loadSolr, MAXIMUM_NUMBER_TO_LOAD)
exc_type, exc_value, exc_traceback = sys.exc_info() traceback.print_exception(exc_type, exc_value, exc_traceback, limit=2, file=sys.stderr) logger.error("don't know what went wrong here") def loadSolr(filename,transactions): global idcnt chunkedTransactions = list(chunks(transactions, 1000)) for chunk in chunkedTransactions: loadChunk(filename,chunk) # Before we load, we need to delete! # This seems a little dangerous, but there is not much we can do. # We really want to make this a command-line argument so # that we can load one data file at a time. # Default param for SolrDeleteExistingData in ppGuiConfig is F if SolrDeleteExistingData=='T': response = solrCon.delete_query('*:*') solrCon.commit() print "Solr Loader Starts" onlyfiles = [ f for f in listdir(PathToActualInputFiles) if isfile(join(PathToActualInputFiles,f)) ] onlycsvfiles = [ f for f in onlyfiles if re.search(".csv$",f)] for filename in onlycsvfiles: splitfiles(filename) SearchApi.applyToLoadedFiles(filename,PathToDataFiles,None,loadSolr,MAXIMUM_NUMBER_TO_LOAD) print "Solr Loader Ends"
def executeMain(): response = solrCon.delete_query('*:*') solrCon.commit() SearchApi.applyToLoadedFiles(PathToDataFiles,None,loadSolr,MAXIMUM_NUMBER_TO_LOAD)