l.append(d);
try:
print "about to add "+str(len(l))
solrCon.add_many(l)
solrCon.commit()
print "success"
except:
print "failure"
exc_type, exc_value, exc_traceback = sys.exc_info()
traceback.print_exception(exc_type, exc_value, exc_traceback,
limit=2, file=sys.stderr)
logger.error("don't know what went wrong here")
def loadSolr(filename,transactions):
global idcnt
chunkedTransactions = list(chunks(transactions, 1000))
for chunk in chunkedTransactions:
loadChunk(filename,chunk)
# Before we load, we need to delete!
# This seems a little dangerous, but there is not much we can do.
# We really want to make this a command-line argument so
# that we can load one data file at a time.
# Default param for SolrDeleteExistingData in ppGuiConfig is F
if SolrDeleteExistingData=='T':
response = solrCon.delete_query('*:*')
solrCon.commit()
SearchApi.applyToLoadedFiles(PathToDataFiles,None,loadSolr,MAXIMUM_NUMBER_TO_LOAD)
def executeMain():
response = solrCon.delete_query('*:*')
solrCon.commit()
SearchApi.applyToLoadedFiles(PathToDataFiles, None, loadSolr,
MAXIMUM_NUMBER_TO_LOAD)
exc_type, exc_value, exc_traceback = sys.exc_info()
traceback.print_exception(exc_type, exc_value, exc_traceback,
limit=2, file=sys.stderr)
logger.error("don't know what went wrong here")
def loadSolr(filename,transactions):
global idcnt
chunkedTransactions = list(chunks(transactions, 1000))
for chunk in chunkedTransactions:
loadChunk(filename,chunk)
# Before we load, we need to delete!
# This seems a little dangerous, but there is not much we can do.
# We really want to make this a command-line argument so
# that we can load one data file at a time.
# Default param for SolrDeleteExistingData in ppGuiConfig is F
if SolrDeleteExistingData=='T':
response = solrCon.delete_query('*:*')
solrCon.commit()
print "Solr Loader Starts"
onlyfiles = [ f for f in listdir(PathToActualInputFiles) if isfile(join(PathToActualInputFiles,f)) ]
onlycsvfiles = [ f for f in onlyfiles if re.search(".csv$",f)]
for filename in onlycsvfiles:
splitfiles(filename)
SearchApi.applyToLoadedFiles(filename,PathToDataFiles,None,loadSolr,MAXIMUM_NUMBER_TO_LOAD)
print "Solr Loader Ends"
def executeMain():
response = solrCon.delete_query('*:*')
solrCon.commit()
SearchApi.applyToLoadedFiles(PathToDataFiles,None,loadSolr,MAXIMUM_NUMBER_TO_LOAD)
