def updateSearchClusters(self):
switchAggregatorTimeout = self.config.switchAggregatorTimeout
aggregatorWrapper = AggregatorWrapper(self.config)
for clusterConfigPath in self.clusterConfigPaths:
self.logger.info('switch cluster for: [%s]' % clusterConfigPath)
# fix switch path info
# if not aggregatorWrapper.stop(index, switchAggregatorTimeout):
# self.logger.debug('switch aggregator failed: [%s]' % clusterConfigPath)
# return error.ERROR_SWITCH_PROXY_ERROR
searchCluster = SearchCluster(clusterConfigPath, self.config)
if self.config.noCopy == clusterConfigPath or self.config.noCopy == 'all':
self.logger.info('[%s] running without Copy' % self.config.noCopy )
copyIndexRet = error.NO_ERROR
else:
copyIndexRet = searchCluster.copyIndex()
if copyIndexRet != error.NO_ERROR:
self.logger.debug('Copy index failed')
return copyIndexRet
switchIndexRet = searchCluster.switchIndex()
if switchIndexRet != error.NO_ERROR:
self.logger.debug('switch cluster failed:[%s]' % clusterConfigPath)
return switchIndexRet
return error.NO_ERROR
def buildSearchClusters(self):
(ckIncrRet, ckIncrList) = self.checkBeforeCopy()
if not ckIncrRet:
self.logger.info('don\'t find increase data.')
return error.ERROR_NO_INCR_DATA
self.logger.info(('get increase directory list: [%s]') % (ckIncrList))
for clusterConfigPath in self.clusterConfigPaths:
self.logger.info('switch cluster for: [%s]' % clusterConfigPath)
searchCluster = SearchCluster(clusterConfigPath, self.config)
copyRet, copyIncrData = searchCluster.copyIncrData(ckIncrList)
if copyRet != error.NO_ERROR:
self.logger.debug('Copy incr data failed')
return error.ERROR_BUILD_INCR_INDEX
self.logger.info(('Copy increase data sucess, the file patern [%s]' % copyIncrData))
buildIndexRet = searchCluster.buildIncr(copyIncrData)
if buildIndexRet != error.NO_ERROR:
self.logger.debug('build increase index failed:[%s]' % clusterConfigPath)
return buildIndexRet
if not searchCluster.movePg2Done(ckIncrList):
self.logger.error('move pangu to done dictionary fail')
return error.ERROR_INCR_MOVE_PG
self.logger.debug('begin to move hadoop to done')
if not searchCluster.moveHdp2Done(ckIncrList):
self.logger.error('move hadoop to done dictionary fail')
return error.ERROR_INCR_MOVE_HDP
return error.NO_ERROR
class SearchController():
def __init__(self, app):
self.app = app
self.es = DocumentES(app)
self.cluster = SearchCluster(app)
def determine_numbers(self, user_id):
# logic to determine how many numbers to use in
# elasticsearch or same cluster.
user_article_count = self.app.query_pool2.get_article_count(user_id)
number = NUMBER_DOCS
if user_article_count < 100:
rss_number = 0
cluster_number = number
elif user_article_count < 500:
rss_number = number / 3
cluster_number = number / 3 * 2
else:
rss_number = number / 3 * 2
cluster_number = number / 3
return rss_number, cluster_number
def search(self, user_id, query):
rss_number, cluster_number = self.determine_numbers(user_id)
self.app.logger.info('RSS NUMBER : %d, CLUSTER NUMBER : %d' % (rss_number, cluster_number))
if rss_number > 0:
rss_list = self.es.search(query, user_id, rss_number)
else:
rss_list = {'hits':[]}
if cluster_number > 0:
cluster_list = self.cluster.get_articles(user_id, query, cluster_number)
else:
cluster_list = []
return self.convert_list_to_json_type(rss_list, cluster_list)
def convert_list_to_json_type(self, rss_list, cluster_list):
result = {'hits':[]}
result['hits'] = rss_list['hits']
for item in cluster_list:
conv_item = {'link': item['originId'],
'summary': item['summary'],
'title': item['title'],
'_id': str(item['_id'])}
result['hits'].append(conv_item)
return result
# <div className="doc-item" onClick={this.onDocumentClick.bind(this, this.props.doc._id)}>
# <a href={this.props.doc.link} data-toggle="tooltip" title={this.props.doc.summary}>
# {this.props.doc.title}
# </a>
# </div>
def __init__(self, app):
self.app = app
self.es = DocumentES(app)
self.cluster = SearchCluster(app)
def __init__(self, app):
self.app = app
self.es = DocumentES(app)
self.cluster = SearchCluster(app)
class SearchController():
def __init__(self, app):
self.app = app
self.es = DocumentES(app)
self.cluster = SearchCluster(app)
def determine_numbers(self, user_id):
# logic to determine how many numbers to use in
# elasticsearch or same cluster.
user_article_count = self.app.query_pool2.get_article_count(user_id)
number = NUMBER_DOCS
if user_article_count < 100:
rss_number = 0
cluster_number = number
elif user_article_count < 500:
rss_number = number / 3
cluster_number = number / 3 * 2
else:
rss_number = number / 3 * 2
cluster_number = number / 3
return rss_number, cluster_number
def search(self, user_id, query):
rss_number, cluster_number = self.determine_numbers(user_id)
self.app.logger.info('RSS NUMBER : %d, CLUSTER NUMBER : %d' %
(rss_number, cluster_number))
if rss_number > 0:
rss_list = self.es.search(query, user_id, rss_number)
else:
rss_list = {'hits': []}
if cluster_number > 0:
cluster_list = self.cluster.get_articles(user_id, query,
cluster_number)
else:
cluster_list = []
return self.convert_list_to_json_type(rss_list, cluster_list)
def convert_list_to_json_type(self, rss_list, cluster_list):
result = {'hits': []}
result['hits'] = rss_list['hits']
for item in cluster_list:
conv_item = {
'link': item['originId'],
'summary': item['summary'],
'title': item['title'],
'_id': str(item['_id'])
}
result['hits'].append(conv_item)
return result
# <div className="doc-item" onClick={this.onDocumentClick.bind(this, this.props.doc._id)}>
# <a href={this.props.doc.link} data-toggle="tooltip" title={this.props.doc.summary}>
# {this.props.doc.title}
# </a>
# </div>
