def updateSearchClusters(self): switchAggregatorTimeout = self.config.switchAggregatorTimeout aggregatorWrapper = AggregatorWrapper(self.config) for clusterConfigPath in self.clusterConfigPaths: self.logger.info('switch cluster for: [%s]' % clusterConfigPath) # fix switch path info # if not aggregatorWrapper.stop(index, switchAggregatorTimeout): # self.logger.debug('switch aggregator failed: [%s]' % clusterConfigPath) # return error.ERROR_SWITCH_PROXY_ERROR searchCluster = SearchCluster(clusterConfigPath, self.config) if self.config.noCopy == clusterConfigPath or self.config.noCopy == 'all': self.logger.info('[%s] running without Copy' % self.config.noCopy ) copyIndexRet = error.NO_ERROR else: copyIndexRet = searchCluster.copyIndex() if copyIndexRet != error.NO_ERROR: self.logger.debug('Copy index failed') return copyIndexRet switchIndexRet = searchCluster.switchIndex() if switchIndexRet != error.NO_ERROR: self.logger.debug('switch cluster failed:[%s]' % clusterConfigPath) return switchIndexRet return error.NO_ERROR
def buildSearchClusters(self): (ckIncrRet, ckIncrList) = self.checkBeforeCopy() if not ckIncrRet: self.logger.info('don\'t find increase data.') return error.ERROR_NO_INCR_DATA self.logger.info(('get increase directory list: [%s]') % (ckIncrList)) for clusterConfigPath in self.clusterConfigPaths: self.logger.info('switch cluster for: [%s]' % clusterConfigPath) searchCluster = SearchCluster(clusterConfigPath, self.config) copyRet, copyIncrData = searchCluster.copyIncrData(ckIncrList) if copyRet != error.NO_ERROR: self.logger.debug('Copy incr data failed') return error.ERROR_BUILD_INCR_INDEX self.logger.info(('Copy increase data sucess, the file patern [%s]' % copyIncrData)) buildIndexRet = searchCluster.buildIncr(copyIncrData) if buildIndexRet != error.NO_ERROR: self.logger.debug('build increase index failed:[%s]' % clusterConfigPath) return buildIndexRet if not searchCluster.movePg2Done(ckIncrList): self.logger.error('move pangu to done dictionary fail') return error.ERROR_INCR_MOVE_PG self.logger.debug('begin to move hadoop to done') if not searchCluster.moveHdp2Done(ckIncrList): self.logger.error('move hadoop to done dictionary fail') return error.ERROR_INCR_MOVE_HDP return error.NO_ERROR
class SearchController(): def __init__(self, app): self.app = app self.es = DocumentES(app) self.cluster = SearchCluster(app) def determine_numbers(self, user_id): # logic to determine how many numbers to use in # elasticsearch or same cluster. user_article_count = self.app.query_pool2.get_article_count(user_id) number = NUMBER_DOCS if user_article_count < 100: rss_number = 0 cluster_number = number elif user_article_count < 500: rss_number = number / 3 cluster_number = number / 3 * 2 else: rss_number = number / 3 * 2 cluster_number = number / 3 return rss_number, cluster_number def search(self, user_id, query): rss_number, cluster_number = self.determine_numbers(user_id) self.app.logger.info('RSS NUMBER : %d, CLUSTER NUMBER : %d' % (rss_number, cluster_number)) if rss_number > 0: rss_list = self.es.search(query, user_id, rss_number) else: rss_list = {'hits':[]} if cluster_number > 0: cluster_list = self.cluster.get_articles(user_id, query, cluster_number) else: cluster_list = [] return self.convert_list_to_json_type(rss_list, cluster_list) def convert_list_to_json_type(self, rss_list, cluster_list): result = {'hits':[]} result['hits'] = rss_list['hits'] for item in cluster_list: conv_item = {'link': item['originId'], 'summary': item['summary'], 'title': item['title'], '_id': str(item['_id'])} result['hits'].append(conv_item) return result # <div className="doc-item" onClick={this.onDocumentClick.bind(this, this.props.doc._id)}> # <a href={this.props.doc.link} data-toggle="tooltip" title={this.props.doc.summary}> # {this.props.doc.title} # </a> # </div>
def __init__(self, app): self.app = app self.es = DocumentES(app) self.cluster = SearchCluster(app)
class SearchController(): def __init__(self, app): self.app = app self.es = DocumentES(app) self.cluster = SearchCluster(app) def determine_numbers(self, user_id): # logic to determine how many numbers to use in # elasticsearch or same cluster. user_article_count = self.app.query_pool2.get_article_count(user_id) number = NUMBER_DOCS if user_article_count < 100: rss_number = 0 cluster_number = number elif user_article_count < 500: rss_number = number / 3 cluster_number = number / 3 * 2 else: rss_number = number / 3 * 2 cluster_number = number / 3 return rss_number, cluster_number def search(self, user_id, query): rss_number, cluster_number = self.determine_numbers(user_id) self.app.logger.info('RSS NUMBER : %d, CLUSTER NUMBER : %d' % (rss_number, cluster_number)) if rss_number > 0: rss_list = self.es.search(query, user_id, rss_number) else: rss_list = {'hits': []} if cluster_number > 0: cluster_list = self.cluster.get_articles(user_id, query, cluster_number) else: cluster_list = [] return self.convert_list_to_json_type(rss_list, cluster_list) def convert_list_to_json_type(self, rss_list, cluster_list): result = {'hits': []} result['hits'] = rss_list['hits'] for item in cluster_list: conv_item = { 'link': item['originId'], 'summary': item['summary'], 'title': item['title'], '_id': str(item['_id']) } result['hits'].append(conv_item) return result # <div className="doc-item" onClick={this.onDocumentClick.bind(this, this.props.doc._id)}> # <a href={this.props.doc.link} data-toggle="tooltip" title={this.props.doc.summary}> # {this.props.doc.title} # </a> # </div>