def _query(self,query,user_id,es_from=0):
total=0
took = 0
datahub = Datahub(CONTEXT,user_id=user_id)
result = datahub.get(query)
if result is not None:
result_type = result['display']
data = result['data']
data['description'] = result['description']
data['source'] = result['source']
data['zone'] = result['zone']
data['id'] = query
data['name'] = result['name']
else:
result,total,took = datahub.query(query,es_from=es_from)
if result is not None: #the query match a datahub entry
if len(result) == 1:
id = result[0][0]
result = result[0][1]
result_type = result['display']
data = result['data']
data['description'] = result['description']
data['source'] = result['source']
data['zone'] = result['zone']
data['name'] = result['name']
data['id'] = id
else:
multi_timelines = len(result) <= 5
if multi_timelines:
for (id,serie) in result:
if serie['display'] != 'timeline':
multi_timelines = False
break
if multi_timelines:
series = []
result_type = "timeline" #force to timeline
data = { "series" : map(lambda (id,item) : item['data']['series'][0] ,result) }
else:
data = result
result_type = 'serie-list'
else:
data = []
result_type = 'serie-list'
return (result_type,data,total,took)
def index_query(self):
if 'query' not in self.ctx.params:
return self.error('No query defined')
else:
query = self.ctx.params['query']
(result_type,data) = self._query(query)
datahub = Datahub(CONTEXT,user_id=self.ctx.user.id)
serie_description = "%s dans %s" % (query,self.ftname)
id_index = datahub.index(query,result_type,data, description=serie_description)
return self.success(id=id_index)
def search(self, search_timeserie,filters=[],lag=12,kernel='linear'):
if self._debug or self._pool:
lag_variable = broadcast(lag)
kernel_variable = broadcast(kernel)
search_timeserie_data = broadcast(transform_serie(search_timeserie))
if self._debug:
result = filter(filter_correlation_results, map(tuple_correlation_search_map,read_index(self.index_file_name,filters,search_timeserie_data,lag_variable,kernel_variable)))
else:
pool = Pool()
result = filter(filter_correlation_results, pool.map(tuple_correlation_search_map,read_index(self.index_file_name,filters,search_timeserie_data,lag_variable,kernel_variable)))
#expand with user series correlation
dh = Datahub(self.context,user_id=self.user.id)
if self._debug:
result.extend(filter(filter_correlation_results, map(lambda value : correlation_search_map(value,search_timeserie_data,lag_variable,kernel_variable),dh.get_user_series())))
else:
result.extend(filter(filter_correlation_results, pool.map(tuple_correlation_search_map,read_from_list(dh.get_user_series(),search_timeserie_data,lag_variable,kernel_variable))))
return result
else:
lag_variable = self._sc.broadcast(lag)
kernel_variable = self._sc.broadcast(kernel)
search_timeserie_data = self._sc.broadcast(transform_serie(search_timeserie))
search_result_rdd = self._index_rdd.filter(lambda line : line.split(';')[3].strip() not in filters).map(lambda value : correlation_search_map(value,search_timeserie_data,lag_variable,kernel_variable))
search_result = search_result_rdd.filter(filter_correlation_results).collect()
pool = Pool()
lag_variable = broadcast(lag)
kernel_variable = broadcast(kernel)
search_timeserie_data = broadcast(transform_serie(search_timeserie))
#expand with user series correlation
dh = Datahub(self.context,user_id=self.user.id)
search_result.extend(filter(filter_correlation_results, pool.map(tuple_correlation_search_map,read_from_list(dh.get_user_series(),search_timeserie_data,lag_variable,kernel_variable))))
return search_result