def collect_data(self, servers, buckets, userId="Administrator", password="******", data_path = None, perNode = True, getReplica = False, mode = "memory"): """ Method to extract all data information from memory or disk using cbtransfer The output is organized like { bucket :{ node { document-key : list of values }}} Paramters: servers: server information bucket: bucket information userId: user id of cb server password: password of cb server data_path: data path on servers, if given we will do cbtransfer on files perNode: if set we organize data for each bucket per node basis else we take a union Returns: If perNode flag is set we return data as follows {bucket {node { key: value list}}} else {bucket {key: value list}} """ completeMap = {} for bucket in buckets: completeMap[bucket.name] = {} headerInfo = None for server in servers: if mode == "disk" and data_path == None: rest = RestConnection(server) data_path = rest.get_data_path() headerInfo = [] bucketMap = {} if server.ip == "127.0.0.1": headerInfo,bucketMap = self.get_local_data_map_using_cbtransfer(server, buckets, data_path=data_path, userId=userId, password=password, getReplica = getReplica, mode = mode) else: remote_client = RemoteMachineShellConnection(server) headerInfo,bucketMap = remote_client.get_data_map_using_cbtransfer(buckets, data_path=data_path, userId=userId, password=password, getReplica = getReplica, mode = mode) remote_client.disconnect() for bucket in bucketMap.keys(): newMap = self.translateDataFromCSVToMap(0,bucketMap[bucket]) if perNode: completeMap[bucket][server.ip] = newMap else: completeMap[bucket].update(newMap) return headerInfo,completeMap
def collect_data(self,servers,buckets,userId="Administrator",password="******", data_path = None, perNode = True, getReplica = False, mode = "memory"): """ Method to extract all data information from memory or disk using cbtransfer The output is organized like { bucket :{ node { document-key : list of values }}} Paramters: servers: server information bucket: bucket information userId: user id of cb server password: password of cb server data_path: data path on servers, if given we will do cbtransfer on files perNode: if set we organize data for each bucket per node basis else we take a union Returns: If perNode flag is set we return data as follows {bucket {node { key: value list}}} else {bucket {key: value list}} """ completeMap = {} for bucket in buckets: completeMap[bucket.name] = {} headerInfo = None for server in servers: if mode == "disk" and data_path == None: rest = RestConnection(server) data_path = rest.get_data_path() headerInfo = [] bucketMap = {} if server.ip == "127.0.0.1": headerInfo,bucketMap = self.get_local_data_map_using_cbtransfer(server,buckets, data_path=data_path, userId=userId,password=password, getReplica = getReplica, mode = mode) else: remote_client = RemoteMachineShellConnection(server) headerInfo,bucketMap = remote_client.get_data_map_using_cbtransfer(buckets, data_path=data_path, userId=userId,password=password, getReplica = getReplica, mode = mode) remote_client.disconnect() for bucket in bucketMap.keys(): newMap = self.translateDataFromCSVToMap(0,bucketMap[bucket]) if perNode: completeMap[bucket][server.ip] = newMap else: completeMap[bucket].update(newMap) return headerInfo,completeMap