def createTable(self, cluster, tableName, *columns): client = self.connectCluster(cluster) client.createTable(tableName, [ get_thrift_type('ColumnDescriptor')(name=column) for column in columns ]) return "%s successfully created" % tableName
def putUpload(self, cluster, tableName, row, column, value): client = self.connectCluster(cluster) Mutation = get_thrift_type('Mutation') return client.mutateRow(tableName, row, [ Mutation(column=smart_str(column), value=value.file.read(value.size)) ], None)
def getRowQuerySet(self, cluster, tableName, columns, queries): client = self.connectCluster(cluster) aggregate_data = [] limit = conf.TRUNCATE_LIMIT.get() queries = sorted(queries, key=lambda query: query["scan_length"]) # sort by scan length for query in queries: scan_length = int(query["scan_length"]) if query["row_key"] == "null": query["row_key"] = "" fs = query.get("filter", None) if fs: fs = " AND (" + fs.strip() + ")" filterstring = "(ColumnPaginationFilter(%i,0) AND PageFilter(%i))" % (limit, limit) + (fs or "") scan = get_thrift_type("TScan")( startRow=smart_str(query["row_key"]), stopRow=None, timestamp=None, columns=[smart_str(column) for column in (query["columns"] or columns)], caching=None, filterString=filterstring, batchSize=None, ) scanner = client.scannerOpenWithScan(tableName, scan, None) aggregate_data += client.scannerGetList(scanner, query["scan_length"]) return aggregate_data
def getRowQuerySet(self, cluster, tableName, columns, queries): client = self.connectCluster(cluster) aggregate_data = [] limit = conf.TRUNCATE_LIMIT.get() queries = sorted( queries, key=lambda query: query['scan_length']) #sort by scan length for query in queries: scan_length = int(query['scan_length']) if query['row_key'] == "null": query['row_key'] = "" fs = query.get('filter', None) if fs: fs = " AND (" + fs.strip() + ")" filterstring = "(ColumnPaginationFilter(%i,0) AND PageFilter(%i))" % ( limit, limit) + (fs or "") scan_columns = [ smart_str(column.strip(':')) for column in query['columns'] ] or [smart_str(column.strip(':')) for column in columns] scan = get_thrift_type('TScan')(startRow=smart_str( query['row_key']), stopRow=None, timestamp=None, columns=scan_columns, caching=None, filterString=filterstring, batchSize=None) scanner = client.scannerOpenWithScan(tableName, scan, None, doas=self.user.username) aggregate_data += client.scannerGetList(scanner, query['scan_length'], doas=self.user.username) return aggregate_data
def putRow(self, cluster, tableName, row, data): client = self.connectCluster(cluster) mutations = [] Mutation = get_thrift_type('Mutation') for column in data.keys(): mutations.append(Mutation(column=smart_str(column), value=smart_str(data[column]))) # must use str for API, does thrift coerce by itself? return client.mutateRow(tableName, smart_str(row), mutations, None)
def getAutocompleteRows(self, cluster, tableName, numRows, query): try: client = self.connectCluster(cluster) scan = get_thrift_type('TScan')(startRow=query, stopRow=None, timestamp=None, columns=[], caching=None, filterString="PrefixFilter('" + query + "') AND ColumnPaginationFilter(1,0)", batchSize=None) scanner = client.scannerOpenWithScan(tableName, scan, None) return [result.row for result in client.scannerGetList(scanner, numRows)] except: return []
def createTable(self, cluster, tableName, columns): client = self.connectCluster(cluster) client.createTable(tableName, [ get_thrift_type('ColumnDescriptor')(**column['properties']) for column in columns ], doas=self.user.username) return "%s successfully created" % tableName
def deleteColumns(self, cluster, tableName, row, columns): client = self.connectCluster(cluster) Mutation = get_thrift_type('Mutation') mutations = [ Mutation(isDelete=True, column=smart_str(column)) for column in columns ] return client.mutateRow(tableName, smart_str(row), mutations, None)
def deserialize(data): if type(data) == dict: special_type = get_thrift_type(data.pop('hue-thrift-type', '')) if special_type: return special_type(data) if hasattr(data, "__iter__"): for i, item in enumerate(data): data[i] = deserialize(item) # Sets local binding, needs to set in data return data
def bulkUpload(self, cluster, tableName, data): client = self.connectCluster(cluster) BatchMutation = get_thrift_type('BatchMutation') Mutation = get_thrift_type('Mutation') columns = [] data = data.read() dialect = csv.Sniffer().sniff(data) reader = csv.reader(data.splitlines(), delimiter=dialect.delimiter) columns = reader.next() batches = [] for row in reader: row_key = row[0] row_data = {} mutations = [] for column_index in range(1, len(row)): if str(row[column_index]) != "": mutations.append(Mutation(column=smart_str(columns[column_index]), value=smart_str(row[column_index]))) batches += [BatchMutation(row=row_key, mutations=mutations)] client.mutateRows(tableName, batches, None, doas=self.user.username) return True
def bulkUpload(self, cluster, tableName, data): client = self.connectCluster(cluster) BatchMutation = get_thrift_type('BatchMutation') Mutation = get_thrift_type('Mutation') columns = [] data = data.read() dialect = csv.Sniffer().sniff(data) reader = csv.reader(data.splitlines(), delimiter=dialect.delimiter) columns = reader.next() batches = [] for row in reader: row_key = row[0] row_data = {} mutations = [] for column_index in range(1, len(row)): if str(row[column_index]) != "": mutations.append(Mutation(column=smart_str(columns[column_index]), value=smart_str(row[column_index]))) batches += [BatchMutation(row=row_key, mutations=mutations)] client.mutateRows(tableName, batches, None) return True
def getRowPartial(self, cluster, tableName, rowKey, offset, number): client = self.connectCluster(cluster) scan = get_thrift_type('TScan')( startRow=rowKey, stopRow=None, timestamp=None, columns=[], caching=None, filterString="ColumnPaginationFilter(%i, %i)" % (number, offset), batchSize=None) scanner = client.scannerOpenWithScan(tableName, scan, None) return client.scannerGetList(scanner, 1)
def getRowPartial(self, cluster, tableName, rowKey, offset, number): client = self.connectCluster(cluster) scan = get_thrift_type("TScan")( startRow=rowKey, stopRow=None, timestamp=None, columns=[], caching=None, filterString="ColumnPaginationFilter(%i, %i)" % (number, offset), batchSize=None, ) scanner = client.scannerOpenWithScan(tableName, scan, None) return client.scannerGetList(scanner, 1)
def putRow(self, cluster, tableName, row, data): client = self.connectCluster(cluster) mutations = [] Mutation = get_thrift_type('Mutation') for column in list(data.keys()): value = smart_str( data[column]) if data[column] is not None else None mutations.append( Mutation(column=smart_str(column), value=value) ) # must use str for API, does thrift coerce by itself? return client.mutateRow(tableName, smart_str(row), mutations, None, doas=self.user.username)
def getRowQuerySet(self, cluster, tableName, columns, queries): client = self.connectCluster(cluster) aggregate_data = [] limit = conf.TRUNCATE_LIMIT.get() queries = sorted(queries, key=lambda query: query['scan_length']) #sort by scan length for query in queries: scan_length = int(query['scan_length']) if query['row_key'] == "null": query['row_key'] = "" fs = query.get('filter', None) if fs: fs = " AND (" + fs + ")" filterstring = "(ColumnPaginationFilter(" + str(limit) + ",0) AND PageFilter(" + str(limit) + "))" + (fs or "") scan = get_thrift_type('TScan')(startRow=query['row_key'], stopRow=None, timestamp=None, columns=query['columns'] or columns, caching=None, filterString=filterstring, batchSize=None) scanner = client.scannerOpenWithScan(tableName, scan, None) aggregate_data += client.scannerGetList(scanner, query['scan_length']) return aggregate_data
def getRowQuerySet(self, cluster, tableName, columns, queries): client = self.connectCluster(cluster) aggregate_data = [] queries = sorted(queries, key=lambda query: query['scan_length']) #sort by scan length for query in queries: scan_length = int(query['scan_length']) if query['row_key'] == "null": query['row_key'] = "" filterstring = query.get('filter', None) if filterstring: scan = get_thrift_type('TScan')(startRow=query['row_key'], stopRow=None, timestamp=None, columns=query['columns'] or columns, caching=None, filterString=filterstring, batchSize=None) scanner = client.scannerOpenWithScan(tableName, scan, None) aggregate_data += client.scannerGetList(scanner, query['scan_length']) else: if scan_length == 1: aggregate_data += self.getRows(cluster, tableName, query['columns'] or columns, query['row_key'], query['scan_length'], query['prefix']) #to be replaced in future elif scan_length > 1: aggregate_data += self.getRows(cluster, tableName, query['columns'] or columns, query['row_key'], query['scan_length'], query['prefix']) return aggregate_data
def putUpload(self, cluster, tableName, row, column, value): client = self.connectCluster(cluster) Mutation = get_thrift_type('Mutation') return client.mutateRow(tableName, smart_str(row), [Mutation(column=smart_str(column), value=value.file.read(value.size))], None)
def deleteColumns(self, cluster, tableName, row, columns): client = self.connectCluster(cluster) Mutation = get_thrift_type('Mutation') mutations = [Mutation(isDelete = True, column=smart_str(column)) for column in columns] return client.mutateRow(tableName, smart_str(row), mutations, None)
def createTable(self, cluster, tableName, *columns): client = self.connectCluster(cluster) client.createTable(tableName, [get_thrift_type('ColumnDescriptor')(name=column) for column in columns]) return "%s successfully created" % tableName
def createTable(self, cluster, tableName, columns): client = self.connectCluster(cluster) client.createTable(tableName, [get_thrift_type('ColumnDescriptor')(**column['properties']) for column in columns], doas=self.user.username) return "%s successfully created" % tableName