def main(project_id, instance_id, table_name):
# [START connecting_to_bigtable]
# The client must be created with admin=True because it will create a
# table.
client = bigtable.Client(project=project_id, admin=True)
instance = client.instance(instance_id)
connection = happybase.Connection(instance=instance)
# [END connecting_to_bigtable]
try:
# [START creating_a_table]
tablese = connection.tables()
print('existing tables: ', tablese)
column_family_name = 'tweet'
table = connection.table(table_name)
# [START scanning_all_rows]
print('Scanning all words in table: ', table_name)
column_name = '{fam}:count'.format(fam=column_family_name)
print('column_name ', column_name)
max_val = 0
max_key = None
keys = []
for i in range(5):
max_val = 0
max_key = None
key, val = get_max_val(table, keys)
print(key, val)
keys.append(key)
# [END scanning_all_rows]
# [START deleting_a_table]
#print('Deleting the {} table.'.format(table_name))
#connection.delete_table(table_name)
# [END deleting_a_table]
finally:
connection.close()
def main():
# connect to bigtable
project_id = "gcp_project_id"
instance_id = "bigtable_instance_id"
client = bigtable.Client(project=project_id, admin=True)
instance = client.instance(instance_id)
connection = happybase.Connection(instance=instance)
# create table
table_name = "apps"
column_family_name = "data"
try:
connection.create_table(table_name, {column_family_name: dict()})
except Exception as e:
print(e.message)
# create rows
table = connection.table(table_name)
names = ["John", "Jane", "Richard", "Alan"]
for name in names:
table.put(name, {
"data:name": name,
"data:age": random.randint(20, 40)
})
# get data
print(table.row(names[0]))
for row in table.rows(names[2:]):
print(row)
for row in table.scan(row_prefix="J"):
print(row)
def function_wrapper(*args, **kwargs):
os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = "key.json"
project_id = "csse433-adb-201117"
instance_id = "project433"
client = bigtable.Client(project=project_id, admin=True)
instance = client.instance(instance_id)
connection = happybase.Connection(instance=instance)
func(connection, *args, **kwargs)
connection.close()
return 0
def main(project_id, instance_id):
# [START connecting_to_bigtable]
client = bigtable.Client(project=project_id, admin=True)
instance = client.instance(instance_id)
connection = happybase.Connection(instance=instance)
# [END connecting_to_bigtable]
table_list = connection.tables()
print('Listing tables in Bigtable clister: ', instance_id)
for table in table_list:
print(table)
connection.close()
def main(project_id, instance_id, table_name):
# [START connecting_to_bigtable]
client = bigtable.Client(project=project_id, admin=True)
instance = client.instance(instance_id)
connection = happybase.Connection(instance=instance)
# [END connecting_to_bigtable]
try:
# [START creating_a_table]
print('Creating the {} table.'.format(table_name))
column_family_name = 'cf1'
connection.create_table(
table_name,
{
column_family_name: dict() # Use default options.
})
# [END creating_a_table]
# [START writing_rows]
print('Writing some words to table: ', table_name)
table = connection.table(table_name)
column_name = '{fam}:words'.format(fam=column_family_name)
print('column_name ', column_name)
# Do your staff here to create words....
#
# For the test we simply hardcode a list of words
words = [
'IamWods1',
'SheIsWord2',
'HeIsWord3',
]
for i, value in enumerate(words):
# Note: for large lists of words, this method might have poor performance
# Since rows are stored in sorted order by key,
# sequential keys can result in poor distribution of operations
# across nodes.
#
# For more information about how to design a Bigtable schema for
# the best performance, see the documentation:
#
# https://cloud.google.com/bigtable/docs/schema-design
print('Writing word to the table.',i,value)
row_key = 'words{}'.format(i)
table.put(row_key, {column_name: value})
# [END writing_rows]
finally:
connection.close()
def main(project_id, instance_id, table_name):
# [START connecting_to_bigtable]
client = bigtable.Client(project=project_id, admin=True)
instance = client.instance(instance_id)
connection = happybase.Connection(instance=instance)
# [END connecting_to_bigtable]
print('Listing tables in Bigtable cluster: ', instance_id, ' before deleting')
list_tables(connection)
#[START deleting_a_table]
print('Deleting the {} table.'.format(table_name))
connection.delete_table(table_name)
# [END deleting_a_table]
print('Listing tables in Bigtable cluster: ', instance_id, ' after deleting')
list_tables(connection)
connection.close()
def main(project_id, instance_id, table_name):
# [START connecting_to_bigtable]
# The client must be created with admin=True because it will create a
# table.
client = bigtable.Client(project=project_id, admin=True)
instance = client.instance(instance_id)
connection = happybase.Connection(instance=instance)
# [END connecting_to_bigtable]
try:
# [START creating_a_table]
tablese = connection.tables()
print('existing tables: ', tablese)
column_family_name = 'cf1'
table = connection.table(table_name)
# [START scanning_all_rows]
print('Scanning words in table: ', table_name)
column_name = '{fam}:words'.format(fam=column_family_name)
print('column_name ', column_name)
for key, row in table.scan(limit=100):
#
# Do your staff here with the words in the table...
#
# For simplycity, we just print them to stdout
print('raw format: ', '\t{}: {}'.format(key, row[column_name.encode('utf-8')]))
print('decoded format':, ' word: ', key.decode("utf-8"), 'count: ', int.from_bytes(row[b'cf:count'], byteorder='big'))
# [END scanning_all_rows]
# [START deleting_a_table]
print('Deleting the {} table.'.format(table_name))
connection.delete_table(table_name)
# [END deleting_a_table]
finally:
connection.close()
def main(project_id, instance_id, table_name):
# [START connecting_to_bigtable]
# The client must be created with admin=True because it will create a
# table.
client = bigtable.Client(project=project_id, admin=True)
instance = client.instance(instance_id)
connection = happybase.Connection(instance=instance)
# [END connecting_to_bigtable]
try:
# [START creating_a_table]
tablese = connection.tables()
print('existing tables: ', tablese)
column_family_name = 'cf'
table = connection.table(table_name)
# [START scanning_all_rows]
print('Scanning all words in table: ', table_name)
column_name = '{fam}:count'.format(fam=column_family_name)
print('column_name ', column_name)
i = 0
for key, row in table.scan(limit=40):
i += 1
print(i, ' word: ', key.decode("utf-8"), 'count: ',
int.from_bytes(row[b'cf:count'], byteorder='big'))
# [END scanning_all_rows]
# [START deleting_a_table]
#print('Deleting the {} table.'.format(table_name))
#connection.delete_table(table_name)
# [END deleting_a_table]
finally:
connection.close()
def main(project_id="project-id",
instance_id="instance-id",
table_id="my-table"):
# Creates a Bigtable client
client = bigtable.Client(project=project_id)
# Connect to an existing instance:my-bigtable-instance
instance = client.instance(instance_id)
connection = happybase.Connection(instance=instance)
try:
# Connect to an existing table:my-table
table = connection.table(table_id)
key = 'r1'
row = table.row(key.encode('utf-8'))
value = {k.decode("utf-8"): v.decode("utf-8") for k, v in row.items()}
print('Row key: {}\nData: {}'.format(
key, json.dumps(value, indent=4, sort_keys=True)))
finally:
connection.close()
def main(project_id="project-id",
instance_id="instance-id",
table_id="my-table"):
# Creates a Bigtable client
client = bigtable.Client(project=project_id)
# Connect to an existing instance:my-bigtable-instance
instance = client.instance(instance_id)
connection = happybase.Connection(instance=instance)
try:
# Connect to an existing table:my-table
table = connection.table(table_id)
key = "r1"
row = table.row(key.encode("utf-8"))
column = "cf1:c1".encode("utf-8")
value = row[column].decode("utf-8")
print("Row key: {}\nData: {}".format(key, value))
finally:
connection.close()
def main(project_id, instance_id, table_name):
# [START bigtable_hw_connect_happybase]
# The client must be created with admin=True because it will create a
# table.
client = bigtable.Client(project=project_id, admin=True)
instance = client.instance(instance_id)
connection = happybase.Connection(instance=instance)
# [END bigtable_hw_connect_happybase]
try:
# [START bigtable_hw_create_table_happybase]
print('Creating the {} table.'.format(table_name))
column_family_name = 'cf1'
connection.create_table(
table_name,
{
column_family_name: dict() # Use default options.
})
# [END bigtable_hw_create_table_happybase]
# [START bigtable_hw_write_rows_happybase]
print('Writing some greetings to the table.')
table = connection.table(table_name)
column_name = '{fam}:greeting'.format(fam=column_family_name)
greetings = [
'Hello World!',
'Hello Cloud Bigtable!',
'Hello HappyBase!',
]
for i, value in enumerate(greetings):
# Note: This example uses sequential numeric IDs for simplicity,
# but this can result in poor performance in a production
# application. Since rows are stored in sorted order by key,
# sequential keys can result in poor distribution of operations
# across nodes.
#
# For more information about how to design a Bigtable schema for
# the best performance, see the documentation:
#
# https://cloud.google.com/bigtable/docs/schema-design
row_key = 'greeting{}'.format(i)
table.put(row_key,
{column_name.encode('utf-8'): value.encode('utf-8')})
# [END bigtable_hw_write_rows_happybase]
# [START bigtable_hw_get_by_key_happybase]
print('Getting a single greeting by row key.')
key = 'greeting0'.encode('utf-8')
row = table.row(key)
print('\t{}: {}'.format(key, row[column_name.encode('utf-8')]))
# [END bigtable_hw_get_by_key_happybase]
# [START bigtable_hw_scan_all_happybase]
print('Scanning for all greetings:')
for key, row in table.scan():
print('\t{}: {}'.format(key, row[column_name.encode('utf-8')]))
# [END bigtable_hw_scan_all_happybase]
# [START bigtable_hw_delete_table_happybase]
print('Deleting the {} table.'.format(table_name))
connection.delete_table(table_name)
# [END bigtable_hw_delete_table_happybase]
finally:
connection.close()
print('Running mapreduce job: ')
print(run_job)
os.system(run_job)
print(
'======================================================================================='
)
table_name = "tweet-words-count"
project_id = 'sharon-project-204821'
instance_id = 'sharon-mapreduce-bigtable'
column_family_name = 'cf'
client = bigtable.Client(project=project_id, admin=True)
instance = client.instance(instance_id)
connection = happybase.Connection(instance=instance)
def ByteToHex(byteStr):
return ''.join(["%02X " % ord(x) for x in byteStr]).strip()
try:
table = connection.table(table_name)
print('scanning <', table_name, '>')
word_freq = []
for key, row in table.scan():
byte_string = row[b'cf:count']
val = int("0x" + ''.join([hex(x)[2:] for x in byte_string]),
base=16)
word_freq.append((val, str(key)[2:-1]))