def small_large_2cf(client, max_rows):
print('\nsmall_large_2cf')
remove_table(client, 'benchtable')
with timer('createTable'):
client.createTable('benchtable', [ColumnDescriptor('cf0:'), ColumnDescriptor('cf1:')])
with timer('mutateRow:Create-small'):
for x in xrange(max_rows):
client.mutateRow('benchtable', str(x), [Mutation(column='cf0:small', value=random_string(2 ** 10))])
for x in ['cf0:small']:
scanner(client, 'benchtable', x, 10, max_rows)
scanner(client, 'benchtable', x, 1, max_rows)
with timer('mutateRow:Create-large'):
for x in xrange(max_rows):
client.mutateRow('benchtable', str(x), [Mutation(column='cf1:large', value=random_string(2 ** 20))])
for x in ['cf0:small', 'cf1:large', '']:
scanner(client, 'benchtable', x, 10, max_rows)
scanner(client, 'benchtable', x, 1, max_rows)
delete_rows(client, 'benchtable', max_rows)
scanner(client, 'benchtable', '', 1, max_rows)
remove_table(client, 'benchtable')
def few_many_2cf(client, max_rows):
print('\nfew_many_2cf')
remove_table(client, 'benchtable')
with timer('createTable'):
client.createTable('benchtable', [ColumnDescriptor('cf0:'), ColumnDescriptor('cf1:')])
with timer('mutateRow:Create-few'):
for x in xrange(max_rows):
if x % 100 == 0:
client.mutateRow('benchtable', str(x), [Mutation(column='cf0:few', value=random_string(2 ** 10))])
for x in ['cf0:few']:
scanner(client, 'benchtable', x, 100, max_rows)
scanner(client, 'benchtable', x, 10, max_rows)
scanner(client, 'benchtable', x, 1, max_rows)
with timer('mutateRow:Create-many'):
for x in xrange(max_rows):
client.mutateRow('benchtable', str(x), [Mutation(column='cf1:many', value=random_string(2 ** 10))])
for x in ['cf0:few', 'cf1:many', '']:
scanner(client, 'benchtable', x, 100, max_rows)
scanner(client, 'benchtable', x, 10, max_rows)
scanner(client, 'benchtable', x, 1, max_rows)
delete_rows(client, 'benchtable', max_rows)
scanner(client, 'benchtable', '', 1, max_rows)
remove_table(client, 'benchtable')
def simple_valsize(client, max_rows, cfparams, value_size):
print('\nsimple')
remove_table(client, 'benchtable')
with timer('createTable'):
create_table('benchtable', ['cf0'], cfparams)
with timer('mutateRow:Create'):
for x in xrange(max_rows):
client.mutateRow('benchtable', str(x), [Mutation(column='cf0:0', value=random_string(value_size))])
scanner(client, 'benchtable', '', 10, max_rows)
scanner(client, 'benchtable', '', 1, max_rows)
delete_rows(client, 'benchtable', max_rows)
remove_table(client, 'benchtable')
def simple(client, max_rows):
print('\nsimple')
remove_table(client, 'benchtable')
with timer('createTable'):
client.createTable('benchtable', [ColumnDescriptor('cf0:')])
with timer('mutateRow:Create'):
for x in xrange(max_rows):
client.mutateRow('benchtable', str(x), [Mutation(column='cf0:small', value=random_string(2 ** 10))])
scanner(client, 'benchtable', '', 10, max_rows)
scanner(client, 'benchtable', '', 1, max_rows)
delete_rows(client, 'benchtable', max_rows)
scanner(client, 'benchtable', '', 1, max_rows)
remove_table(client, 'benchtable')
def manycols_manycf(client, max_rows, cfparams):
print('\nmanycols_manycf')
num_cols = 100
remove_table(client, 'benchtable')
with timer('createTable'):
create_table('benchtable', ['cf%d' % x for x in xrange(num_cols)], cfparams)
with timer('mutateRow:Create'):
for x in xrange(max_rows):
client.mutateRow('benchtable', str(x), [Mutation(column='cf%d:%d' % (y, y), value=random_string(2 ** 5)) for y in xrange(num_cols)])
for x in ['cf0:0', '']:
scanner(client, 'benchtable', x, 10, max_rows)
scanner(client, 'benchtable', x, 1, max_rows)
delete_rows(client, 'benchtable', max_rows)
scanner(client, 'benchtable', '', 1, max_rows)
remove_table(client, 'benchtable')
def main():
#tags = ' animals architecture art asia australia autumn baby band barcelona beach berlin bike bird birds birthday black blackandwhite blue bw california canada canon car cat chicago china christmas church city clouds color concert dance day de dog england europe fall family fashion festival film florida flower flowers food football france friends fun garden geotagged germany girl graffiti green halloween hawaii holiday house india instagramapp iphone iphoneography island italia italy japan kids la lake landscape light live london love macro me mexico model museum music nature new newyork newyorkcity night nikon nyc ocean old paris park party people photo photography photos portrait raw red river rock san sanfrancisco scotland sea seattle show sky snow spain spring square squareformat street summer sun sunset taiwan texas thailand tokyo travel tree trees trip uk unitedstates urban usa vacation vintage washington water wedding white winter woman yellow zoo '.strip().split()
tags = [
'Pyramids Of Giza', 'Great Wall Of China', 'Terracotta Warriors',
'Statue Of Liberty', 'Edinburgh Castle', 'Stirling Castle',
'Empire State Building', 'Stonehenge', 'Blackpool Tower',
'London Bridge', 'Tower Bridge', 'Buckinghampalace', 'Sphinx',
'Eiffle Tower', 'Arc Du Triomph', 'Louvre', 'Cristo Redentor',
'CN Tower', 'Norte Dame', 'River Nile', 'Mount Rushmore', 'Pentagon',
'White House', 'Lincoln Memorial', 'Grand Canyon',
'Leaning Tower Of Piza', 'Easter Island Heads', 'Niagara Falls',
'Abbey Road', 'Ayers Rock', 'Evangeline Oak', 'Lone Cyprus',
'Golden Gate Bridge', 'Colosseum', 'Taj Mahal', 'Santorini'
]
client = hadoopy_hbase.connect('localhost')
random.shuffle(tags)
flickr = vision_data.Flickr(max_iters=1)
#remove_table(client, 'flickr')
#client.createTable('flickr', [ColumnDescriptor('metadata:'), ColumnDescriptor('images:')])
while True:
for tag in tags:
mutations = []
try:
for url_m, metadata in flickr.image_class_meta_url(tag):
mutations.append(
BatchMutation(row=url_m,
mutations=[
Mutation(column='metadata:%s' % x,
value=y.encode('utf-8'))
for x, y in metadata.items()
]))
except Exception, e:
print(e)
continue
st = time.time()
client.mutateRows('flickr', mutations)
if mutations:
print(
(tag, (time.time() - st) / len(mutations), len(mutations)))
else:
print((tag, 0., len(mutations)))