def queue_multi_client_example(db):
descriptions = ["simple queue", "high contention queue"]
for highContention in range(2):
print 'Starting %s test' % descriptions[highContention]
queue = Queue(directory.create_or_open(db, ('tests', 'queue')),
highContention > 0)
queue.clear(db)
pushThreads = [
threading.Thread(target=push_thread, args=(queue, db, i, 100))
for i in range(10)
]
popThreads = [
threading.Thread(target=pop_thread, args=(queue, db, i, 100))
for i in range(10)
]
start = time.time()
for push in pushThreads:
push.start()
for pop in popThreads:
pop.start()
for push in pushThreads:
push.join()
for pop in popThreads:
pop.join()
end = time.time()
print 'Finished %s in %f seconds' % (descriptions[highContention],
end - start)
def test_blob():
import time
db = fdb.open()
location = directory.create_or_open(db, ('tests','blob'))
b = Blob(location)
print "deleting old"
b.delete(db)
print "writing"
b.append(db, 'asdf')
b.append(db, 'jkl;')
b.append(db, 'foo')
b.append(db, 'bar')
print_blob(db, b)
big_data = 1000000
print "writing lots of data"
for i in range(50):
print ".",
b.append(db, '.'*100000)
print ""
print "reading section of large blob..."
t = time.time()
s = len(b.read(db, 1234567, big_data))
assert s == big_data
print "got big section of blob"
def test_blob():
import time
db = fdb.open()
location = directory.create_or_open(db, ('tests', 'blob'))
b = Blob(location)
print "deleting old"
b.delete(db)
print "writing"
b.append(db, 'asdf')
b.append(db, 'jkl;')
b.append(db, 'foo')
b.append(db, 'bar')
print_blob(db, b)
big_data = 1000000
print "writing lots of data"
for i in range(50):
print ".",
b.append(db, '.' * 100000)
print ""
print "reading section of large blob..."
t = time.time()
s = len(b.read(db, 1234567, big_data))
assert s == big_data
print "got big section of blob"
def __init__(self, subspace) :
fdb.api_version(100)
# self._db = fdb.open('/home/gruppe5/fdbconf/fdb.cluster')
self._db = fdb.open()
self._directory = directory.create_or_open(self._db, ('twitter',))
if subspace != None :
self._subspace = self._directory[subspace]
def counter_example_1(db, location):
location = directory.create_or_open( db, ('tests','counter') )
c = Counter(db, location)
for i in range(500):
c.add(db, 1)
print c.get_snapshot(db) #500
def queue_single_client_example(db):
queue = Queue(directory.create_or_open(db, ('tests', 'queue')), False)
queue.clear(db)
for i in range(10):
queue.push(db, i)
for i in range(10):
print queue.pop(db)
def queue_single_client_example(db):
queue = Queue(directory.create_or_open(db, ('tests','queue')), False)
queue.clear(db)
for i in range(10):
queue.push(db, i)
for i in range(10):
print queue.pop(db)
def stringintern_example():
db = fdb.open()
location = directory.create_or_open(db, ('tests', 'stringintern'))
strs = StringIntern(location)
def test_insert(tr):
tr["0"] = strs.intern(tr, "testing 123456789")
tr["1"] = strs.intern(tr, "dog")
tr["2"] = strs.intern(tr, "testing 123456789")
tr["3"] = strs.intern(tr, "cat")
tr["4"] = strs.intern(tr, "cat")
test_insert(db)
tr = db.create_transaction()
for k, v in tr['0':'9']:
print k, '=', strs.lookup(tr, v)
def spatial_example():
db = fdb.open()
index_location = directory.create_or_open(db, ('tests','spatial'))
s = SpatialIndex( index_location )
s.clear(db)
print "point d is at", s.get_location(db, 'd')
s.set_location(db, 'a', (3,2))
s.set_location(db, 'b', (1,4))
s.set_location(db, 'c', (5,3))
s.set_location(db, 'd', (2,3))
s.set_location(db, 'e', (0,0))
print "point d is at", s.get_location(db, 'd')
print "Searching in rectangle (1,1) -> (5,5):"
print s.get_in_rectangle(db, Rect((1,1), (5,5)))
def vector_example(tr):
vector = Vector(directory.create_or_open(tr, ('tests','vector')), 0)
with vector.use_transaction(tr):
vector.clear()
for i in range(0, 5):
vector.push(i)
_print_vector(vector, tr)
print 'Pop last item: %d' % vector.pop()
_print_vector(vector, tr)
vector[1] = 10
vector.resize(11)
_print_vector(vector, tr)
vector.swap(1, 10)
_print_vector(vector, tr)
def queue_test(db):
queue = Queue(directory.create_or_open(db, ('tests', 'queue')), False)
print 'Clear Queue'
queue.clear(db)
print 'Empty? %s' % queue.empty(db)
print 'Push 10, 8, 6'
queue.push(db, 10)
queue.push(db, 8)
queue.push(db, 6)
print 'Empty? %s' % queue.empty(db)
print 'Pop item: %d' % queue.pop(db)
print 'Next item: %d' % queue.peek(db)
print 'Pop item: %d' % queue.pop(db)
print 'Pop item: %d' % queue.pop(db)
print 'Empty? %s' % queue.empty(db)
print 'Push 5'
queue.push(db, 5)
print 'Clear Queue'
queue.clear(db)
print 'Empty? %s' % queue.empty(db)
def queue_multi_client_example(db):
descriptions = ["simple queue", "high contention queue"]
for highContention in range(2):
print 'Starting %s test' % descriptions[highContention]
queue = Queue(directory.create_or_open(db, ('tests','queue')), highContention > 0)
queue.clear(db)
pushThreads = [ threading.Thread(target=push_thread, args=(queue, db, i, 100)) for i in range(10) ]
popThreads = [ threading.Thread(target=pop_thread, args=(queue, db, i, 100)) for i in range(10) ]
start = time.time()
for push in pushThreads: push.start()
for pop in popThreads: pop.start()
for push in pushThreads: push.join()
for pop in popThreads: pop.join()
end = time.time()
print 'Finished %s in %f seconds' % (descriptions[highContention], end - start)
def queue_test(db):
queue = Queue(directory.create_or_open(db, ('tests','queue')), False)
print 'Clear Queue'
queue.clear(db)
print 'Empty? %s' % queue.empty(db)
print 'Push 10, 8, 6'
queue.push(db, 10)
queue.push(db, 8)
queue.push(db, 6)
print 'Empty? %s' % queue.empty(db)
print 'Pop item: %d' % queue.pop(db)
print 'Next item: %d' % queue.peek(db)
print 'Pop item: %d' % queue.pop(db)
print 'Pop item: %d' % queue.pop(db)
print 'Empty? %s' % queue.empty(db)
print 'Push 5'
queue.push(db, 5)
print 'Clear Queue'
queue.clear(db)
print 'Empty? %s' % queue.empty(db)
def __init__(self, subspace) :
fdb.api_version(100)
self._db = fdb.open()
self._directory = directory.create_or_open(self._db, ('twitter',))
if subspace != None :
self._subspace = self._directory[subspace]
from directory import directory
import threading
import json
import shapefile
import fdb
import datetime
import colorsys
import sys
import traceback
import os
import re
# FoundationDB setup
fdb.api_version(100)
db = fdb.open()
twitter_mood = directory.create_or_open(db, ('twitter_mood'))
tweet_by_state = twitter_mood['tweet_state']
tweet_by_time = twitter_mood['tweet_time']
tweet = twitter_mood['tweet']
statistic = twitter_mood['statistic']
# Twitter OAUTH Credentials
APP_KEY = os.environ['APP_KEY']
APP_SECRET = os.environ['APP_SECRET']
OAUTH_TOKEN = os.environ['OAUTH_TOKEN']
OAUTH_TOKEN_SECRET = os.environ['OAUTH_TOKEN_SECRET']
# Parse arguments
save_lag = int(sys.argv[1])
if len(sys.argv) > 2:
track_word = sys.argv[3]
def incrementer_thread(counter, db, n):
for i in range(n):
counter.add(db, 1)
def counter_example_2(db, location):
import threading
c = Counter(db, location)
## 50 incrementer_threads, each doing 10 increments
threads = [
threading.Thread(target=incrementer_thread, args=(c, db, 10))
for i in range(50)]
for thr in threads: thr.start()
for thr in threads: thr.join()
print c.get_snapshot(db) #500
if __name__ == "__main__":
db = fdb.open()
location = directory.create_or_open( db, ('tests','counter') )
del db[location.range()]
print "doing 500 inserts in 50 threads"
counter_example_2(db, location)
print len(db[:]), "counter shards remain in database"
print "doing 500 inserts in one thread"
counter_example_1(db, location)
print len(db[:]), "counter shards remain in database"
def vector_test(tr):
vector = Vector(directory.create_or_open(tr, ('tests','vector')), 0)
with vector.use_transaction(tr):
print 'WITH IMPLICIT TRANSACTIONS:\n'
print 'Clearing any previous values in the vector'
vector.clear()
print '\nMODIFIERS'
# Set + Push
vector[0] = 1
vector[1] = 2
vector.push(3)
_print_vector(vector, tr)
# Swap
vector.swap(0,2)
_print_vector(vector, tr)
# Pop
print 'Popped:', vector.pop()
_print_vector(vector, tr)
# Clear
vector.clear()
print 'Pop empty:', vector.pop()
_print_vector(vector, tr)
vector.push('Foo')
print 'Pop size 1:', vector.pop()
_print_vector(vector, tr)
print '\nCAPACITY OPERATIONS'
# Capacity
print 'Size:', vector.size()
print 'Empty:', vector.empty()
print 'Resizing to length 5'
vector.resize(5)
_print_vector(vector, tr)
print 'Size:', vector.size()
print 'Setting values'
vector[0] = 'The'
vector[1] = 'Quick'
vector[2] = 'Brown'
vector[3] = 'Fox'
vector[4] = 'Jumps'
vector[5] = 'Over'
_print_vector(vector, tr)
print '\nFRONT'
print vector.front()
print '\nBACK'
print vector.back()
print '\nELEMENT ACCESS'
print 'Index 0:', vector[0]
print 'Index 5:', vector[5]
_print_vector(vector, tr)
print 'Size:', vector.size()
print '\nRESIZING'
print 'Resizing to 3'
vector.resize(3)
_print_vector(vector, tr)
print 'Size:', vector.size()
print 'Resizing to 3 again'
vector.resize(3)
_print_vector(vector, tr)
print 'Size:', vector.size()
print 'Resizing to 6'
vector.resize(6)
_print_vector(vector, tr)
print 'Size:', vector.size()
print '\nSPARSE TESTS'
print 'Popping sparse vector'
vector.pop()
_print_vector(vector, tr)
print 'Size:', vector.size()
print 'Resizing to 4'
vector.resize(4)
_print_vector(vector, tr)
print 'Size:', vector.size()
print 'Adding "word" to index 10, resize to 25'
vector[10] = 'word'
vector.resize(25)
_print_vector(vector, tr)
print 'Size:', vector.size()
print 'Popping sparse vector'
vector.pop()
_print_vector(vector, tr)
print 'Size:', vector.size()
print 'Swapping with sparse element'
vector.swap(10, 15)
_print_vector(vector, tr)
print 'Size:', vector.size()
print 'Swapping sparse elements'
vector.swap(12, 13)
_print_vector(vector, tr)
print 'Size:', vector.size()
print '\nWITHOUT IMPLICIT TRANSACTIONS:\n'
print 'Clearing any previous values in the vector'
vector.clear(tr)
print '\nMODIFIERS'
# Set + Push
vector.set(0, 1, tr)
vector.set(1, 2, tr)
vector.push(3, tr)
_print_vector(vector, tr)
# Swap
vector.swap(0,2, tr)
_print_vector(vector, tr)
# Pop
print 'Popped:', vector.pop(tr)
_print_vector(vector, tr)
# Clear
vector.clear(tr)
print 'Pop empty:', vector.pop(tr)
_print_vector(vector, tr)
vector.push('Foo', tr)
print 'Pop size 1:', vector.pop(tr)
_print_vector(vector, tr)
print '\nCAPACITY OPERATIONS'
# Capacity
print 'Size:', vector.size(tr)
print 'Empty:', vector.empty(tr)
print 'Resizing to length 5'
vector.resize(5, tr)
_print_vector(vector, tr)
print 'Size:', vector.size(tr)
print 'Setting values'
vector.set(0, 'The', tr)
vector.set(1, 'Quick', tr)
vector.set(2, 'Brown', tr)
vector.set(3, 'Fox', tr)
vector.set(4, 'Jumps', tr)
vector.set(5, 'Over', tr)
_print_vector(vector, tr)
print '\nFRONT'
print vector.front(tr)
print '\nBACK'
print vector.back(tr)
print '\nELEMENT ACCESS'
print 'Index 0:', vector.get(0, tr)
print 'Index 5:', vector.get(5, tr)
_print_vector(vector, tr)
print 'Size:', vector.size(tr)
print '\nRESIZING'
print 'Resizing to 3'
vector.resize(3, tr)
_print_vector(vector, tr)
print 'Size:', vector.size(tr)
print 'Resizing to 3 again'
vector.resize(3, tr)
_print_vector(vector, tr)
print 'Size:', vector.size(tr)
print 'Resizing to 6'
vector.resize(6, tr)
_print_vector(vector, tr)
print 'Size:', vector.size(tr)
print '\nSPARSE TESTS'
print 'Popping sparse vector'
vector.pop(tr)
_print_vector(vector, tr)
print 'Size:', vector.size(tr)
print 'Resizing to 4'
vector.resize(4, tr)
_print_vector(vector, tr)
print 'Size:', vector.size(tr)
print 'Adding "word" to index 10, resize to 25'
vector.set(10, 'word', tr)
vector.resize(25, tr)
_print_vector(vector, tr)
print 'Size:', vector.size(tr)
print 'Popping sparse vector'
vector.pop(tr)
_print_vector(vector, tr)
print 'Size:', vector.size(tr)
print 'Swapping with sparse element'
vector.swap(10, 15, tr)
_print_vector(vector, tr)
print 'Size:', vector.size(tr)
print 'Swapping sparse elements'
vector.swap(12, 13, tr)
_print_vector(vector, tr)
print 'Size:', vector.size(tr)