class TestCassandraStore(Store, Spawn, unittest.TestCase):
cmd = ['cassandra', '-f']
@classmethod
def setUpClass(cls):
super(TestCassandraStore, cls).setUpClass()
import time
time.sleep(5.0)
def setUp(self):
from shove import Shove
from pycassa.system_manager import SystemManager # @UnresolvedImport @IgnorePep8
system_manager = SystemManager('localhost:9160')
try:
system_manager.create_column_family('Murk', 'shove')
except:
pass
self.store = Shove('cassandra://localhost:9160/Murk/shove')
def tearDown(self):
if self.store._store is not None:
self.store.clear()
self.store.close()
from pycassa.system_manager import SystemManager # @UnresolvedImport @IgnorePep8
system_manager = SystemManager('localhost:9160')
system_manager.drop_column_family('Murk', 'shove')
@classmethod
def tearDownClass(cls):
from fabric.api import local
local('killall java')
def getRSSFeed(self, url, post_data=None):
# create provider storaqe cache
storage = Shove('sqlite:///' + ek.ek(os.path.join, sickbeard.CACHE_DIR, self.provider.name) + '.db')
fc = cache.Cache(storage)
parsed = list(urlparse.urlparse(url))
parsed[2] = re.sub("/{2,}", "/", parsed[2]) # replace two or more / with one
if post_data:
url = url + 'api?' + urllib.urlencode(post_data)
f = fc.fetch(url)
if not f:
logger.log(u"Error loading " + self.providerID + " URL: " + url, logger.ERROR)
return None
elif 'error' in f.feed:
logger.log(u"Newznab ERROR:[%s] CODE:[%s]" % (f.feed['error']['description'], f.feed['error']['code']),
logger.DEBUG)
return None
elif not f.entries:
logger.log(u"No items found on " + self.providerID + " using URL: " + url, logger.WARNING)
return None
storage.close()
return f
class TestCassandraStore(Store, Spawn, unittest.TestCase):
cmd = ['cassandra', '-f']
@classmethod
def setUpClass(cls):
super(TestCassandraStore, cls).setUpClass()
import time
time.sleep(5.0)
def setUp(self):
from shove import Shove
from pycassa.system_manager import SystemManager # @UnresolvedImport @IgnorePep8
system_manager = SystemManager('localhost:9160')
try:
system_manager.create_column_family('Murk', 'shove')
except:
pass
self.store = Shove('cassandra://localhost:9160/Murk/shove')
def tearDown(self):
if self.store._store is not None:
self.store.clear()
self.store.close()
from pycassa.system_manager import SystemManager # @UnresolvedImport @IgnorePep8
system_manager = SystemManager('localhost:9160')
system_manager.drop_column_family('Murk', 'shove')
@classmethod
def tearDownClass(cls):
from fabric.api import local
local('killall java')
class _ShoveWrapper(object):
def __init__(self, loc):
self._loc = loc
self._shove = Shove(self._loc)
def __enter__(self):
return self._shove
def __exit__(self, type, value, traceback):
self._shove.close()
def create_sequence_dbs_for_GAF(gaf, transcripts_file, output_dir):
from Bio import SeqIO
from Bio import Seq
import os
print "Indexing GAF db by transcript id...\n"
gaf_transcript_idx = dict()
for i, g in enumerate(gaf):
for k in gaf[g].keys():
for ctr, t in enumerate(gaf[g][k]):
gaf_transcript_idx[t['transcript_id']] = (ctr, g, k)
fh_transcripts = SeqIO.parse(transcripts_file, 'fasta')
# transcripts_shlv = shelve.open(os.path.join(output_dir, 'GAF_transcript_seqs.fa.shlv'), 'c')
# proteins_shlv = shelve.open(os.path.join(output_dir, 'GAF_protein_seqs.fa.shlv'), 'c')
transcripts_shlv = Shove(
"file://" + os.path.join(output_dir, 'GAF_transcript_seqs.fa.shove'))
protein_seqs_url = "file://" + os.path.join(output_dir,
'GAF_protein_seqs.fa.shove')
proteins_shlv = Shove(protein_seqs_url)
print "Writing transcript and protein shove dbs..."
j = 0
transcripts_to_remove = list()
for transcript in fh_transcripts:
if j % 1000 == 0: print j
j += 1
if transcript.name not in gaf_transcript_idx:
continue
gaf_record = gaf[gaf_transcript_idx[transcript.name][1]][
gaf_transcript_idx[transcript.name][2]][gaf_transcript_idx[
transcript.name][0]]
raw_seq = str(transcript.seq)
transcripts_shlv[transcript.name] = raw_seq
if 'cds_start' not in gaf_record or not gaf_record['cds_start']:
continue
prot_seq = Seq.translate(raw_seq[gaf_record['cds_start'] -
1:gaf_record['cds_stop']])
if prot_seq[-1] == '*':
prot_seq = prot_seq[:-1]
elif prot_seq.find('*') != -1:
# skip small number (n=12) transcripts with incorrect CDS coordinates
transcripts_to_remove.append(transcript.name)
continue
proteins_shlv[transcript.name] = prot_seq
for t in transcripts_to_remove:
del transcripts_shlv[t]
transcripts_shlv.close()
proteins_shlv.close()
return transcripts_to_remove, protein_seqs_url
class TestFTPStore(Store, unittest.TestCase):
initstring = 'ftp://127.0.0.1/'
def setUp(self):
from shove import Shove
self.store = Shove(self.initstring, compress=True)
def tearDown(self):
self.store.clear()
self.store.close()
def primary_file_name(self):
"""The file name for the 'primary' file in the bucket. It is this file from which data is loaded. Other files
within the bucket should be auxiliary to this file. (E.g. they should contain projection information.)
"""
shove = Shove(self._shove_url)
try:
return shove['primary_file_name']
except KeyError:
return None
finally:
shove.close()
class TestFTPStore(Store, unittest.TestCase):
initstring = 'ftp://127.0.0.1/'
def setUp(self):
from shove import Shove
self.store = Shove(self.initstring, compress=True)
def tearDown(self):
self.store.clear()
self.store.close()
def create_sequence_dbs_for_GAF(gaf, transcripts_file, output_dir):
from Bio import SeqIO
from Bio import Seq
import os
print "Indexing GAF db by transcript id...\n"
gaf_transcript_idx = dict()
for i,g in enumerate(gaf):
for k in gaf[g].keys():
for ctr,t in enumerate(gaf[g][k]):
gaf_transcript_idx[t['transcript_id']] = (ctr,g,k)
fh_transcripts = SeqIO.parse(transcripts_file, 'fasta')
# transcripts_shlv = shelve.open(os.path.join(output_dir, 'GAF_transcript_seqs.fa.shlv'), 'c')
# proteins_shlv = shelve.open(os.path.join(output_dir, 'GAF_protein_seqs.fa.shlv'), 'c')
transcripts_shlv = Shove("file://" + os.path.join(output_dir, 'GAF_transcript_seqs.fa.shove'))
protein_seqs_url = "file://" + os.path.join(output_dir, 'GAF_protein_seqs.fa.shove')
proteins_shlv = Shove(protein_seqs_url)
print "Writing transcript and protein shove dbs..."
j = 0
transcripts_to_remove = list()
for transcript in fh_transcripts:
if j % 1000 == 0: print j
j += 1
if transcript.name not in gaf_transcript_idx:
continue
gaf_record = gaf[gaf_transcript_idx[transcript.name][1]][gaf_transcript_idx[transcript.name][2]][gaf_transcript_idx[transcript.name][0]]
raw_seq = str(transcript.seq)
transcripts_shlv[transcript.name] = raw_seq
if 'cds_start' not in gaf_record or not gaf_record['cds_start']: continue
prot_seq = Seq.translate(raw_seq[gaf_record['cds_start']-1:gaf_record['cds_stop']])
if prot_seq[-1] == '*':
prot_seq = prot_seq[:-1]
elif prot_seq.find('*') != -1:
# skip small number (n=12) transcripts with incorrect CDS coordinates
transcripts_to_remove.append(transcript.name)
continue
proteins_shlv[transcript.name] = prot_seq
for t in transcripts_to_remove:
del transcripts_shlv[t]
transcripts_shlv.close()
proteins_shlv.close()
return transcripts_to_remove,protein_seqs_url
class TestHDF5Store(Store, unittest.TestCase):
initstring = 'hdf5://test.hdf5/test'
def setUp(self):
from shove import Shove
self.store = Shove()
def tearDown(self):
import os
self.store.close()
try:
os.remove('test.hdf5')
except OSError:
pass
class TestHDF5Store(Store, unittest.TestCase):
initstring = 'hdf5://test.hdf5/test'
def setUp(self):
from shove import Shove
self.store = Shove()
def tearDown(self):
import os
self.store.close()
try:
os.remove('test.hdf5')
except OSError:
pass
def save_data(self, data):
db = Shove(self.conf['data_file'])
modified = False
for item in data:
try:
db[item['guid']]
except KeyError:
db[item['guid']] = item
modified = True
else:
if db[item['guid']] != item:
db[item['guid']] = item
modified = True
db.close()
return modified
class ShoveCache(Cache):
"""Expects a url in the form that shove requires.
Maintains a cache of keys to speed performance."""
def __init__(self, url_db, url_cache):
self.db = Shove(url_db, url_cache, optimize=False, max_entries=2000)
def retrieve_from_cache(self, key):
try:
val = self.db[key]
return val
except KeyError:
return None
def store_into_cache(self, key, value):
self.db[key] = value
def close_cache(self):
self.db.close()
class ShoveCache(Cache):
"""Expects a url in the form that shove requires.
Maintains a cache of keys to speed performance."""
def __init__(self, url_db, url_cache):
self.db = Shove(url_db, url_cache, optimize=False, max_entries=2000)
def retrieve_from_cache(self, key):
try:
val = self.db[key]
return val
except KeyError:
return None
def store_into_cache(self, key, value):
self.db[key] = value
def close_cache(self):
self.db.close()
class TestZodbStore(unittest.TestCase):
init = "zodb://test.db"
def setUp(self):
from shove import Shove
self.store = Shove(self.init, compress=True)
def tearDown(self):
self.store.close()
import os
os.remove("test.db")
os.remove("test.db.index")
os.remove("test.db.tmp")
os.remove("test.db.lock")
def test__getitem__(self):
self.store["max"] = 3
self.assertEqual(self.store["max"], 3)
def test__setitem__(self):
self.store["max"] = 3
self.assertEqual(self.store["max"], 3)
def test__delitem__(self):
self.store["max"] = 3
del self.store["max"]
self.assertEqual("max" in self.store, False)
def test_get(self):
self.store["max"] = 3
self.assertEqual(self.store.get("min"), None)
def test__cmp__(self):
from shove import Shove
tstore = Shove()
self.store["max"] = 3
tstore["max"] = 3
self.assertEqual(self.store, tstore)
def test__len__(self):
self.store["max"] = 3
self.store["min"] = 6
self.assertEqual(len(self.store), 2)
def test_close(self):
self.store.close()
self.assertEqual(self.store, None)
def test_clear(self):
self.store["max"] = 3
self.store["min"] = 6
self.store["pow"] = 7
self.store.clear()
self.assertEqual(len(self.store), 0)
def test_items(self):
self.store["max"] = 3
self.store["min"] = 6
self.store["pow"] = 7
slist = list(self.store.items())
self.assertEqual(("min", 6) in slist, True)
def test_iteritems(self):
self.store["max"] = 3
self.store["min"] = 6
self.store["pow"] = 7
slist = list(self.store.iteritems())
self.assertEqual(("min", 6) in slist, True)
def test_iterkeys(self):
self.store["max"] = 3
self.store["min"] = 6
self.store["pow"] = 7
slist = list(self.store.iterkeys())
self.assertEqual("min" in slist, True)
def test_itervalues(self):
self.store["max"] = 3
self.store["min"] = 6
self.store["pow"] = 7
slist = list(self.store.itervalues())
self.assertEqual(6 in slist, True)
def test_pop(self):
self.store["max"] = 3
self.store["min"] = 6
item = self.store.pop("min")
self.assertEqual(item, 6)
def test_popitem(self):
self.store["max"] = 3
self.store["min"] = 6
self.store["pow"] = 7
item = self.store.popitem()
self.assertEqual(len(item) + len(self.store), 4)
def test_setdefault(self):
self.store["max"] = 3
self.store["min"] = 6
self.store["powl"] = 7
self.store.setdefault("pow", 8)
self.assertEqual(self.store["pow"], 8)
def test_update(self):
from shove import Shove
tstore = Shove()
tstore["max"] = 3
tstore["min"] = 6
tstore["pow"] = 7
self.store["max"] = 2
self.store["min"] = 3
self.store["pow"] = 7
self.store.update(tstore)
self.assertEqual(self.store["min"], 6)
def test_values(self):
self.store["max"] = 3
self.store["min"] = 6
self.store["pow"] = 7
slist = self.store.values()
self.assertEqual(6 in slist, True)
def test_keys(self):
self.store["max"] = 3
self.store["min"] = 6
self.store["pow"] = 7
slist = self.store.keys()
self.assertEqual("min" in slist, True)
class DataManager():
def __init__(self, filename, connection_details=VERTICA_CONNECTION):
self.filename = filename
self.store = Shove('file://'+self.filename, 'memory://', optimize=False)
self.connection_details = connection_details
@staticmethod
def get_attribute_name(data_set, attribute):
return '_' + data_set + '_' + attribute
def get_data_set_attribute(self, data_set, key):
attrib_name = self.get_attribute_name(data_set, key)
if self.has_attribute(data_set, key):
return self.store[attrib_name]
else:
return None
self.store.close()
gc.collect()
self.store = Shove('file://'+self.filename, 'memory://', optimize=False)
def set_data_set_attribute(self, data_set, key, value):
attrib_name = self.get_attribute_name(data_set, key)
self.store.update({attrib_name: value})
self.store.sync()
self.store.close()
gc.collect()
self.store = Shove('file://'+self.filename, 'memory://', optimize=False)
def has_attribute(self, data_set, key):
attrib_name = self.get_attribute_name(data_set, key)
if attrib_name in self.store.keys():
return True
else:
return False
def remove_attribute(self, data_set, key):
attrib_name = self.get_attribute_name(data_set, key)
if attrib_name in self.store:
del self.store[attrib_name]
self.store.sync()
def get_data_set_list(self):
data_sets = [key.split('_')[1] for key in self.store.keys() if '_query' in key]
return data_sets
def fetch_from_vertica_to_df(self, data_set, query):
data_set_query = self.get_data_set_attribute(data_set, 'query')
if data_set_query != query:
connection = connect(self.connection_details)
cursor = connection.cursor()
print 'Executing ', data_set, 'Query...'
print query
columns = get_column_names_from_sql_query(query)
cursor.execute(query)
data = []
while True:
rows = cursor.fetchmany(10000)
data.extend([[str(ele) for ele in row] for row in rows])
if len(rows) <= 1:
break
df = MetadataDataFrame(data=data, columns=columns, meta_info={'query': query, 'built_features': [], 'aggregate_values': {},
'columns': columns})
cursor.close()
if len(df) == 0:
raise(ValueError('SQL result in empty fetch!!'))
else:
self.set_data_set_attribute(data_set, 'data', df)
self.set_data_set_attribute(data_set, 'query', query)
self.set_data_set_attribute(data_set, 'columns', columns)
self.set_data_set_attribute(data_set, 'built_features', [])
def primary_file_name(self, name):
shove = Shove(self._shove_url)
try:
shove['primary_file_name'] = name
finally:
shove.close()
class EntryManager(object):
def __init__(self, location, **kwargs):
self.store = Shove(location, **kwargs)
def create_entry(self, entry):
"""
Create an entry in the entry manager::
>>> entry = {}
>>> entry['content'] = {'content': "<p>This is an entry.</p>"}
>>> entry['author'] = {'name': "Roberto De Almeida",
... 'email': "*****@*****.**"}
>>> em.create_entry(entry)
If empty, 'id' and 'updated' will be automatically set.
"""
# Required elements.
entry.setdefault('id', self._next_id())
entry.setdefault('updated', time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()))
# Store entry.
self.store[str(entry["id"])] = entry
return entry
def get_entry(self, key):
"""
Retrieve an entry by its id::
>>> entry = em.get_entry("1")
Will return a dict describing the entry.
"""
entry = self.store[str(key)]
return entry
def get_entries(self, size=None, offset=0):
"""
Retrieve last n entries, sorted by the 'updated' attribute and in
descending order::
>>> entries = em.get_entries(10)
If you omit the number, all entries will be retrieved.
You can also specify an offset::
>>> entries = em.get_entries(10, 5)
This will skip the first 5 entries.
"""
entries = self.store.values()
# Sort by time.
entries.sort(key=operator.itemgetter('updated'), reverse=True)
if size is not None: size += offset
entries = entries[offset:size]
return entries
def delete_entry(self, key):
"""
Delete an entry identified by its key::
>>> em.delete_entry("1")
"""
del self.store[str(key)]
def update_entry(self, new_entry):
"""
Update an entry.
>>> entry = {'title': "This is an entry"}
>>> entry = em.create_entry(entry)
>>> print entry['id']
0
>>> entry['title'] = "A different title"
>>> em.update_entry(entry)
"""
# Retrieve old entry and update id.
entry = self.store[str(new_entry["id"])]
entry.update(new_entry)
# Update 'updated'.
entry['updated'] = time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())
# Store it back.
self.store[str(entry["id"])] = entry
return entry
def search(self, filters, flags=0, size=None, offset=0):
"""
Search all entries.
A simple, case-sensitive search for the word "Python" in the title::
>>> entries = em.search({"title": "Python"})
For case-insensitive searching in the content::
>>> entries = em.search({"content": {"content": "Python"}}, re.IGNORECASE)
"""
entries = self.get_entries()
entries = [entry for entry in entries if filter_(entry, filters, flags)]
if size is not None: size += offset
entries = entries[offset:size]
return entries
def _next_id(self):
# List keys.
keys = self.store.keys()
keys = [key for key in keys if re.match(r"^\d+$", key)]
keys.sort(key=int)
# Get a new key.
key = keys and str(int(keys[-1]) + 1) or "0"
return key
def close(self):
self.store.close()
class TestLevelDBStore(unittest2.TestCase):
def setUp(self):
from shove import Shove
self.store = Shove('leveldb://test', compress=True)
def tearDown(self):
import shutil
shutil.rmtree('test')
def test__getitem__(self):
self.store['max'] = 3
self.assertEqual(self.store['max'], 3)
def test__setitem__(self):
self.store['max'] = 3
self.assertEqual(self.store['max'], 3)
def test__delitem__(self):
self.store['max'] = 3
del self.store['max']
self.assertEqual('max' in self.store, False)
def test_get(self):
self.store['max'] = 3
self.assertEqual(self.store.get('min'), None)
def test__cmp__(self):
from shove import Shove
tstore = Shove()
self.store['max'] = 3
tstore['max'] = 3
self.assertEqual(self.store, tstore)
def test__len__(self):
self.store['max'] = 3
self.store['min'] = 6
self.assertEqual(len(self.store), 2)
def test_close(self):
self.store.close()
self.assertEqual(self.store, None)
def test_clear(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
self.store.clear()
self.assertEqual(len(self.store), 0)
def test_items(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = list(self.store.items())
self.assertEqual(('min', 6) in slist, True)
def test_iteritems(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = list(self.store.iteritems())
self.assertEqual(('min', 6) in slist, True)
def test_iterkeys(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = list(self.store.iterkeys())
self.assertEqual('min' in slist, True)
def test_itervalues(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = list(self.store.itervalues())
self.assertEqual(6 in slist, True)
def test_pop(self):
self.store['max'] = 3
self.store['min'] = 6
item = self.store.pop('min')
self.assertEqual(item, 6)
def test_popitem(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
item = self.store.popitem()
self.assertEqual(len(item) + len(self.store), 4)
def test_setdefault(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
self.store.setdefault('bow', 8)
self.assertEqual(self.store['bow'], 8)
def test_update(self):
from shove import Shove
tstore = Shove()
tstore['max'] = 3
tstore['min'] = 6
tstore['pow'] = 7
self.store['max'] = 2
self.store['min'] = 3
self.store['pow'] = 7
self.store.update(tstore)
self.assertEqual(self.store['min'], 6)
def test_values(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = self.store.values()
self.assertEqual(6 in slist, True)
def test_keys(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = self.store.keys()
self.assertEqual('min' in slist, True)
vocabulary_c_dict[category] = words_c
if __name__ == '__main__':
start_time = time.time()
print("Processing articles dataset")
process_articles()
log.info("Syncing articles dict")
articles_dict.sync()
log.info("Syncing title dict")
titles_dict.sync()
log.info("Syncing words dict")
words_dict.sync()
print("Calculating cf_w")
calculate_cf_w()
log.info("Closing articles dict")
articles_dict.close()
log.info("Closing titles dict")
titles_dict.close()
log.info("Closing words dict")
words_dict.close()
log.info("Syncing R_w dict")
R_w_dict.sync()
log.info("Syncing vocabulary_c dict")
vocabulary_c_dict.sync()
log.info("Closing R_w dict")
R_w_dict.close()
log.info("Closing vocabulary_c dict")
vocabulary_c_dict.close()
print("Execution time took %.3f seconds" % (time.time() - start_time))
class TestFtpStore(unittest.TestCase):
ftpstring = "put ftp string here"
def setUp(self):
self.store = Shove(self.ftpstring, compress=True)
def tearDown(self):
self.store.clear()
self.store.close()
def test__getitem__(self):
self.store["max"] = 3
self.store.sync()
self.assertEqual(self.store["max"], 3)
def test__setitem__(self):
self.store["max"] = 3
self.store.sync()
self.assertEqual(self.store["max"], 3)
def test__delitem__(self):
self.store["max"] = 3
del self.store["max"]
self.assertEqual("max" in self.store, False)
def test_get(self):
self.store["max"] = 3
self.store.sync()
self.assertEqual(self.store.get("min"), None)
def test__cmp__(self):
tstore = Shove()
self.store["max"] = 3
tstore["max"] = 3
self.store.sync()
tstore.sync()
self.assertEqual(self.store, tstore)
def test__len__(self):
self.store["max"] = 3
self.store["min"] = 6
self.store.sync()
self.assertEqual(len(self.store), 2)
def test_clear(self):
self.store["max"] = 3
self.store["min"] = 6
self.store["pow"] = 7
self.store.sync()
self.store.clear()
self.assertEqual(len(self.store), 0)
def test_items(self):
self.store["max"] = 3
self.store["min"] = 6
self.store["pow"] = 7
self.store.sync()
slist = list(self.store.items())
self.assertEqual(("min", 6) in slist, True)
def test_iteritems(self):
self.store["max"] = 3
self.store["min"] = 6
self.store["pow"] = 7
self.store.sync()
slist = list(self.store.iteritems())
self.assertEqual(("min", 6) in slist, True)
def test_iterkeys(self):
self.store["max"] = 3
self.store["min"] = 6
self.store["pow"] = 7
self.store.sync()
slist = list(self.store.iterkeys())
self.assertEqual("min" in slist, True)
def test_itervalues(self):
self.store["max"] = 3
self.store["min"] = 6
self.store["pow"] = 7
self.store.sync()
slist = list(self.store.itervalues())
self.assertEqual(6 in slist, True)
def test_pop(self):
self.store["max"] = 3
self.store["min"] = 6
self.store.sync()
item = self.store.pop("min")
self.assertEqual(item, 6)
def test_popitem(self):
self.store["max"] = 3
self.store["min"] = 6
self.store["pow"] = 7
self.store.sync()
item = self.store.popitem()
self.store.sync()
self.assertEqual(len(item) + len(self.store), 4)
def test_setdefault(self):
self.store["max"] = 3
self.store["min"] = 6
self.store["powl"] = 7
self.store.setdefault("pow", 8)
self.store.sync()
self.assertEqual(self.store["pow"], 8)
def test_update(self):
tstore = Shove()
tstore["max"] = 3
tstore["min"] = 6
tstore["pow"] = 7
self.store["max"] = 2
self.store["min"] = 3
self.store["pow"] = 7
self.store.sync()
self.store.update(tstore)
self.store.sync()
self.assertEqual(self.store["min"], 6)
def test_values(self):
self.store["max"] = 3
self.store["min"] = 6
self.store["pow"] = 7
self.store.sync()
slist = self.store.values()
self.assertEqual(6 in slist, True)
def test_keys(self):
self.store["max"] = 3
self.store["min"] = 6
self.store["pow"] = 7
self.store.sync()
slist = self.store.keys()
self.assertEqual("min" in slist, True)
factor_data[f] = data[column_name].reindex(
indx, method='ffill') * column_scale
if transformation == "rdiff":
factor_data[
f] = factor_data[f] / factor_data[f].shift(1) - 1
return factor_data
def get_adjusted_close(self, ticker, start_date, end_date):
data = self.fetch_data(ticker, start_date, end_date, 11)
return data["Adj. Close"]
if __name__ == '__main__':
#from cache import ShelveCache
#cache = ShelveCache(filename="D:\\cache.db", writeback=True)
from shove import Shove
cache = Shove("zodb:///home/gouthaman/stock.zodb")
data = QuandlData(cache=cache)
import datetime
dd = data.fetch_data("WIKI/WFM",
datetime.date(2014, 1, 1),
datetime.date(2014, 7, 31),
column=11)
dd = data.fetch_data("WIKI/WFM",
datetime.date(2014, 1, 1),
datetime.date(2014, 7, 31),
column=11)
cache.close()
class TestFileStore(unittest.TestCase):
def setUp(self):
from shove import Shove
self.store = Shove('file://test', compress=True)
def tearDown(self):
import os
self.store.close()
for x in os.listdir('test'):
os.remove(os.path.join('test', x))
os.rmdir('test')
def test__getitem__(self):
self.store['max'] = 3
self.store.sync()
self.assertEqual(self.store['max'], 3)
def test__setitem__(self):
self.store['max'] = 3
self.store.sync()
self.assertEqual(self.store['max'], 3)
def test__delitem__(self):
self.store['max'] = 3
del self.store['max']
self.assertEqual('max' in self.store, False)
def test_get(self):
self.store['max'] = 3
self.store.sync()
self.assertEqual(self.store.get('min'), None)
def test__cmp__(self):
from shove import Shove
tstore = Shove()
self.store['max'] = 3
tstore['max'] = 3
self.store.sync()
tstore.sync()
self.assertEqual(self.store, tstore)
def test__len__(self):
self.store['max'] = 3
self.store['min'] = 6
self.assertEqual(len(self.store), 2)
def test_close(self):
self.store.close()
self.assertEqual(self.store, None)
def test_clear(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
self.store.clear()
self.assertEqual(len(self.store), 0)
def test_items(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = list(self.store.items())
self.assertEqual(('min', 6) in slist, True)
def test_iteritems(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = list(self.store.iteritems())
self.assertEqual(('min', 6) in slist, True)
def test_iterkeys(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = list(self.store.iterkeys())
self.assertEqual('min' in slist, True)
def test_itervalues(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = list(self.store.itervalues())
self.assertEqual(6 in slist, True)
def test_pop(self):
self.store['max'] = 3
self.store['min'] = 6
item = self.store.pop('min')
self.assertEqual(item, 6)
def test_popitem(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
item = self.store.popitem()
self.assertEqual(len(item) + len(self.store), 4)
def test_setdefault(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['powl'] = 7
self.store.setdefault('pow', 8)
self.assertEqual(self.store['pow'], 8)
def test_update(self):
from shove import Shove
tstore = Shove()
tstore['max'] = 3
tstore['min'] = 6
tstore['pow'] = 7
self.store['max'] = 2
self.store['min'] = 3
self.store['pow'] = 7
self.store.update(tstore)
self.assertEqual(self.store['min'], 6)
def test_values(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = self.store.values()
self.assertEqual(6 in slist, True)
def test_keys(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = self.store.keys()
self.assertEqual('min' in slist, True)
class TestSvnStore(unittest.TestCase):
svnstring = 'SVN test string here'
def setUp(self):
from shove import Shove
self.store = Shove(self.svnstring, compress=True)
def tearDown(self):
self.store.clear()
self.store.close()
def test__getitem__(self):
self.store['max'] = 3
self.store.sync()
self.assertEqual(self.store['max'], 3)
def test__setitem__(self):
self.store['max'] = 3
self.store.sync()
self.assertEqual(self.store['max'], 3)
def test__delitem__(self):
self.store['max'] = 3
del self.store['max']
self.assertEqual('max' in self.store, False)
def test_get(self):
self.store['max'] = 3
self.store.sync()
self.assertEqual(self.store.get('min'), None)
def test__cmp__(self):
from shove import Shove
tstore = Shove()
self.store['max'] = 3
tstore['max'] = 3
self.store.sync()
tstore.sync()
self.assertEqual(self.store, tstore)
def test__len__(self):
self.store['max'] = 3
self.store['min'] = 6
self.store.sync()
self.assertEqual(len(self.store), 2)
def test_clear(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
self.store.sync()
self.store.clear()
self.assertEqual(len(self.store), 0)
def test_items(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
self.store.sync()
slist = list(self.store.items())
self.assertEqual(('min', 6) in slist, True)
def test_iteritems(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
self.store.sync()
slist = list(self.store.iteritems())
self.assertEqual(('min', 6) in slist, True)
def test_iterkeys(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
self.store.sync()
slist = list(self.store.iterkeys())
self.assertEqual('min' in slist, True)
def test_itervalues(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
self.store.sync()
slist = list(self.store.itervalues())
self.assertEqual(6 in slist, True)
def test_pop(self):
self.store['max'] = 3
self.store['min'] = 6
self.store.sync()
item = self.store.pop('min')
self.assertEqual(item, 6)
def test_popitem(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
self.store.sync()
item = self.store.popitem()
self.store.sync()
self.assertEqual(len(item) + len(self.store), 4)
def test_setdefault(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['powl'] = 7
self.store.setdefault('pow', 8)
self.store.sync()
self.assertEqual(self.store['pow'], 8)
def test_update(self):
from shove import Shove
tstore = Shove()
tstore['max'] = 3
tstore['min'] = 6
tstore['pow'] = 7
self.store['max'] = 2
self.store['min'] = 3
self.store['pow'] = 7
self.store.sync()
self.store.update(tstore)
self.store.sync()
self.assertEqual(self.store['min'], 6)
def test_values(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
self.store.sync()
slist = self.store.values()
self.assertEqual(6 in slist, True)
def test_keys(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
self.store.sync()
slist = self.store.keys()
self.assertEqual('min' in slist, True)
class TestHDF5Store(unittest2.TestCase):
def setUp(self):
from shove import Shove
self.store = Shove('hdf5://test.hdf5/test')
def tearDown(self):
import os
self.store.close()
try:
os.remove('test.hdf5')
except OSError:
pass
def test__getitem__(self):
self.store['max'] = 3
self.assertEqual(self.store['max'], 3)
def test__setitem__(self):
self.store['max'] = 3
self.assertEqual(self.store['max'], 3)
def test__delitem__(self):
self.store['max'] = 3
del self.store['max']
self.assertEqual('max' in self.store, False)
def test_get(self):
self.store['max'] = 3
self.assertEqual(self.store.get('min'), None)
def test__cmp__(self):
from shove import Shove
tstore = Shove()
self.store['max'] = 3
tstore['max'] = 3
self.assertEqual(self.store, tstore)
def test__len__(self):
self.store['max'] = 3
self.store['min'] = 6
self.assertEqual(len(self.store), 2)
def test_close(self):
self.store.close()
self.assertEqual(self.store, None)
def test_clear(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
self.store.clear()
self.assertEqual(len(self.store), 0)
def test_items(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = list(self.store.items())
self.assertEqual(('min', 6) in slist, True)
def test_iteritems(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = list(self.store.iteritems())
self.assertEqual(('min', 6) in slist, True)
def test_iterkeys(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = list(self.store.iterkeys())
self.assertEqual('min' in slist, True)
def test_itervalues(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = list(self.store.itervalues())
self.assertEqual(6 in slist, True)
def test_pop(self):
self.store['max'] = 3
self.store['min'] = 6
item = self.store.pop('min')
self.assertEqual(item, 6)
def test_popitem(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
item = self.store.popitem()
self.assertEqual(len(item) + len(self.store), 4)
def test_setdefault(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
self.store.setdefault('bow', 8)
self.assertEqual(self.store['bow'], 8)
def test_update(self):
from shove import Shove
tstore = Shove()
tstore['max'] = 3
tstore['min'] = 6
tstore['pow'] = 7
self.store['max'] = 2
self.store['min'] = 3
self.store['pow'] = 7
self.store.update(tstore)
self.assertEqual(self.store['min'], 6)
def test_values(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = self.store.values()
self.assertEqual(6 in slist, True)
def test_keys(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = self.store.keys()
self.assertEqual('min' in slist, True)
class TestFileStore(unittest.TestCase):
def setUp(self):
self.store = Shove('file://test', compress=True)
def tearDown(self):
self.store.close()
for x in os.listdir('test'):
os.remove(os.path.join('test', x))
os.rmdir('test')
def test__getitem__(self):
self.store['max'] = 3
self.store.sync()
self.assertEqual(self.store['max'], 3)
def test__setitem__(self):
self.store['max'] = 3
self.store.sync()
self.assertEqual(self.store['max'], 3)
def test__delitem__(self):
self.store['max'] = 3
del self.store['max']
self.assertEqual('max' in self.store, False)
def test_get(self):
self.store['max'] = 3
self.store.sync()
self.assertEqual(self.store.get('min'), None)
def test__cmp__(self):
tstore = Shove()
self.store['max'] = 3
tstore['max'] = 3
self.store.sync()
tstore.sync()
self.assertEqual(self.store, tstore)
def test__len__(self):
self.store['max'] = 3
self.store['min'] = 6
self.assertEqual(len(self.store), 2)
def test_close(self):
self.store.close()
self.assertEqual(self.store, None)
def test_clear(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
self.store.clear()
self.assertEqual(len(self.store), 0)
def test_items(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = list(self.store.items())
self.assertEqual(('min', 6) in slist, True)
def test_iteritems(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = list(self.store.iteritems())
self.assertEqual(('min', 6) in slist, True)
def test_iterkeys(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = list(self.store.iterkeys())
self.assertEqual('min' in slist, True)
def test_itervalues(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = list(self.store.itervalues())
self.assertEqual(6 in slist, True)
def test_pop(self):
self.store['max'] = 3
self.store['min'] = 6
item = self.store.pop('min')
self.assertEqual(item, 6)
def test_popitem(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
item = self.store.popitem()
self.assertEqual(len(item) + len(self.store), 4)
def test_setdefault(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['powl'] = 7
self.store.setdefault('pow', 8)
self.assertEqual(self.store['pow'], 8)
def test_update(self):
tstore = Shove()
tstore['max'] = 3
tstore['min'] = 6
tstore['pow'] = 7
self.store['max'] = 2
self.store['min'] = 3
self.store['pow'] = 7
self.store.update(tstore)
self.assertEqual(self.store['min'], 6)
def test_values(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = self.store.values()
self.assertEqual(6 in slist, True)
def test_keys(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = self.store.keys()
self.assertEqual('min' in slist, True)
class Store(object):
def setUp(self):
from shove import Shove
self.store = Shove(
self.initstring, sync=0
)
def tearDown(self):
self.store.close()
def test__getitem__(self):
self.store['max'] = 3
self.store.sync()
self.assertEqual(self.store['max'], 3)
def test__setitem__(self):
self.store['max'] = 3
self.store['d'] = {'A': 1}, {'A': 1}
self.store['d'] = {'AA': 1}, {'A': 1}
self.store['d'] = {'AA': 1}, {'AA': 1}
self.store.sync()
self.assertEqual(self.store['max'], 3)
def test__delitem__(self):
self.store['max'] = 3
self.store.sync()
del self.store['max']
self.store.sync()
self.assertEqual('max' in self.store, False)
def test_get(self):
self.store['max'] = 3
self.store.sync()
self.assertEqual(self.store.get('min'), None)
def test__cmp__(self):
from shove import Shove
tstore = Shove()
self.store['max'] = 3
self.store.sync()
tstore['max'] = 3
self.assertEqual(self.store, tstore)
def test__len__(self):
self.store['max'] = 3
self.store['min'] = 6
self.store.sync()
self.assertEqual(len(self.store), 2)
def test_items(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
self.store.sync()
slist = list(items(self.store))
self.assertEqual(('min', 6) in slist, True)
def test_keys(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
self.store.sync()
slist = list(keys(self.store))
self.assertEqual('min' in slist, True)
def test_values(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
self.store.sync()
slist = list(values(self.store))
self.assertEqual(6 in slist, True)
def test_pop(self):
self.store['max'] = 3
self.store['min'] = 6
self.store.sync()
item = self.store.pop('min')
self.store.sync()
self.assertEqual(item, 6)
def test_setdefault(self):
self.store['max'] = 3
self.store['min'] = 6
self.store.sync()
self.assertEqual(self.store.setdefault('pow', 8), 8)
self.store.sync()
self.assertEqual(self.store['pow'], 8)
def test_update(self):
from shove import Shove
tstore = Shove()
tstore['max'] = 3
tstore['min'] = 6
tstore['pow'] = 7
self.store['max'] = 2
self.store['min'] = 3
self.store['pow'] = 7
self.store.update(tstore)
self.store.sync()
self.assertEqual(self.store['min'], 6)
def test_clear(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
self.store.sync()
self.store.clear()
self.store.sync()
self.assertEqual(len(self.store), 0)
def test_popitem(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
self.store.sync()
item = self.store.popitem()
self.store.sync()
self.assertEqual(len(item) + len(self.store), 4)
def test_close(self):
self.store.close()
self.assertEqual(self.store._store, None)
self.assertEqual(self.store._buffer, None)
self.assertEqual(self.store._cache, None)
factor_data = {}
indx = pd.DatetimeIndex(start=start_date, end=end_date, freq='M')
for qkey,factors in quandl_keys.items():
data = self.fetch_data(qkey,start_date, end_date)
for f in factors:
column_name , column_scale, transformation = self.data_items[qkey][0][f]
factor_data[f] = data[column_name].reindex(indx,
method='ffill')*column_scale
if transformation=="rdiff":
factor_data[f] = factor_data[f]/factor_data[f].shift(1)-1
return factor_data
def get_adjusted_close(self, ticker, start_date, end_date):
data = self.fetch_data(ticker, start_date, end_date,11)
return data["Adj. Close"]
if __name__ == '__main__':
#from cache import ShelveCache
#cache = ShelveCache(filename="D:\\cache.db", writeback=True)
from shove import Shove
cache = Shove("zodb:///home/gouthaman/stock.zodb")
data = QuandlData(cache=cache)
import datetime
dd = data.fetch_data("WIKI/WFM", datetime.date(2014,1,1) , datetime.date(2014,7,31),column=11)
dd = data.fetch_data("WIKI/WFM", datetime.date(2014,1,1) , datetime.date(2014,7,31),column=11)
cache.close()
class TestZodbStore(unittest.TestCase):
init = 'zodb://test.db'
def setUp(self):
from shove import Shove
self.store = Shove(self.init, compress=True)
def tearDown(self):
self.store.close()
import os
os.remove('test.db')
os.remove('test.db.index')
os.remove('test.db.tmp')
os.remove('test.db.lock')
def test__getitem__(self):
self.store['max'] = 3
self.assertEqual(self.store['max'], 3)
def test__setitem__(self):
self.store['max'] = 3
self.assertEqual(self.store['max'], 3)
def test__delitem__(self):
self.store['max'] = 3
del self.store['max']
self.assertEqual('max' in self.store, False)
def test_get(self):
self.store['max'] = 3
self.assertEqual(self.store.get('min'), None)
def test__cmp__(self):
from shove import Shove
tstore = Shove()
self.store['max'] = 3
tstore['max'] = 3
self.assertEqual(self.store, tstore)
def test__len__(self):
self.store['max'] = 3
self.store['min'] = 6
self.assertEqual(len(self.store), 2)
def test_close(self):
self.store.close()
self.assertEqual(self.store, None)
def test_clear(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
self.store.clear()
self.assertEqual(len(self.store), 0)
def test_items(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = list(self.store.items())
self.assertEqual(('min', 6) in slist, True)
def test_iteritems(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = list(self.store.iteritems())
self.assertEqual(('min', 6) in slist, True)
def test_iterkeys(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = list(self.store.iterkeys())
self.assertEqual('min' in slist, True)
def test_itervalues(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = list(self.store.itervalues())
self.assertEqual(6 in slist, True)
def test_pop(self):
self.store['max'] = 3
self.store['min'] = 6
item = self.store.pop('min')
self.assertEqual(item, 6)
def test_popitem(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
item = self.store.popitem()
self.assertEqual(len(item) + len(self.store), 4)
def test_setdefault(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['powl'] = 7
self.store.setdefault('pow', 8)
self.assertEqual(self.store['pow'], 8)
def test_update(self):
from shove import Shove
tstore = Shove()
tstore['max'] = 3
tstore['min'] = 6
tstore['pow'] = 7
self.store['max'] = 2
self.store['min'] = 3
self.store['pow'] = 7
self.store.update(tstore)
self.assertEqual(self.store['min'], 6)
def test_values(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = self.store.values()
self.assertEqual(6 in slist, True)
def test_keys(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = self.store.keys()
self.assertEqual('min' in slist, True)
class DataManager():
def __init__(self, filename, connection_details=VERTICA_CONNECTION):
self.filename = filename
self.store = Shove('file://' + self.filename,
'memory://',
optimize=False)
self.connection_details = connection_details
@staticmethod
def get_attribute_name(data_set, attribute):
return '_' + data_set + '_' + attribute
def get_data_set_attribute(self, data_set, key):
attrib_name = self.get_attribute_name(data_set, key)
if self.has_attribute(data_set, key):
return self.store[attrib_name]
else:
return None
self.store.close()
gc.collect()
self.store = Shove('file://' + self.filename,
'memory://',
optimize=False)
def set_data_set_attribute(self, data_set, key, value):
attrib_name = self.get_attribute_name(data_set, key)
self.store.update({attrib_name: value})
self.store.sync()
self.store.close()
gc.collect()
self.store = Shove('file://' + self.filename,
'memory://',
optimize=False)
def has_attribute(self, data_set, key):
attrib_name = self.get_attribute_name(data_set, key)
if attrib_name in self.store.keys():
return True
else:
return False
def remove_attribute(self, data_set, key):
attrib_name = self.get_attribute_name(data_set, key)
if attrib_name in self.store:
del self.store[attrib_name]
self.store.sync()
def get_data_set_list(self):
data_sets = [
key.split('_')[1] for key in self.store.keys() if '_query' in key
]
return data_sets
def fetch_from_vertica_to_df(self, data_set, query):
data_set_query = self.get_data_set_attribute(data_set, 'query')
if data_set_query != query:
connection = connect(self.connection_details)
cursor = connection.cursor()
print 'Executing ', data_set, 'Query...'
print query
columns = get_column_names_from_sql_query(query)
cursor.execute(query)
data = []
while True:
rows = cursor.fetchmany(10000)
data.extend([[str(ele) for ele in row] for row in rows])
if len(rows) <= 1:
break
df = MetadataDataFrame(data=data,
columns=columns,
meta_info={
'query': query,
'built_features': [],
'aggregate_values': {},
'columns': columns
})
cursor.close()
if len(df) == 0:
raise (ValueError('SQL result in empty fetch!!'))
else:
self.set_data_set_attribute(data_set, 'data', df)
self.set_data_set_attribute(data_set, 'query', query)
self.set_data_set_attribute(data_set, 'columns', columns)
self.set_data_set_attribute(data_set, 'built_features', [])
class TestLevelDBStore(unittest2.TestCase):
def setUp(self):
from shove import Shove
self.store = Shove('leveldb://test', compress=True)
def tearDown(self):
import shutil
shutil.rmtree('test')
def test__getitem__(self):
self.store['max'] = 3
self.assertEqual(self.store['max'], 3)
def test__setitem__(self):
self.store['max'] = 3
self.assertEqual(self.store['max'], 3)
def test__delitem__(self):
self.store['max'] = 3
del self.store['max']
self.assertEqual('max' in self.store, False)
def test_get(self):
self.store['max'] = 3
self.assertEqual(self.store.get('min'), None)
def test__cmp__(self):
from shove import Shove
tstore = Shove()
self.store['max'] = 3
tstore['max'] = 3
self.assertEqual(self.store, tstore)
def test__len__(self):
self.store['max'] = 3
self.store['min'] = 6
self.assertEqual(len(self.store), 2)
def test_close(self):
self.store.close()
self.assertEqual(self.store, None)
def test_clear(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
self.store.clear()
self.assertEqual(len(self.store), 0)
def test_items(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = list(self.store.items())
self.assertEqual(('min', 6) in slist, True)
def test_iteritems(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = list(self.store.iteritems())
self.assertEqual(('min', 6) in slist, True)
def test_iterkeys(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = list(self.store.iterkeys())
self.assertEqual('min' in slist, True)
def test_itervalues(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = list(self.store.itervalues())
self.assertEqual(6 in slist, True)
def test_pop(self):
self.store['max'] = 3
self.store['min'] = 6
item = self.store.pop('min')
self.assertEqual(item, 6)
def test_popitem(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
item = self.store.popitem()
self.assertEqual(len(item) + len(self.store), 4)
def test_setdefault(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
self.store.setdefault('bow', 8)
self.assertEqual(self.store['bow'], 8)
def test_update(self):
from shove import Shove
tstore = Shove()
tstore['max'] = 3
tstore['min'] = 6
tstore['pow'] = 7
self.store['max'] = 2
self.store['min'] = 3
self.store['pow'] = 7
self.store.update(tstore)
self.assertEqual(self.store['min'], 6)
def test_values(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = self.store.values()
self.assertEqual(6 in slist, True)
def test_keys(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = self.store.keys()
self.assertEqual('min' in slist, True)
class TestCassandraStore(unittest.TestCase):
def setUp(self):
from shove import Shove
from pycassa.system_manager import SystemManager
system_manager = SystemManager('localhost:9160')
try:
system_manager.create_column_family('Foo', 'shove')
except:
pass
self.store = Shove('cassandra://localhost:9160/Foo/shove')
def tearDown(self):
self.store.clear()
self.store.close()
from pycassa.system_manager import SystemManager
system_manager = SystemManager('localhost:9160')
system_manager.drop_column_family('Foo', 'shove')
def test__getitem__(self):
self.store['max'] = 3
self.assertEqual(self.store['max'], 3)
def test__setitem__(self):
self.store['max'] = 3
self.assertEqual(self.store['max'], 3)
def test__delitem__(self):
self.store['max'] = 3
del self.store['max']
self.assertEqual('max' in self.store, False)
def test_get(self):
self.store['max'] = 3
self.assertEqual(self.store.get('min'), None)
def test__cmp__(self):
from shove import Shove
tstore = Shove()
self.store['max'] = 3
tstore['max'] = 3
self.assertEqual(self.store, tstore)
def test__len__(self):
self.store['max'] = 3
self.store['min'] = 6
self.assertEqual(len(self.store), 2)
# def test_clear(self):
# self.store['max'] = 3
# self.store['min'] = 6
# self.store['pow'] = 7
# self.store.clear()
# self.assertEqual(len(self.store), 0)
def test_items(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = list(self.store.items())
self.assertEqual(('min', 6) in slist, True)
def test_iteritems(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = list(self.store.iteritems())
self.assertEqual(('min', 6) in slist, True)
def test_iterkeys(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = list(self.store.iterkeys())
self.assertEqual('min' in slist, True)
def test_itervalues(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = list(self.store.itervalues())
self.assertEqual(6 in slist, True)
def test_pop(self):
self.store['max'] = 3
self.store['min'] = 6
item = self.store.pop('min')
self.assertEqual(item, 6)
# def test_popitem(self):
# self.store['max'] = 3
# self.store['min'] = 6
# self.store['pow'] = 7
# item = self.store.popitem()
# self.assertEqual(len(item) + len(self.store), 4)
def test_setdefault(self):
self.store['max'] = 3
self.store['min'] = 6
# self.store['pow'] = 7
self.store.setdefault('pow', 8)
self.assertEqual(self.store.setdefault('pow', 8), 8)
self.assertEqual(self.store['pow'], 8)
def test_update(self):
from shove import Shove
tstore = Shove()
tstore['max'] = 3
tstore['min'] = 6
tstore['pow'] = 7
self.store['max'] = 2
self.store['min'] = 3
self.store['pow'] = 7
self.store.update(tstore)
self.assertEqual(self.store['min'], 6)
def test_values(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = self.store.values()
self.assertEqual(6 in slist, True)
def test_keys(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = self.store.keys()
self.assertEqual('min' in slist, True)
class Personis_server:
def __init__(self, modeldir=None, adminsfile=None, oauthconfig=None):
self.modeldir = modeldir
self.admins = yaml.load(file(adminsfile,'r'))
self.oauth_clients = Shove('sqlite:///oauth_clients.dat')
self.users = Shove('sqlite:///oauth_users.dat')
self.bearers = Shove('sqlite:///oauth_bearers.dat')
self.oauthconf = yaml.load(file(oauthconfig,'r'))
def stopper():
print 'saving persistant data'
self.oauth_clients.close()
self.users.close()
self.bearers.close()
cherrypy.engine.subscribe('stop', stopper)
@cherrypy.expose
def list_clients(self):
if cherrypy.session.get('user') == None:
cherrypy.session['target_url'] = '/list_clients'
raise cherrypy.HTTPRedirect('/login')
if not cherrypy.session.get('user')['id'] in self.admins.keys():
raise cherrypy.HTTPError()
base_path = os.path.dirname(os.path.abspath(__file__))
loader = TemplateLoader([base_path])
tmpl = loader.load('list_clients.html')
for k, v in self.oauth_clients.items():
print k, v.friendly_name
stream = tmpl.generate(clients=self.oauth_clients.values())
return stream.render('xhtml')
@cherrypy.expose
def list_clients_save(self, id, value, _method='get'):
if cherrypy.session.get('user') == None:
raise cherrypy.HTTPError()
if not cherrypy.session.get('user')['id'] in self.admins.keys():
raise cherrypy.HTTPError()
if id == "removeOneForMe":
del(self.oauth_clients[value])
self.oauth_clients.sync()
print "removed a client"
raise cherrypy.HTTPRedirect('/list_clients')
if id == "addOneForMe":
clid = ''
secret = ''
for i in range(10):
clid = clid + str(int(random.random()*100))
secret = secret + str(int(random.random()*100))
self.oauth_clients[clid] = oauth_client(
client_id = clid,
friendly_name= 'my client',
secret= secret,
redirect_uri='http://www.example.com/',
icon='/static/images/icon.svg')
self.oauth_clients.sync()
print "added a client"
raise cherrypy.HTTPRedirect('/list_clients')
clid, field = id.split('|')
print 'saving: ',clid, field, value
oldc = self.oauth_clients[clid]
oldc.__dict__[field] = value
self.oauth_clients[clid] = oldc
for k, v in self.oauth_clients.items():
print k, v.friendly_name
self.oauth_clients.sync()
return value
@cherrypy.expose
def authorize(self, client_id, redirect_uri, scope, access_type, response_type='code', approval_prompt='auto', state=None):
body = cherrypy.request.body.fp.read()
rurl = cherrypy.request.base+cherrypy.request.path_info
cherrypy.session['client_id'] = client_id
cli = self.oauth_clients[client_id]
if state <> None:
cherrypy.session['state'] = state
if cli.redirect_uri <> redirect_uri:
raise cherrypy.HTTPError()
raise cherrypy.HTTPRedirect('/login')
@cherrypy.expose
def login(self):
flow = OAuth2WebServerFlow(client_id=self.oauthconf['personis_client_id'],
client_secret=self.oauthconf['personis_client_secret'],
scope='https://www.googleapis.com/auth/userinfo.profile https://www.googleapis.com/auth/userinfo.email',
user_agent='personis-server/1.0')
callback = callback = cherrypy.request.base + '/logged_in'
authorize_url = flow.step1_get_authorize_url(callback)
cherrypy.session['flow'] = flow
raise cherrypy.HTTPRedirect(authorize_url)
@cherrypy.expose
def logged_in(self, code):
flow = cherrypy.session.get('flow')
if not flow:
raise IOError()
credentials = flow.step2_exchange(cherrypy.request.params)
http = httplib2.Http()
http = credentials.authorize(http)
cjson = credentials.to_json()
cjson = json.loads(cjson)
content = http.request('https://www.googleapis.com/oauth2/v1/userinfo?access_token='+cjson['access_token'])
#print 'content', content
usr = json.loads(content[1])
cherrypy.session['user'] = usr
self.users[usr['id']] = [usr, credentials]
self.users.sync()
if not 'client_id' in cherrypy.session:
raise cherrypy.HTTPRedirect(cherrypy.session['target_url'])
cli = self.oauth_clients[cherrypy.session['client_id']]
print 'loggedin session id',cherrypy.session.id
# if no model for user, create one.
if not os.path.exists(os.path.join(self.modeldir,usr['id'])):
fn = os.path.join('tmp',usr['id'])
#f1 = open('Modeldefs/user.template','r').read()
#f = open(fn, 'w')
#f.write(f1%(usr['given_name'],usr['family_name'], usr['gender'], usr['email']))
#f.close()
mkmodel(model=usr['id'], mfile='Modeldefs/user.prod', modeldir=self.modeldir, user=usr['id'], password='')
um = Personis_a.Access(model=usr['id'], modeldir=self.modeldir, user=usr['id'], password='')
ev = Personis_base.Evidence(source="Create_Model", evidence_type="explicit", value=usr['given_name'])
um.tell(context=["Personal"], componentid='firstname', evidence=ev)
ev = Personis_base.Evidence(source="Create_Model", evidence_type="explicit", value=usr['family_name'])
um.tell(context=["Personal"], componentid='lastname', evidence=ev)
ev = Personis_base.Evidence(source="Create_Model", evidence_type="explicit", value=usr['gender'])
um.tell(context=["Personal"], componentid='gender', evidence=ev)
ev = Personis_base.Evidence(source="Create_Model", evidence_type="explicit", value=usr['email'])
um.tell(context=["Personal"], componentid='email', evidence=ev)
reslist = um.ask(context=["Personal"], view=['firstname','email'])
Personis_util.printcomplist(reslist)
# if it's mneme, then don't ask whether it's all ok. just do it.
#if cli.secret == 'personis_client_secret_mneme':
#raise cherrypy.HTTPRedirect('/allow')
#otherwise, ask yes/no
base_path = os.path.dirname(os.path.abspath(__file__))
loader = TemplateLoader([base_path])
tmpl = loader.load('appQuery.html')
stream = tmpl.generate(name=usr['given_name'], app=cli.friendly_name, icon=cli.icon)
return stream.render('xhtml')
@cherrypy.expose
def allow(self):
usr = cherrypy.session.get('user')
cli = self.oauth_clients[cherrypy.session['client_id']]
val_key = ''
for i in range(10):
val_key = val_key+ str(int(random.random()*100))
rdi = cli.redirect_uri + '?'
if 'state' in cherrypy.session:
rdi = rdi + 'state='+cherrypy.session['state']+'&'
rdi = rdi + 'code=' + val_key
cherrypy.session['auth_code'] = val_key
cli.auth_codes[val_key] = [time.time(),usr['id']]
cli = self.oauth_clients[cherrypy.session.get('client_id')]
redr = cli.redirect_uri
um = Personis_a.Access(model=usr['id'], modeldir=self.modeldir, user=usr['id'], password='')
cherrypy.session['um'] = um
result = um.registerapp(app=cherrypy.session['client_id'], desc='', password='')
raise cherrypy.HTTPRedirect(redr+'?code='+cherrypy.session.get('auth_code'))
@cherrypy.expose
def dissallow(self):
cli = self.oauth_clients[cherrypy.session.get('client_id')]
redr = cli.redirect_uri+'?error=access_denied'
#print redr
raise cherrypy.HTTPRedirect(redr)
@cherrypy.expose
def request_token(self, code, redirect_uri, client_id, client_secret, scope, grant_type):
print code, client_id, client_secret
cli = self.oauth_clients[client_id]
if not code in cli.auth_codes:
raise cherrypy.HTTPError()
atoken = ''
rtoken = ''
for i in range(10):
atoken = atoken + str(int(random.random()*100))
rtoken = rtoken + str(int(random.random()*100))
user = self.users[cli.auth_codes[code][1]][0]
b = {}
b['user'] = user
b['credentials'] = self.users[cli.auth_codes[code][1]][1]
print b
self.bearers[atoken] = b
#print user
cli.request_tokens[atoken] = user['id']
cli.refresh_tokens[rtoken] = ''
s = '''
{
"access_token":"%s",
"token_type":"bearer",
"expires_in":3600,
"refresh_token":"%s",
"example_parameter":"example_value"
}
'''%(atoken,rtoken)
del(self.users[cli.auth_codes[code][1]])
del(cli.auth_codes[code])
self.oauth_clients[client_id] = cli
print s
return s
@cherrypy.expose
def default(self, *args):
print 'path', cherrypy.request.path_info
print 'headers', cherrypy.request.headers
jsonobj = cherrypy.request.body.fp.read()
print 'body', jsonobj
try:
access_token = cherrypy.request.headers['Authorization'].split()[1]
print self.bearers
usr = self.bearers[access_token]['user']
except:
raise cherrypy.HTTPError()
try:
pargs = json.loads(jsonobj)
except:
print "bad request - cannot decode json - possible access from web browser"
return json.dumps("Personis User Model server. Not accessible using a web browser.")
# dirty kludge to get around unicode
for k,v in pargs.items():
if type(v) == type(u''):
pargs[k] = str(v)
if type(k) == type(u''):
del pargs[k]
pargs[str(k)] = v
try:
result = False
if args[0] == 'mkmodel':
# fixme need to implement security
# and error handling
Personis_base.MkModel(model=usr['id'], modeldir=self.modeldir, \
user=usr['id'], password='', description=pargs['description'])
result = True
else:
um = Personis_a.Access(model=usr['id'], modeldir=self.modeldir, user=usr['id'], password='')
if args[0] == 'access':
result = True
elif args[0] == 'tell':
result = um.tell(context=pargs['context'], componentid=pargs['componentid'], evidence=Personis_base.Evidence(**pargs['evidence']))
elif args[0] == 'ask':
reslist = um.ask(context=pargs['context'], view=pargs['view'], resolver=pargs['resolver'], \
showcontexts=pargs['showcontexts'])
if pargs['showcontexts']:
cobjlist, contexts, theviews, thesubs = reslist
cobjlist = [c.__dict__ for c in cobjlist]
for c in cobjlist:
if c["evidencelist"]:
c["evidencelist"] = [e for e in c["evidencelist"]]
newviews = {}
if theviews != None:
for vname,v in theviews.items():
newviews[vname] = v.__dict__
else:
newviews = None
reslist = [cobjlist, contexts, newviews, thesubs]
else:
reslist = [c.__dict__ for c in reslist]
for c in reslist:
if c["evidencelist"]:
c["evidencelist"] = [e for e in c["evidencelist"]]
result = reslist
elif args[0] == 'subscribe':
result = um.subscribe(context=pargs['context'], view=pargs['view'], subscription=pargs['subscription'])
elif args[0] == 'delete_sub':
result = um.delete_sub(context=pargs['context'], componentid=pargs['componentid'], subname=pargs['subname'])
elif args[0] == 'list_subs':
result = um.list_subs(context=pargs['context'], componentid=pargs['componentid'])
elif args[0] == 'export_model':
result = um.export_model(context=pargs['context'], resolver=pargs['resolver'])
elif args[0] == 'import_model':
result = um.import_model(context=pargs['context'], partial_model=pargs['partial_model'])
elif args[0] == 'set_goals':
result = um.set_goals(context=pargs['context'], componentid=pargs['componentid'], goals=pargs['goals'])
elif args[0] == 'registerapp':
result = um.registerapp(app=pargs['app'], desc=pargs['description'], password=pargs['apppassword'])
elif args[0] == 'deleteapp':
result = um.deleteapp(app=pargs['app'])
elif args[0] == 'getpermission':
result = um.getpermission(context=pargs['context'], componentid=pargs['componentid'], app=pargs['app'])
elif args[0] == 'setpermission':
result = um.setpermission(context=pargs['context'], componentid=pargs['componentid'], app=pargs['app'], permissions=pargs['permissions'])
elif args[0] == 'listapps':
result = um.listapps()
elif args[0] == 'mkcomponent':
comp = Personis_base.Component(**pargs["componentobj"])
result = um.mkcomponent(pargs["context"], comp)
elif args[0] == 'delcomponent':
result = um.delcomponent(pargs["context"], pargs["componentid"])
elif args[0] == 'delcontext':
result = um.delcontext(pargs["context"])
elif args[0] == 'setresolver':
result = um.setresolver(pargs["context"], pargs["componentid"], pargs["resolver"])
elif args[0] == 'getresolvers':
result = um.getresolvers()
elif args[0] == 'mkview':
viewobj = Personis_base.View(**pargs["viewobj"])
result = um.mkview(pargs["context"], viewobj)
elif args[0] == 'delview':
result = um.delview(pargs["context"], pargs["viewid"])
elif args[0] == 'mkcontext':
contextobj = Personis_base.Context(**pargs["contextobj"])
result = um.mkcontext(pargs["context"], contextobj)
elif args[0] == 'getcontext':
result = um.getcontext(pargs["context"], pargs["getsize"])
# Repackage result code with error values IF there is a version string.
if pargs.has_key("version"):
new_result = {}
new_result["result"] = "ok"
new_result["val"] = result
result = new_result
except Exception, e:
print "Exception:", e
traceback.print_exc()
if pargs.has_key("version"):
new_result = {}
#new_result["errorType"] = e.__class__.__name__
#new_result["errorData"] = e.__dict__.copy()
#new_result["pythonPickel"] = cPickle.dumps(e)
new_result["val"] = [e.__class__.__name__, e.__dict__.copy(), cPickle.dumps(e)]
new_result["result"] = "error"
result = new_result
else:
result = False
return json.dumps(result)
class TestHDF5Store(unittest2.TestCase):
def setUp(self):
from shove import Shove
self.store = Shove('hdf5://test.hdf5/test')
def tearDown(self):
import os
self.store.close()
try:
os.remove('test.hdf5')
except OSError:
pass
def test__getitem__(self):
self.store['max'] = 3
self.assertEqual(self.store['max'], 3)
def test__setitem__(self):
self.store['max'] = 3
self.assertEqual(self.store['max'], 3)
def test__delitem__(self):
self.store['max'] = 3
del self.store['max']
self.assertEqual('max' in self.store, False)
def test_get(self):
self.store['max'] = 3
self.assertEqual(self.store.get('min'), None)
def test__cmp__(self):
from shove import Shove
tstore = Shove()
self.store['max'] = 3
tstore['max'] = 3
self.assertEqual(self.store, tstore)
def test__len__(self):
self.store['max'] = 3
self.store['min'] = 6
self.assertEqual(len(self.store), 2)
def test_close(self):
self.store.close()
self.assertEqual(self.store, None)
def test_clear(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
self.store.clear()
self.assertEqual(len(self.store), 0)
def test_items(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = list(self.store.items())
self.assertEqual(('min', 6) in slist, True)
def test_iteritems(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = list(self.store.iteritems())
self.assertEqual(('min', 6) in slist, True)
def test_iterkeys(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = list(self.store.iterkeys())
self.assertEqual('min' in slist, True)
def test_itervalues(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = list(self.store.itervalues())
self.assertEqual(6 in slist, True)
def test_pop(self):
self.store['max'] = 3
self.store['min'] = 6
item = self.store.pop('min')
self.assertEqual(item, 6)
def test_popitem(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
item = self.store.popitem()
self.assertEqual(len(item) + len(self.store), 4)
def test_setdefault(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
self.store.setdefault('bow', 8)
self.assertEqual(self.store['bow'], 8)
def test_update(self):
from shove import Shove
tstore = Shove()
tstore['max'] = 3
tstore['min'] = 6
tstore['pow'] = 7
self.store['max'] = 2
self.store['min'] = 3
self.store['pow'] = 7
self.store.update(tstore)
self.assertEqual(self.store['min'], 6)
def test_values(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = self.store.values()
self.assertEqual(6 in slist, True)
def test_keys(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = self.store.keys()
self.assertEqual('min' in slist, True)
class Store(object):
def setUp(self):
from shove import Shove
self.store = Shove(
self.initstring,
optimize=False,
compress=True,
sync=0,
)
def tearDown(self):
self.store.close()
def test__getitem__(self):
self.store['max'] = 3
self.store.sync()
self.assertEqual(self.store['max'], 3)
def test__setitem__(self):
self.store['max'] = 3
self.store['d'] = {'A': 1}, {'A': 1}
self.store['d'] = {'AA': 1}, {'A': 1}
self.store['d'] = {'AA': 1}, {'AA': 1}
self.store.sync()
self.assertEqual(self.store['max'], 3)
def test__delitem__(self):
self.store['max'] = 3
self.store.sync()
del self.store['max']
self.store.sync()
self.assertEqual('max' in self.store, False)
def test_get(self):
self.store['max'] = 3
self.store.sync()
self.assertEqual(self.store.get('min'), None)
def test__cmp__(self):
from shove import Shove
tstore = Shove()
self.store['max'] = 3
self.store.sync()
tstore['max'] = 3
self.assertEqual(self.store, tstore)
def test__len__(self):
self.store['max'] = 3
self.store['min'] = 6
self.store.sync()
self.assertEqual(len(self.store), 2)
def test_items(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
self.store.sync()
slist = list(items(self.store))
self.assertEqual(('min', 6) in slist, True)
def test_keys(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
self.store.sync()
slist = list(keys(self.store))
self.assertEqual('min' in slist, True)
def test_values(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
self.store.sync()
slist = list(values(self.store))
self.assertEqual(6 in slist, True)
def test_pop(self):
self.store['max'] = 3
self.store['min'] = 6
self.store.sync()
item = self.store.pop('min')
self.store.sync()
self.assertEqual(item, 6)
def test_setdefault(self):
self.store['max'] = 3
self.store['min'] = 6
self.store.sync()
self.assertEqual(self.store.setdefault('pow', 8), 8)
self.store.sync()
self.assertEqual(self.store['pow'], 8)
def test_update(self):
from shove import Shove
tstore = Shove()
tstore['max'] = 3
tstore['min'] = 6
tstore['pow'] = 7
self.store['max'] = 2
self.store['min'] = 3
self.store['pow'] = 7
self.store.update(tstore)
self.store.sync()
self.assertEqual(self.store['min'], 6)
def test_clear(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
self.store.sync()
self.store.clear()
self.store.sync()
self.assertEqual(len(self.store), 0)
def test_popitem(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
self.store.sync()
item = self.store.popitem()
self.store.sync()
self.assertEqual(len(item) + len(self.store), 4)
def test_close(self):
self.store.close()
self.assertEqual(self.store._store, None)
self.assertEqual(self.store._buffer, None)
self.assertEqual(self.store._cache, None)
class TestSvnStore(unittest.TestCase):
svnstring = 'SVN test string here'
def setUp(self):
from shove import Shove
self.store = Shove(self.svnstring, compress=True)
def tearDown(self):
self.store.clear()
self.store.close()
def test__getitem__(self):
self.store['max'] = 3
self.store.sync()
self.assertEqual(self.store['max'], 3)
def test__setitem__(self):
self.store['max'] = 3
self.store.sync()
self.assertEqual(self.store['max'], 3)
def test__delitem__(self):
self.store['max'] = 3
del self.store['max']
self.assertEqual('max' in self.store, False)
def test_get(self):
self.store['max'] = 3
self.store.sync()
self.assertEqual(self.store.get('min'), None)
def test__cmp__(self):
from shove import Shove
tstore = Shove()
self.store['max'] = 3
tstore['max'] = 3
self.store.sync()
tstore.sync()
self.assertEqual(self.store, tstore)
def test__len__(self):
self.store['max'] = 3
self.store['min'] = 6
self.store.sync()
self.assertEqual(len(self.store), 2)
def test_clear(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
self.store.sync()
self.store.clear()
self.assertEqual(len(self.store), 0)
def test_items(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
self.store.sync()
slist = list(self.store.items())
self.assertEqual(('min', 6) in slist, True)
def test_iteritems(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
self.store.sync()
slist = list(self.store.iteritems())
self.assertEqual(('min', 6) in slist, True)
def test_iterkeys(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
self.store.sync()
slist = list(self.store.iterkeys())
self.assertEqual('min' in slist, True)
def test_itervalues(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
self.store.sync()
slist = list(self.store.itervalues())
self.assertEqual(6 in slist, True)
def test_pop(self):
self.store['max'] = 3
self.store['min'] = 6
self.store.sync()
item = self.store.pop('min')
self.assertEqual(item, 6)
def test_popitem(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
self.store.sync()
item = self.store.popitem()
self.store.sync()
self.assertEqual(len(item) + len(self.store), 4)
def test_setdefault(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['powl'] = 7
self.store.setdefault('pow', 8)
self.store.sync()
self.assertEqual(self.store['pow'], 8)
def test_update(self):
from shove import Shove
tstore = Shove()
tstore['max'] = 3
tstore['min'] = 6
tstore['pow'] = 7
self.store['max'] = 2
self.store['min'] = 3
self.store['pow'] = 7
self.store.sync()
self.store.update(tstore)
self.store.sync()
self.assertEqual(self.store['min'], 6)
def test_values(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
self.store.sync()
slist = self.store.values()
self.assertEqual(6 in slist, True)
def test_keys(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
self.store.sync()
slist = self.store.keys()
self.assertEqual('min' in slist, True)
class TestRedisStore(unittest.TestCase):
def setUp(self):
from shove import Shove
self.store = Shove('redis://localhost:6379/0')
def tearDown(self):
self.store.clear()
self.store.close()
def test__getitem__(self):
self.store['max'] = 3
self.assertEqual(self.store['max'], 3)
def test__setitem__(self):
self.store['max'] = 3
self.assertEqual(self.store['max'], 3)
def test__delitem__(self):
self.store['max'] = 3
del self.store['max']
self.assertEqual('max' in self.store, False)
def test_get(self):
self.store['max'] = 3
self.assertEqual(self.store.get('min'), None)
def test__cmp__(self):
from shove import Shove
tstore = Shove()
self.store['max'] = 3
tstore['max'] = 3
self.assertEqual(self.store, tstore)
def test__len__(self):
self.store['max'] = 3
self.store['min'] = 6
self.assertEqual(len(self.store), 2)
def test_clear(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
self.store.clear()
self.assertEqual(len(self.store), 0)
def test_items(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = list(self.store.items())
self.assertEqual(('min', 6) in slist, True)
def test_iteritems(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = list(self.store.iteritems())
self.assertEqual(('min', 6) in slist, True)
def test_iterkeys(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = list(self.store.iterkeys())
self.assertEqual('min' in slist, True)
def test_itervalues(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = list(self.store.itervalues())
self.assertEqual(6 in slist, True)
def test_pop(self):
self.store['max'] = 3
self.store['min'] = 6
item = self.store.pop('min')
self.assertEqual(item, 6)
def test_popitem(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
item = self.store.popitem()
self.assertEqual(len(item) + len(self.store), 4)
def test_setdefault(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['powl'] = 7
self.store.setdefault('pow', 8)
self.assertEqual(self.store.setdefault('pow', 8), 8)
self.assertEqual(self.store['pow'], 8)
def test_update(self):
from shove import Shove
tstore = Shove()
tstore['max'] = 3
tstore['min'] = 6
tstore['pow'] = 7
self.store['max'] = 2
self.store['min'] = 3
self.store['pow'] = 7
self.store.update(tstore)
self.assertEqual(self.store['min'], 6)
def test_values(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = self.store.values()
self.assertEqual(6 in slist, True)
def test_keys(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = self.store.keys()
self.assertEqual('min' in slist, True)
class TestDbmStore(unittest.TestCase):
def setUp(self):
self.store = Shove('dbm://test.dbm', compress=True)
def tearDown(self):
self.store.close()
os.remove('test.dbm')
def test__getitem__(self):
self.store['max'] = 3
self.assertEqual(self.store['max'], 3)
def test__setitem__(self):
self.store['max'] = 3
self.assertEqual(self.store['max'], 3)
def test__delitem__(self):
self.store['max'] = 3
del self.store['max']
self.assertEqual('max' in self.store, False)
def test_get(self):
self.store['max'] = 3
self.assertEqual(self.store.get('min'), None)
def test__cmp__(self):
tstore = Shove()
self.store['max'] = 3
tstore['max'] = 3
self.assertEqual(self.store, tstore)
def test__len__(self):
self.store['max'] = 3
self.store['min'] = 6
self.assertEqual(len(self.store), 2)
def test_close(self):
self.store.close()
self.assertEqual(self.store, None)
def test_clear(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
self.store.clear()
self.assertEqual(len(self.store), 0)
def test_items(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = list(self.store.items())
self.assertEqual(('min', 6) in slist, True)
def test_iteritems(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = list(self.store.iteritems())
self.assertEqual(('min', 6) in slist, True)
def test_iterkeys(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = list(self.store.iterkeys())
self.assertEqual('min' in slist, True)
def test_itervalues(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = list(self.store.itervalues())
self.assertEqual(6 in slist, True)
def test_pop(self):
self.store['max'] = 3
self.store['min'] = 6
item = self.store.pop('min')
self.assertEqual(item, 6)
def test_popitem(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
item = self.store.popitem()
self.assertEqual(len(item) + len(self.store), 4)
def test_setdefault(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['powl'] = 7
self.store.setdefault('pow', 8)
self.assertEqual(self.store['pow'], 8)
def test_update(self):
tstore = Shove()
tstore['max'] = 3
tstore['min'] = 6
tstore['pow'] = 7
self.store['max'] = 2
self.store['min'] = 3
self.store['pow'] = 7
self.store.update(tstore)
self.assertEqual(self.store['min'], 6)
def test_values(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = self.store.values()
self.assertEqual(6 in slist, True)
def test_keys(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = self.store.keys()
self.assertEqual('min' in slist, True)
class TestCassandraStore(unittest.TestCase):
def setUp(self):
from shove import Shove
from pycassa.system_manager import SystemManager
system_manager = SystemManager('localhost:9160')
try:
system_manager.create_column_family('Foo', 'shove')
except:
pass
self.store = Shove('cassandra://localhost:9160/Foo/shove')
def tearDown(self):
self.store.clear()
self.store.close()
from pycassa.system_manager import SystemManager
system_manager = SystemManager('localhost:9160')
system_manager.drop_column_family('Foo', 'shove')
def test__getitem__(self):
self.store['max'] = 3
self.assertEqual(self.store['max'], 3)
def test__setitem__(self):
self.store['max'] = 3
self.assertEqual(self.store['max'], 3)
def test__delitem__(self):
self.store['max'] = 3
del self.store['max']
self.assertEqual('max' in self.store, False)
def test_get(self):
self.store['max'] = 3
self.assertEqual(self.store.get('min'), None)
def test__cmp__(self):
from shove import Shove
tstore = Shove()
self.store['max'] = 3
tstore['max'] = 3
self.assertEqual(self.store, tstore)
def test__len__(self):
self.store['max'] = 3
self.store['min'] = 6
self.assertEqual(len(self.store), 2)
# def test_clear(self):
# self.store['max'] = 3
# self.store['min'] = 6
# self.store['pow'] = 7
# self.store.clear()
# self.assertEqual(len(self.store), 0)
def test_items(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = list(self.store.items())
self.assertEqual(('min', 6) in slist, True)
def test_iteritems(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = list(self.store.iteritems())
self.assertEqual(('min', 6) in slist, True)
def test_iterkeys(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = list(self.store.iterkeys())
self.assertEqual('min' in slist, True)
def test_itervalues(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = list(self.store.itervalues())
self.assertEqual(6 in slist, True)
def test_pop(self):
self.store['max'] = 3
self.store['min'] = 6
item = self.store.pop('min')
self.assertEqual(item, 6)
# def test_popitem(self):
# self.store['max'] = 3
# self.store['min'] = 6
# self.store['pow'] = 7
# item = self.store.popitem()
# self.assertEqual(len(item) + len(self.store), 4)
def test_setdefault(self):
self.store['max'] = 3
self.store['min'] = 6
# self.store['pow'] = 7
self.store.setdefault('pow', 8)
self.assertEqual(self.store.setdefault('pow', 8), 8)
self.assertEqual(self.store['pow'], 8)
def test_update(self):
from shove import Shove
tstore = Shove()
tstore['max'] = 3
tstore['min'] = 6
tstore['pow'] = 7
self.store['max'] = 2
self.store['min'] = 3
self.store['pow'] = 7
self.store.update(tstore)
self.assertEqual(self.store['min'], 6)
def test_values(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = self.store.values()
self.assertEqual(6 in slist, True)
def test_keys(self):
self.store['max'] = 3
self.store['min'] = 6
self.store['pow'] = 7
slist = self.store.keys()
self.assertEqual('min' in slist, True)
if pr and args.profile:
# Finish profiling
pr.disable()
pr.dump_stats('library.prof')
print '[PROFILE] Done'
if pr and args.graph:
# Convert profile to dot
subprocess.Popen(['python', os.environ['GPROF2DOT'], '-fpstats', 'library.prof', '-olibrary.dot'])
print '[DOT] Done'
# Convert dot to png
with open('library.png', 'w') as fp:
p = subprocess.Popen(['dot', '-Tpng', 'library.dot'], stdout=fp)
ret_code = p.wait()
print '[PNG] Done (%s)' % ret_code
if pr and args.graph and args.view:
# Display image (in default viewer)
path = os.path.abspath(os.path.join(os.path.dirname(__file__), 'library.png'))
print '[VIEW] Opening image at "%s"' % path
os.system("start %s" % path)
# Close shove caches
http_cache.close()
matcher_cache.close()
metadata_cache.close()
print '[PROFILE] Done'
if pr and args.graph:
# Convert profile to dot
subprocess.Popen([
'python', os.environ['GPROF2DOT'], '-fpstats', 'library.prof',
'-olibrary.dot'
])
print '[DOT] Done'
# Convert dot to png
with open('library.png', 'w') as fp:
p = subprocess.Popen(['dot', '-Tpng', 'library.dot'], stdout=fp)
ret_code = p.wait()
print '[PNG] Done (%s)' % ret_code
if pr and args.graph and args.view:
# Display image (in default viewer)
path = os.path.abspath(
os.path.join(os.path.dirname(__file__), 'library.png'))
print '[VIEW] Opening image at "%s"' % path
os.system("start %s" % path)
# Close shove caches
http_cache.close()
matcher_cache.close()
metadata_cache.close()
