class TestCassandraStore(Store, Spawn, unittest.TestCase): cmd = ['cassandra', '-f'] @classmethod def setUpClass(cls): super(TestCassandraStore, cls).setUpClass() import time time.sleep(5.0) def setUp(self): from shove import Shove from pycassa.system_manager import SystemManager # @UnresolvedImport @IgnorePep8 system_manager = SystemManager('localhost:9160') try: system_manager.create_column_family('Murk', 'shove') except: pass self.store = Shove('cassandra://localhost:9160/Murk/shove') def tearDown(self): if self.store._store is not None: self.store.clear() self.store.close() from pycassa.system_manager import SystemManager # @UnresolvedImport @IgnorePep8 system_manager = SystemManager('localhost:9160') system_manager.drop_column_family('Murk', 'shove') @classmethod def tearDownClass(cls): from fabric.api import local local('killall java')
def getRSSFeed(self, url, post_data=None): # create provider storaqe cache storage = Shove('sqlite:///' + ek.ek(os.path.join, sickbeard.CACHE_DIR, self.provider.name) + '.db') fc = cache.Cache(storage) parsed = list(urlparse.urlparse(url)) parsed[2] = re.sub("/{2,}", "/", parsed[2]) # replace two or more / with one if post_data: url = url + 'api?' + urllib.urlencode(post_data) f = fc.fetch(url) if not f: logger.log(u"Error loading " + self.providerID + " URL: " + url, logger.ERROR) return None elif 'error' in f.feed: logger.log(u"Newznab ERROR:[%s] CODE:[%s]" % (f.feed['error']['description'], f.feed['error']['code']), logger.DEBUG) return None elif not f.entries: logger.log(u"No items found on " + self.providerID + " using URL: " + url, logger.WARNING) return None storage.close() return f
class _ShoveWrapper(object): def __init__(self, loc): self._loc = loc self._shove = Shove(self._loc) def __enter__(self): return self._shove def __exit__(self, type, value, traceback): self._shove.close()
def create_sequence_dbs_for_GAF(gaf, transcripts_file, output_dir): from Bio import SeqIO from Bio import Seq import os print "Indexing GAF db by transcript id...\n" gaf_transcript_idx = dict() for i, g in enumerate(gaf): for k in gaf[g].keys(): for ctr, t in enumerate(gaf[g][k]): gaf_transcript_idx[t['transcript_id']] = (ctr, g, k) fh_transcripts = SeqIO.parse(transcripts_file, 'fasta') # transcripts_shlv = shelve.open(os.path.join(output_dir, 'GAF_transcript_seqs.fa.shlv'), 'c') # proteins_shlv = shelve.open(os.path.join(output_dir, 'GAF_protein_seqs.fa.shlv'), 'c') transcripts_shlv = Shove( "file://" + os.path.join(output_dir, 'GAF_transcript_seqs.fa.shove')) protein_seqs_url = "file://" + os.path.join(output_dir, 'GAF_protein_seqs.fa.shove') proteins_shlv = Shove(protein_seqs_url) print "Writing transcript and protein shove dbs..." j = 0 transcripts_to_remove = list() for transcript in fh_transcripts: if j % 1000 == 0: print j j += 1 if transcript.name not in gaf_transcript_idx: continue gaf_record = gaf[gaf_transcript_idx[transcript.name][1]][ gaf_transcript_idx[transcript.name][2]][gaf_transcript_idx[ transcript.name][0]] raw_seq = str(transcript.seq) transcripts_shlv[transcript.name] = raw_seq if 'cds_start' not in gaf_record or not gaf_record['cds_start']: continue prot_seq = Seq.translate(raw_seq[gaf_record['cds_start'] - 1:gaf_record['cds_stop']]) if prot_seq[-1] == '*': prot_seq = prot_seq[:-1] elif prot_seq.find('*') != -1: # skip small number (n=12) transcripts with incorrect CDS coordinates transcripts_to_remove.append(transcript.name) continue proteins_shlv[transcript.name] = prot_seq for t in transcripts_to_remove: del transcripts_shlv[t] transcripts_shlv.close() proteins_shlv.close() return transcripts_to_remove, protein_seqs_url
class TestFTPStore(Store, unittest.TestCase): initstring = 'ftp://127.0.0.1/' def setUp(self): from shove import Shove self.store = Shove(self.initstring, compress=True) def tearDown(self): self.store.clear() self.store.close()
def primary_file_name(self): """The file name for the 'primary' file in the bucket. It is this file from which data is loaded. Other files within the bucket should be auxiliary to this file. (E.g. they should contain projection information.) """ shove = Shove(self._shove_url) try: return shove['primary_file_name'] except KeyError: return None finally: shove.close()
def create_sequence_dbs_for_GAF(gaf, transcripts_file, output_dir): from Bio import SeqIO from Bio import Seq import os print "Indexing GAF db by transcript id...\n" gaf_transcript_idx = dict() for i,g in enumerate(gaf): for k in gaf[g].keys(): for ctr,t in enumerate(gaf[g][k]): gaf_transcript_idx[t['transcript_id']] = (ctr,g,k) fh_transcripts = SeqIO.parse(transcripts_file, 'fasta') # transcripts_shlv = shelve.open(os.path.join(output_dir, 'GAF_transcript_seqs.fa.shlv'), 'c') # proteins_shlv = shelve.open(os.path.join(output_dir, 'GAF_protein_seqs.fa.shlv'), 'c') transcripts_shlv = Shove("file://" + os.path.join(output_dir, 'GAF_transcript_seqs.fa.shove')) protein_seqs_url = "file://" + os.path.join(output_dir, 'GAF_protein_seqs.fa.shove') proteins_shlv = Shove(protein_seqs_url) print "Writing transcript and protein shove dbs..." j = 0 transcripts_to_remove = list() for transcript in fh_transcripts: if j % 1000 == 0: print j j += 1 if transcript.name not in gaf_transcript_idx: continue gaf_record = gaf[gaf_transcript_idx[transcript.name][1]][gaf_transcript_idx[transcript.name][2]][gaf_transcript_idx[transcript.name][0]] raw_seq = str(transcript.seq) transcripts_shlv[transcript.name] = raw_seq if 'cds_start' not in gaf_record or not gaf_record['cds_start']: continue prot_seq = Seq.translate(raw_seq[gaf_record['cds_start']-1:gaf_record['cds_stop']]) if prot_seq[-1] == '*': prot_seq = prot_seq[:-1] elif prot_seq.find('*') != -1: # skip small number (n=12) transcripts with incorrect CDS coordinates transcripts_to_remove.append(transcript.name) continue proteins_shlv[transcript.name] = prot_seq for t in transcripts_to_remove: del transcripts_shlv[t] transcripts_shlv.close() proteins_shlv.close() return transcripts_to_remove,protein_seqs_url
class TestHDF5Store(Store, unittest.TestCase): initstring = 'hdf5://test.hdf5/test' def setUp(self): from shove import Shove self.store = Shove() def tearDown(self): import os self.store.close() try: os.remove('test.hdf5') except OSError: pass
def save_data(self, data): db = Shove(self.conf['data_file']) modified = False for item in data: try: db[item['guid']] except KeyError: db[item['guid']] = item modified = True else: if db[item['guid']] != item: db[item['guid']] = item modified = True db.close() return modified
class ShoveCache(Cache): """Expects a url in the form that shove requires. Maintains a cache of keys to speed performance.""" def __init__(self, url_db, url_cache): self.db = Shove(url_db, url_cache, optimize=False, max_entries=2000) def retrieve_from_cache(self, key): try: val = self.db[key] return val except KeyError: return None def store_into_cache(self, key, value): self.db[key] = value def close_cache(self): self.db.close()
class TestZodbStore(unittest.TestCase): init = "zodb://test.db" def setUp(self): from shove import Shove self.store = Shove(self.init, compress=True) def tearDown(self): self.store.close() import os os.remove("test.db") os.remove("test.db.index") os.remove("test.db.tmp") os.remove("test.db.lock") def test__getitem__(self): self.store["max"] = 3 self.assertEqual(self.store["max"], 3) def test__setitem__(self): self.store["max"] = 3 self.assertEqual(self.store["max"], 3) def test__delitem__(self): self.store["max"] = 3 del self.store["max"] self.assertEqual("max" in self.store, False) def test_get(self): self.store["max"] = 3 self.assertEqual(self.store.get("min"), None) def test__cmp__(self): from shove import Shove tstore = Shove() self.store["max"] = 3 tstore["max"] = 3 self.assertEqual(self.store, tstore) def test__len__(self): self.store["max"] = 3 self.store["min"] = 6 self.assertEqual(len(self.store), 2) def test_close(self): self.store.close() self.assertEqual(self.store, None) def test_clear(self): self.store["max"] = 3 self.store["min"] = 6 self.store["pow"] = 7 self.store.clear() self.assertEqual(len(self.store), 0) def test_items(self): self.store["max"] = 3 self.store["min"] = 6 self.store["pow"] = 7 slist = list(self.store.items()) self.assertEqual(("min", 6) in slist, True) def test_iteritems(self): self.store["max"] = 3 self.store["min"] = 6 self.store["pow"] = 7 slist = list(self.store.iteritems()) self.assertEqual(("min", 6) in slist, True) def test_iterkeys(self): self.store["max"] = 3 self.store["min"] = 6 self.store["pow"] = 7 slist = list(self.store.iterkeys()) self.assertEqual("min" in slist, True) def test_itervalues(self): self.store["max"] = 3 self.store["min"] = 6 self.store["pow"] = 7 slist = list(self.store.itervalues()) self.assertEqual(6 in slist, True) def test_pop(self): self.store["max"] = 3 self.store["min"] = 6 item = self.store.pop("min") self.assertEqual(item, 6) def test_popitem(self): self.store["max"] = 3 self.store["min"] = 6 self.store["pow"] = 7 item = self.store.popitem() self.assertEqual(len(item) + len(self.store), 4) def test_setdefault(self): self.store["max"] = 3 self.store["min"] = 6 self.store["powl"] = 7 self.store.setdefault("pow", 8) self.assertEqual(self.store["pow"], 8) def test_update(self): from shove import Shove tstore = Shove() tstore["max"] = 3 tstore["min"] = 6 tstore["pow"] = 7 self.store["max"] = 2 self.store["min"] = 3 self.store["pow"] = 7 self.store.update(tstore) self.assertEqual(self.store["min"], 6) def test_values(self): self.store["max"] = 3 self.store["min"] = 6 self.store["pow"] = 7 slist = self.store.values() self.assertEqual(6 in slist, True) def test_keys(self): self.store["max"] = 3 self.store["min"] = 6 self.store["pow"] = 7 slist = self.store.keys() self.assertEqual("min" in slist, True)
class DataManager(): def __init__(self, filename, connection_details=VERTICA_CONNECTION): self.filename = filename self.store = Shove('file://'+self.filename, 'memory://', optimize=False) self.connection_details = connection_details @staticmethod def get_attribute_name(data_set, attribute): return '_' + data_set + '_' + attribute def get_data_set_attribute(self, data_set, key): attrib_name = self.get_attribute_name(data_set, key) if self.has_attribute(data_set, key): return self.store[attrib_name] else: return None self.store.close() gc.collect() self.store = Shove('file://'+self.filename, 'memory://', optimize=False) def set_data_set_attribute(self, data_set, key, value): attrib_name = self.get_attribute_name(data_set, key) self.store.update({attrib_name: value}) self.store.sync() self.store.close() gc.collect() self.store = Shove('file://'+self.filename, 'memory://', optimize=False) def has_attribute(self, data_set, key): attrib_name = self.get_attribute_name(data_set, key) if attrib_name in self.store.keys(): return True else: return False def remove_attribute(self, data_set, key): attrib_name = self.get_attribute_name(data_set, key) if attrib_name in self.store: del self.store[attrib_name] self.store.sync() def get_data_set_list(self): data_sets = [key.split('_')[1] for key in self.store.keys() if '_query' in key] return data_sets def fetch_from_vertica_to_df(self, data_set, query): data_set_query = self.get_data_set_attribute(data_set, 'query') if data_set_query != query: connection = connect(self.connection_details) cursor = connection.cursor() print 'Executing ', data_set, 'Query...' print query columns = get_column_names_from_sql_query(query) cursor.execute(query) data = [] while True: rows = cursor.fetchmany(10000) data.extend([[str(ele) for ele in row] for row in rows]) if len(rows) <= 1: break df = MetadataDataFrame(data=data, columns=columns, meta_info={'query': query, 'built_features': [], 'aggregate_values': {}, 'columns': columns}) cursor.close() if len(df) == 0: raise(ValueError('SQL result in empty fetch!!')) else: self.set_data_set_attribute(data_set, 'data', df) self.set_data_set_attribute(data_set, 'query', query) self.set_data_set_attribute(data_set, 'columns', columns) self.set_data_set_attribute(data_set, 'built_features', [])
def primary_file_name(self, name): shove = Shove(self._shove_url) try: shove['primary_file_name'] = name finally: shove.close()
class EntryManager(object): def __init__(self, location, **kwargs): self.store = Shove(location, **kwargs) def create_entry(self, entry): """ Create an entry in the entry manager:: >>> entry = {} >>> entry['content'] = {'content': "<p>This is an entry.</p>"} >>> entry['author'] = {'name': "Roberto De Almeida", ... 'email': "*****@*****.**"} >>> em.create_entry(entry) If empty, 'id' and 'updated' will be automatically set. """ # Required elements. entry.setdefault('id', self._next_id()) entry.setdefault('updated', time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())) # Store entry. self.store[str(entry["id"])] = entry return entry def get_entry(self, key): """ Retrieve an entry by its id:: >>> entry = em.get_entry("1") Will return a dict describing the entry. """ entry = self.store[str(key)] return entry def get_entries(self, size=None, offset=0): """ Retrieve last n entries, sorted by the 'updated' attribute and in descending order:: >>> entries = em.get_entries(10) If you omit the number, all entries will be retrieved. You can also specify an offset:: >>> entries = em.get_entries(10, 5) This will skip the first 5 entries. """ entries = self.store.values() # Sort by time. entries.sort(key=operator.itemgetter('updated'), reverse=True) if size is not None: size += offset entries = entries[offset:size] return entries def delete_entry(self, key): """ Delete an entry identified by its key:: >>> em.delete_entry("1") """ del self.store[str(key)] def update_entry(self, new_entry): """ Update an entry. >>> entry = {'title': "This is an entry"} >>> entry = em.create_entry(entry) >>> print entry['id'] 0 >>> entry['title'] = "A different title" >>> em.update_entry(entry) """ # Retrieve old entry and update id. entry = self.store[str(new_entry["id"])] entry.update(new_entry) # Update 'updated'. entry['updated'] = time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()) # Store it back. self.store[str(entry["id"])] = entry return entry def search(self, filters, flags=0, size=None, offset=0): """ Search all entries. A simple, case-sensitive search for the word "Python" in the title:: >>> entries = em.search({"title": "Python"}) For case-insensitive searching in the content:: >>> entries = em.search({"content": {"content": "Python"}}, re.IGNORECASE) """ entries = self.get_entries() entries = [entry for entry in entries if filter_(entry, filters, flags)] if size is not None: size += offset entries = entries[offset:size] return entries def _next_id(self): # List keys. keys = self.store.keys() keys = [key for key in keys if re.match(r"^\d+$", key)] keys.sort(key=int) # Get a new key. key = keys and str(int(keys[-1]) + 1) or "0" return key def close(self): self.store.close()
class TestLevelDBStore(unittest2.TestCase): def setUp(self): from shove import Shove self.store = Shove('leveldb://test', compress=True) def tearDown(self): import shutil shutil.rmtree('test') def test__getitem__(self): self.store['max'] = 3 self.assertEqual(self.store['max'], 3) def test__setitem__(self): self.store['max'] = 3 self.assertEqual(self.store['max'], 3) def test__delitem__(self): self.store['max'] = 3 del self.store['max'] self.assertEqual('max' in self.store, False) def test_get(self): self.store['max'] = 3 self.assertEqual(self.store.get('min'), None) def test__cmp__(self): from shove import Shove tstore = Shove() self.store['max'] = 3 tstore['max'] = 3 self.assertEqual(self.store, tstore) def test__len__(self): self.store['max'] = 3 self.store['min'] = 6 self.assertEqual(len(self.store), 2) def test_close(self): self.store.close() self.assertEqual(self.store, None) def test_clear(self): self.store['max'] = 3 self.store['min'] = 6 self.store['pow'] = 7 self.store.clear() self.assertEqual(len(self.store), 0) def test_items(self): self.store['max'] = 3 self.store['min'] = 6 self.store['pow'] = 7 slist = list(self.store.items()) self.assertEqual(('min', 6) in slist, True) def test_iteritems(self): self.store['max'] = 3 self.store['min'] = 6 self.store['pow'] = 7 slist = list(self.store.iteritems()) self.assertEqual(('min', 6) in slist, True) def test_iterkeys(self): self.store['max'] = 3 self.store['min'] = 6 self.store['pow'] = 7 slist = list(self.store.iterkeys()) self.assertEqual('min' in slist, True) def test_itervalues(self): self.store['max'] = 3 self.store['min'] = 6 self.store['pow'] = 7 slist = list(self.store.itervalues()) self.assertEqual(6 in slist, True) def test_pop(self): self.store['max'] = 3 self.store['min'] = 6 item = self.store.pop('min') self.assertEqual(item, 6) def test_popitem(self): self.store['max'] = 3 self.store['min'] = 6 self.store['pow'] = 7 item = self.store.popitem() self.assertEqual(len(item) + len(self.store), 4) def test_setdefault(self): self.store['max'] = 3 self.store['min'] = 6 self.store['pow'] = 7 self.store.setdefault('bow', 8) self.assertEqual(self.store['bow'], 8) def test_update(self): from shove import Shove tstore = Shove() tstore['max'] = 3 tstore['min'] = 6 tstore['pow'] = 7 self.store['max'] = 2 self.store['min'] = 3 self.store['pow'] = 7 self.store.update(tstore) self.assertEqual(self.store['min'], 6) def test_values(self): self.store['max'] = 3 self.store['min'] = 6 self.store['pow'] = 7 slist = self.store.values() self.assertEqual(6 in slist, True) def test_keys(self): self.store['max'] = 3 self.store['min'] = 6 self.store['pow'] = 7 slist = self.store.keys() self.assertEqual('min' in slist, True)
vocabulary_c_dict[category] = words_c if __name__ == '__main__': start_time = time.time() print("Processing articles dataset") process_articles() log.info("Syncing articles dict") articles_dict.sync() log.info("Syncing title dict") titles_dict.sync() log.info("Syncing words dict") words_dict.sync() print("Calculating cf_w") calculate_cf_w() log.info("Closing articles dict") articles_dict.close() log.info("Closing titles dict") titles_dict.close() log.info("Closing words dict") words_dict.close() log.info("Syncing R_w dict") R_w_dict.sync() log.info("Syncing vocabulary_c dict") vocabulary_c_dict.sync() log.info("Closing R_w dict") R_w_dict.close() log.info("Closing vocabulary_c dict") vocabulary_c_dict.close() print("Execution time took %.3f seconds" % (time.time() - start_time))
class TestFtpStore(unittest.TestCase): ftpstring = "put ftp string here" def setUp(self): self.store = Shove(self.ftpstring, compress=True) def tearDown(self): self.store.clear() self.store.close() def test__getitem__(self): self.store["max"] = 3 self.store.sync() self.assertEqual(self.store["max"], 3) def test__setitem__(self): self.store["max"] = 3 self.store.sync() self.assertEqual(self.store["max"], 3) def test__delitem__(self): self.store["max"] = 3 del self.store["max"] self.assertEqual("max" in self.store, False) def test_get(self): self.store["max"] = 3 self.store.sync() self.assertEqual(self.store.get("min"), None) def test__cmp__(self): tstore = Shove() self.store["max"] = 3 tstore["max"] = 3 self.store.sync() tstore.sync() self.assertEqual(self.store, tstore) def test__len__(self): self.store["max"] = 3 self.store["min"] = 6 self.store.sync() self.assertEqual(len(self.store), 2) def test_clear(self): self.store["max"] = 3 self.store["min"] = 6 self.store["pow"] = 7 self.store.sync() self.store.clear() self.assertEqual(len(self.store), 0) def test_items(self): self.store["max"] = 3 self.store["min"] = 6 self.store["pow"] = 7 self.store.sync() slist = list(self.store.items()) self.assertEqual(("min", 6) in slist, True) def test_iteritems(self): self.store["max"] = 3 self.store["min"] = 6 self.store["pow"] = 7 self.store.sync() slist = list(self.store.iteritems()) self.assertEqual(("min", 6) in slist, True) def test_iterkeys(self): self.store["max"] = 3 self.store["min"] = 6 self.store["pow"] = 7 self.store.sync() slist = list(self.store.iterkeys()) self.assertEqual("min" in slist, True) def test_itervalues(self): self.store["max"] = 3 self.store["min"] = 6 self.store["pow"] = 7 self.store.sync() slist = list(self.store.itervalues()) self.assertEqual(6 in slist, True) def test_pop(self): self.store["max"] = 3 self.store["min"] = 6 self.store.sync() item = self.store.pop("min") self.assertEqual(item, 6) def test_popitem(self): self.store["max"] = 3 self.store["min"] = 6 self.store["pow"] = 7 self.store.sync() item = self.store.popitem() self.store.sync() self.assertEqual(len(item) + len(self.store), 4) def test_setdefault(self): self.store["max"] = 3 self.store["min"] = 6 self.store["powl"] = 7 self.store.setdefault("pow", 8) self.store.sync() self.assertEqual(self.store["pow"], 8) def test_update(self): tstore = Shove() tstore["max"] = 3 tstore["min"] = 6 tstore["pow"] = 7 self.store["max"] = 2 self.store["min"] = 3 self.store["pow"] = 7 self.store.sync() self.store.update(tstore) self.store.sync() self.assertEqual(self.store["min"], 6) def test_values(self): self.store["max"] = 3 self.store["min"] = 6 self.store["pow"] = 7 self.store.sync() slist = self.store.values() self.assertEqual(6 in slist, True) def test_keys(self): self.store["max"] = 3 self.store["min"] = 6 self.store["pow"] = 7 self.store.sync() slist = self.store.keys() self.assertEqual("min" in slist, True)
factor_data[f] = data[column_name].reindex( indx, method='ffill') * column_scale if transformation == "rdiff": factor_data[ f] = factor_data[f] / factor_data[f].shift(1) - 1 return factor_data def get_adjusted_close(self, ticker, start_date, end_date): data = self.fetch_data(ticker, start_date, end_date, 11) return data["Adj. Close"] if __name__ == '__main__': #from cache import ShelveCache #cache = ShelveCache(filename="D:\\cache.db", writeback=True) from shove import Shove cache = Shove("zodb:///home/gouthaman/stock.zodb") data = QuandlData(cache=cache) import datetime dd = data.fetch_data("WIKI/WFM", datetime.date(2014, 1, 1), datetime.date(2014, 7, 31), column=11) dd = data.fetch_data("WIKI/WFM", datetime.date(2014, 1, 1), datetime.date(2014, 7, 31), column=11) cache.close()
class TestFileStore(unittest.TestCase): def setUp(self): from shove import Shove self.store = Shove('file://test', compress=True) def tearDown(self): import os self.store.close() for x in os.listdir('test'): os.remove(os.path.join('test', x)) os.rmdir('test') def test__getitem__(self): self.store['max'] = 3 self.store.sync() self.assertEqual(self.store['max'], 3) def test__setitem__(self): self.store['max'] = 3 self.store.sync() self.assertEqual(self.store['max'], 3) def test__delitem__(self): self.store['max'] = 3 del self.store['max'] self.assertEqual('max' in self.store, False) def test_get(self): self.store['max'] = 3 self.store.sync() self.assertEqual(self.store.get('min'), None) def test__cmp__(self): from shove import Shove tstore = Shove() self.store['max'] = 3 tstore['max'] = 3 self.store.sync() tstore.sync() self.assertEqual(self.store, tstore) def test__len__(self): self.store['max'] = 3 self.store['min'] = 6 self.assertEqual(len(self.store), 2) def test_close(self): self.store.close() self.assertEqual(self.store, None) def test_clear(self): self.store['max'] = 3 self.store['min'] = 6 self.store['pow'] = 7 self.store.clear() self.assertEqual(len(self.store), 0) def test_items(self): self.store['max'] = 3 self.store['min'] = 6 self.store['pow'] = 7 slist = list(self.store.items()) self.assertEqual(('min', 6) in slist, True) def test_iteritems(self): self.store['max'] = 3 self.store['min'] = 6 self.store['pow'] = 7 slist = list(self.store.iteritems()) self.assertEqual(('min', 6) in slist, True) def test_iterkeys(self): self.store['max'] = 3 self.store['min'] = 6 self.store['pow'] = 7 slist = list(self.store.iterkeys()) self.assertEqual('min' in slist, True) def test_itervalues(self): self.store['max'] = 3 self.store['min'] = 6 self.store['pow'] = 7 slist = list(self.store.itervalues()) self.assertEqual(6 in slist, True) def test_pop(self): self.store['max'] = 3 self.store['min'] = 6 item = self.store.pop('min') self.assertEqual(item, 6) def test_popitem(self): self.store['max'] = 3 self.store['min'] = 6 self.store['pow'] = 7 item = self.store.popitem() self.assertEqual(len(item) + len(self.store), 4) def test_setdefault(self): self.store['max'] = 3 self.store['min'] = 6 self.store['powl'] = 7 self.store.setdefault('pow', 8) self.assertEqual(self.store['pow'], 8) def test_update(self): from shove import Shove tstore = Shove() tstore['max'] = 3 tstore['min'] = 6 tstore['pow'] = 7 self.store['max'] = 2 self.store['min'] = 3 self.store['pow'] = 7 self.store.update(tstore) self.assertEqual(self.store['min'], 6) def test_values(self): self.store['max'] = 3 self.store['min'] = 6 self.store['pow'] = 7 slist = self.store.values() self.assertEqual(6 in slist, True) def test_keys(self): self.store['max'] = 3 self.store['min'] = 6 self.store['pow'] = 7 slist = self.store.keys() self.assertEqual('min' in slist, True)
class TestSvnStore(unittest.TestCase): svnstring = 'SVN test string here' def setUp(self): from shove import Shove self.store = Shove(self.svnstring, compress=True) def tearDown(self): self.store.clear() self.store.close() def test__getitem__(self): self.store['max'] = 3 self.store.sync() self.assertEqual(self.store['max'], 3) def test__setitem__(self): self.store['max'] = 3 self.store.sync() self.assertEqual(self.store['max'], 3) def test__delitem__(self): self.store['max'] = 3 del self.store['max'] self.assertEqual('max' in self.store, False) def test_get(self): self.store['max'] = 3 self.store.sync() self.assertEqual(self.store.get('min'), None) def test__cmp__(self): from shove import Shove tstore = Shove() self.store['max'] = 3 tstore['max'] = 3 self.store.sync() tstore.sync() self.assertEqual(self.store, tstore) def test__len__(self): self.store['max'] = 3 self.store['min'] = 6 self.store.sync() self.assertEqual(len(self.store), 2) def test_clear(self): self.store['max'] = 3 self.store['min'] = 6 self.store['pow'] = 7 self.store.sync() self.store.clear() self.assertEqual(len(self.store), 0) def test_items(self): self.store['max'] = 3 self.store['min'] = 6 self.store['pow'] = 7 self.store.sync() slist = list(self.store.items()) self.assertEqual(('min', 6) in slist, True) def test_iteritems(self): self.store['max'] = 3 self.store['min'] = 6 self.store['pow'] = 7 self.store.sync() slist = list(self.store.iteritems()) self.assertEqual(('min', 6) in slist, True) def test_iterkeys(self): self.store['max'] = 3 self.store['min'] = 6 self.store['pow'] = 7 self.store.sync() slist = list(self.store.iterkeys()) self.assertEqual('min' in slist, True) def test_itervalues(self): self.store['max'] = 3 self.store['min'] = 6 self.store['pow'] = 7 self.store.sync() slist = list(self.store.itervalues()) self.assertEqual(6 in slist, True) def test_pop(self): self.store['max'] = 3 self.store['min'] = 6 self.store.sync() item = self.store.pop('min') self.assertEqual(item, 6) def test_popitem(self): self.store['max'] = 3 self.store['min'] = 6 self.store['pow'] = 7 self.store.sync() item = self.store.popitem() self.store.sync() self.assertEqual(len(item) + len(self.store), 4) def test_setdefault(self): self.store['max'] = 3 self.store['min'] = 6 self.store['powl'] = 7 self.store.setdefault('pow', 8) self.store.sync() self.assertEqual(self.store['pow'], 8) def test_update(self): from shove import Shove tstore = Shove() tstore['max'] = 3 tstore['min'] = 6 tstore['pow'] = 7 self.store['max'] = 2 self.store['min'] = 3 self.store['pow'] = 7 self.store.sync() self.store.update(tstore) self.store.sync() self.assertEqual(self.store['min'], 6) def test_values(self): self.store['max'] = 3 self.store['min'] = 6 self.store['pow'] = 7 self.store.sync() slist = self.store.values() self.assertEqual(6 in slist, True) def test_keys(self): self.store['max'] = 3 self.store['min'] = 6 self.store['pow'] = 7 self.store.sync() slist = self.store.keys() self.assertEqual('min' in slist, True)
class TestHDF5Store(unittest2.TestCase): def setUp(self): from shove import Shove self.store = Shove('hdf5://test.hdf5/test') def tearDown(self): import os self.store.close() try: os.remove('test.hdf5') except OSError: pass def test__getitem__(self): self.store['max'] = 3 self.assertEqual(self.store['max'], 3) def test__setitem__(self): self.store['max'] = 3 self.assertEqual(self.store['max'], 3) def test__delitem__(self): self.store['max'] = 3 del self.store['max'] self.assertEqual('max' in self.store, False) def test_get(self): self.store['max'] = 3 self.assertEqual(self.store.get('min'), None) def test__cmp__(self): from shove import Shove tstore = Shove() self.store['max'] = 3 tstore['max'] = 3 self.assertEqual(self.store, tstore) def test__len__(self): self.store['max'] = 3 self.store['min'] = 6 self.assertEqual(len(self.store), 2) def test_close(self): self.store.close() self.assertEqual(self.store, None) def test_clear(self): self.store['max'] = 3 self.store['min'] = 6 self.store['pow'] = 7 self.store.clear() self.assertEqual(len(self.store), 0) def test_items(self): self.store['max'] = 3 self.store['min'] = 6 self.store['pow'] = 7 slist = list(self.store.items()) self.assertEqual(('min', 6) in slist, True) def test_iteritems(self): self.store['max'] = 3 self.store['min'] = 6 self.store['pow'] = 7 slist = list(self.store.iteritems()) self.assertEqual(('min', 6) in slist, True) def test_iterkeys(self): self.store['max'] = 3 self.store['min'] = 6 self.store['pow'] = 7 slist = list(self.store.iterkeys()) self.assertEqual('min' in slist, True) def test_itervalues(self): self.store['max'] = 3 self.store['min'] = 6 self.store['pow'] = 7 slist = list(self.store.itervalues()) self.assertEqual(6 in slist, True) def test_pop(self): self.store['max'] = 3 self.store['min'] = 6 item = self.store.pop('min') self.assertEqual(item, 6) def test_popitem(self): self.store['max'] = 3 self.store['min'] = 6 self.store['pow'] = 7 item = self.store.popitem() self.assertEqual(len(item) + len(self.store), 4) def test_setdefault(self): self.store['max'] = 3 self.store['min'] = 6 self.store['pow'] = 7 self.store.setdefault('bow', 8) self.assertEqual(self.store['bow'], 8) def test_update(self): from shove import Shove tstore = Shove() tstore['max'] = 3 tstore['min'] = 6 tstore['pow'] = 7 self.store['max'] = 2 self.store['min'] = 3 self.store['pow'] = 7 self.store.update(tstore) self.assertEqual(self.store['min'], 6) def test_values(self): self.store['max'] = 3 self.store['min'] = 6 self.store['pow'] = 7 slist = self.store.values() self.assertEqual(6 in slist, True) def test_keys(self): self.store['max'] = 3 self.store['min'] = 6 self.store['pow'] = 7 slist = self.store.keys() self.assertEqual('min' in slist, True)
class TestFileStore(unittest.TestCase): def setUp(self): self.store = Shove('file://test', compress=True) def tearDown(self): self.store.close() for x in os.listdir('test'): os.remove(os.path.join('test', x)) os.rmdir('test') def test__getitem__(self): self.store['max'] = 3 self.store.sync() self.assertEqual(self.store['max'], 3) def test__setitem__(self): self.store['max'] = 3 self.store.sync() self.assertEqual(self.store['max'], 3) def test__delitem__(self): self.store['max'] = 3 del self.store['max'] self.assertEqual('max' in self.store, False) def test_get(self): self.store['max'] = 3 self.store.sync() self.assertEqual(self.store.get('min'), None) def test__cmp__(self): tstore = Shove() self.store['max'] = 3 tstore['max'] = 3 self.store.sync() tstore.sync() self.assertEqual(self.store, tstore) def test__len__(self): self.store['max'] = 3 self.store['min'] = 6 self.assertEqual(len(self.store), 2) def test_close(self): self.store.close() self.assertEqual(self.store, None) def test_clear(self): self.store['max'] = 3 self.store['min'] = 6 self.store['pow'] = 7 self.store.clear() self.assertEqual(len(self.store), 0) def test_items(self): self.store['max'] = 3 self.store['min'] = 6 self.store['pow'] = 7 slist = list(self.store.items()) self.assertEqual(('min', 6) in slist, True) def test_iteritems(self): self.store['max'] = 3 self.store['min'] = 6 self.store['pow'] = 7 slist = list(self.store.iteritems()) self.assertEqual(('min', 6) in slist, True) def test_iterkeys(self): self.store['max'] = 3 self.store['min'] = 6 self.store['pow'] = 7 slist = list(self.store.iterkeys()) self.assertEqual('min' in slist, True) def test_itervalues(self): self.store['max'] = 3 self.store['min'] = 6 self.store['pow'] = 7 slist = list(self.store.itervalues()) self.assertEqual(6 in slist, True) def test_pop(self): self.store['max'] = 3 self.store['min'] = 6 item = self.store.pop('min') self.assertEqual(item, 6) def test_popitem(self): self.store['max'] = 3 self.store['min'] = 6 self.store['pow'] = 7 item = self.store.popitem() self.assertEqual(len(item) + len(self.store), 4) def test_setdefault(self): self.store['max'] = 3 self.store['min'] = 6 self.store['powl'] = 7 self.store.setdefault('pow', 8) self.assertEqual(self.store['pow'], 8) def test_update(self): tstore = Shove() tstore['max'] = 3 tstore['min'] = 6 tstore['pow'] = 7 self.store['max'] = 2 self.store['min'] = 3 self.store['pow'] = 7 self.store.update(tstore) self.assertEqual(self.store['min'], 6) def test_values(self): self.store['max'] = 3 self.store['min'] = 6 self.store['pow'] = 7 slist = self.store.values() self.assertEqual(6 in slist, True) def test_keys(self): self.store['max'] = 3 self.store['min'] = 6 self.store['pow'] = 7 slist = self.store.keys() self.assertEqual('min' in slist, True)
class Store(object): def setUp(self): from shove import Shove self.store = Shove( self.initstring, sync=0 ) def tearDown(self): self.store.close() def test__getitem__(self): self.store['max'] = 3 self.store.sync() self.assertEqual(self.store['max'], 3) def test__setitem__(self): self.store['max'] = 3 self.store['d'] = {'A': 1}, {'A': 1} self.store['d'] = {'AA': 1}, {'A': 1} self.store['d'] = {'AA': 1}, {'AA': 1} self.store.sync() self.assertEqual(self.store['max'], 3) def test__delitem__(self): self.store['max'] = 3 self.store.sync() del self.store['max'] self.store.sync() self.assertEqual('max' in self.store, False) def test_get(self): self.store['max'] = 3 self.store.sync() self.assertEqual(self.store.get('min'), None) def test__cmp__(self): from shove import Shove tstore = Shove() self.store['max'] = 3 self.store.sync() tstore['max'] = 3 self.assertEqual(self.store, tstore) def test__len__(self): self.store['max'] = 3 self.store['min'] = 6 self.store.sync() self.assertEqual(len(self.store), 2) def test_items(self): self.store['max'] = 3 self.store['min'] = 6 self.store['pow'] = 7 self.store.sync() slist = list(items(self.store)) self.assertEqual(('min', 6) in slist, True) def test_keys(self): self.store['max'] = 3 self.store['min'] = 6 self.store['pow'] = 7 self.store.sync() slist = list(keys(self.store)) self.assertEqual('min' in slist, True) def test_values(self): self.store['max'] = 3 self.store['min'] = 6 self.store['pow'] = 7 self.store.sync() slist = list(values(self.store)) self.assertEqual(6 in slist, True) def test_pop(self): self.store['max'] = 3 self.store['min'] = 6 self.store.sync() item = self.store.pop('min') self.store.sync() self.assertEqual(item, 6) def test_setdefault(self): self.store['max'] = 3 self.store['min'] = 6 self.store.sync() self.assertEqual(self.store.setdefault('pow', 8), 8) self.store.sync() self.assertEqual(self.store['pow'], 8) def test_update(self): from shove import Shove tstore = Shove() tstore['max'] = 3 tstore['min'] = 6 tstore['pow'] = 7 self.store['max'] = 2 self.store['min'] = 3 self.store['pow'] = 7 self.store.update(tstore) self.store.sync() self.assertEqual(self.store['min'], 6) def test_clear(self): self.store['max'] = 3 self.store['min'] = 6 self.store['pow'] = 7 self.store.sync() self.store.clear() self.store.sync() self.assertEqual(len(self.store), 0) def test_popitem(self): self.store['max'] = 3 self.store['min'] = 6 self.store['pow'] = 7 self.store.sync() item = self.store.popitem() self.store.sync() self.assertEqual(len(item) + len(self.store), 4) def test_close(self): self.store.close() self.assertEqual(self.store._store, None) self.assertEqual(self.store._buffer, None) self.assertEqual(self.store._cache, None)
factor_data = {} indx = pd.DatetimeIndex(start=start_date, end=end_date, freq='M') for qkey,factors in quandl_keys.items(): data = self.fetch_data(qkey,start_date, end_date) for f in factors: column_name , column_scale, transformation = self.data_items[qkey][0][f] factor_data[f] = data[column_name].reindex(indx, method='ffill')*column_scale if transformation=="rdiff": factor_data[f] = factor_data[f]/factor_data[f].shift(1)-1 return factor_data def get_adjusted_close(self, ticker, start_date, end_date): data = self.fetch_data(ticker, start_date, end_date,11) return data["Adj. Close"] if __name__ == '__main__': #from cache import ShelveCache #cache = ShelveCache(filename="D:\\cache.db", writeback=True) from shove import Shove cache = Shove("zodb:///home/gouthaman/stock.zodb") data = QuandlData(cache=cache) import datetime dd = data.fetch_data("WIKI/WFM", datetime.date(2014,1,1) , datetime.date(2014,7,31),column=11) dd = data.fetch_data("WIKI/WFM", datetime.date(2014,1,1) , datetime.date(2014,7,31),column=11) cache.close()
class TestZodbStore(unittest.TestCase): init = 'zodb://test.db' def setUp(self): from shove import Shove self.store = Shove(self.init, compress=True) def tearDown(self): self.store.close() import os os.remove('test.db') os.remove('test.db.index') os.remove('test.db.tmp') os.remove('test.db.lock') def test__getitem__(self): self.store['max'] = 3 self.assertEqual(self.store['max'], 3) def test__setitem__(self): self.store['max'] = 3 self.assertEqual(self.store['max'], 3) def test__delitem__(self): self.store['max'] = 3 del self.store['max'] self.assertEqual('max' in self.store, False) def test_get(self): self.store['max'] = 3 self.assertEqual(self.store.get('min'), None) def test__cmp__(self): from shove import Shove tstore = Shove() self.store['max'] = 3 tstore['max'] = 3 self.assertEqual(self.store, tstore) def test__len__(self): self.store['max'] = 3 self.store['min'] = 6 self.assertEqual(len(self.store), 2) def test_close(self): self.store.close() self.assertEqual(self.store, None) def test_clear(self): self.store['max'] = 3 self.store['min'] = 6 self.store['pow'] = 7 self.store.clear() self.assertEqual(len(self.store), 0) def test_items(self): self.store['max'] = 3 self.store['min'] = 6 self.store['pow'] = 7 slist = list(self.store.items()) self.assertEqual(('min', 6) in slist, True) def test_iteritems(self): self.store['max'] = 3 self.store['min'] = 6 self.store['pow'] = 7 slist = list(self.store.iteritems()) self.assertEqual(('min', 6) in slist, True) def test_iterkeys(self): self.store['max'] = 3 self.store['min'] = 6 self.store['pow'] = 7 slist = list(self.store.iterkeys()) self.assertEqual('min' in slist, True) def test_itervalues(self): self.store['max'] = 3 self.store['min'] = 6 self.store['pow'] = 7 slist = list(self.store.itervalues()) self.assertEqual(6 in slist, True) def test_pop(self): self.store['max'] = 3 self.store['min'] = 6 item = self.store.pop('min') self.assertEqual(item, 6) def test_popitem(self): self.store['max'] = 3 self.store['min'] = 6 self.store['pow'] = 7 item = self.store.popitem() self.assertEqual(len(item) + len(self.store), 4) def test_setdefault(self): self.store['max'] = 3 self.store['min'] = 6 self.store['powl'] = 7 self.store.setdefault('pow', 8) self.assertEqual(self.store['pow'], 8) def test_update(self): from shove import Shove tstore = Shove() tstore['max'] = 3 tstore['min'] = 6 tstore['pow'] = 7 self.store['max'] = 2 self.store['min'] = 3 self.store['pow'] = 7 self.store.update(tstore) self.assertEqual(self.store['min'], 6) def test_values(self): self.store['max'] = 3 self.store['min'] = 6 self.store['pow'] = 7 slist = self.store.values() self.assertEqual(6 in slist, True) def test_keys(self): self.store['max'] = 3 self.store['min'] = 6 self.store['pow'] = 7 slist = self.store.keys() self.assertEqual('min' in slist, True)
class DataManager(): def __init__(self, filename, connection_details=VERTICA_CONNECTION): self.filename = filename self.store = Shove('file://' + self.filename, 'memory://', optimize=False) self.connection_details = connection_details @staticmethod def get_attribute_name(data_set, attribute): return '_' + data_set + '_' + attribute def get_data_set_attribute(self, data_set, key): attrib_name = self.get_attribute_name(data_set, key) if self.has_attribute(data_set, key): return self.store[attrib_name] else: return None self.store.close() gc.collect() self.store = Shove('file://' + self.filename, 'memory://', optimize=False) def set_data_set_attribute(self, data_set, key, value): attrib_name = self.get_attribute_name(data_set, key) self.store.update({attrib_name: value}) self.store.sync() self.store.close() gc.collect() self.store = Shove('file://' + self.filename, 'memory://', optimize=False) def has_attribute(self, data_set, key): attrib_name = self.get_attribute_name(data_set, key) if attrib_name in self.store.keys(): return True else: return False def remove_attribute(self, data_set, key): attrib_name = self.get_attribute_name(data_set, key) if attrib_name in self.store: del self.store[attrib_name] self.store.sync() def get_data_set_list(self): data_sets = [ key.split('_')[1] for key in self.store.keys() if '_query' in key ] return data_sets def fetch_from_vertica_to_df(self, data_set, query): data_set_query = self.get_data_set_attribute(data_set, 'query') if data_set_query != query: connection = connect(self.connection_details) cursor = connection.cursor() print 'Executing ', data_set, 'Query...' print query columns = get_column_names_from_sql_query(query) cursor.execute(query) data = [] while True: rows = cursor.fetchmany(10000) data.extend([[str(ele) for ele in row] for row in rows]) if len(rows) <= 1: break df = MetadataDataFrame(data=data, columns=columns, meta_info={ 'query': query, 'built_features': [], 'aggregate_values': {}, 'columns': columns }) cursor.close() if len(df) == 0: raise (ValueError('SQL result in empty fetch!!')) else: self.set_data_set_attribute(data_set, 'data', df) self.set_data_set_attribute(data_set, 'query', query) self.set_data_set_attribute(data_set, 'columns', columns) self.set_data_set_attribute(data_set, 'built_features', [])
class TestCassandraStore(unittest.TestCase): def setUp(self): from shove import Shove from pycassa.system_manager import SystemManager system_manager = SystemManager('localhost:9160') try: system_manager.create_column_family('Foo', 'shove') except: pass self.store = Shove('cassandra://localhost:9160/Foo/shove') def tearDown(self): self.store.clear() self.store.close() from pycassa.system_manager import SystemManager system_manager = SystemManager('localhost:9160') system_manager.drop_column_family('Foo', 'shove') def test__getitem__(self): self.store['max'] = 3 self.assertEqual(self.store['max'], 3) def test__setitem__(self): self.store['max'] = 3 self.assertEqual(self.store['max'], 3) def test__delitem__(self): self.store['max'] = 3 del self.store['max'] self.assertEqual('max' in self.store, False) def test_get(self): self.store['max'] = 3 self.assertEqual(self.store.get('min'), None) def test__cmp__(self): from shove import Shove tstore = Shove() self.store['max'] = 3 tstore['max'] = 3 self.assertEqual(self.store, tstore) def test__len__(self): self.store['max'] = 3 self.store['min'] = 6 self.assertEqual(len(self.store), 2) # def test_clear(self): # self.store['max'] = 3 # self.store['min'] = 6 # self.store['pow'] = 7 # self.store.clear() # self.assertEqual(len(self.store), 0) def test_items(self): self.store['max'] = 3 self.store['min'] = 6 self.store['pow'] = 7 slist = list(self.store.items()) self.assertEqual(('min', 6) in slist, True) def test_iteritems(self): self.store['max'] = 3 self.store['min'] = 6 self.store['pow'] = 7 slist = list(self.store.iteritems()) self.assertEqual(('min', 6) in slist, True) def test_iterkeys(self): self.store['max'] = 3 self.store['min'] = 6 self.store['pow'] = 7 slist = list(self.store.iterkeys()) self.assertEqual('min' in slist, True) def test_itervalues(self): self.store['max'] = 3 self.store['min'] = 6 self.store['pow'] = 7 slist = list(self.store.itervalues()) self.assertEqual(6 in slist, True) def test_pop(self): self.store['max'] = 3 self.store['min'] = 6 item = self.store.pop('min') self.assertEqual(item, 6) # def test_popitem(self): # self.store['max'] = 3 # self.store['min'] = 6 # self.store['pow'] = 7 # item = self.store.popitem() # self.assertEqual(len(item) + len(self.store), 4) def test_setdefault(self): self.store['max'] = 3 self.store['min'] = 6 # self.store['pow'] = 7 self.store.setdefault('pow', 8) self.assertEqual(self.store.setdefault('pow', 8), 8) self.assertEqual(self.store['pow'], 8) def test_update(self): from shove import Shove tstore = Shove() tstore['max'] = 3 tstore['min'] = 6 tstore['pow'] = 7 self.store['max'] = 2 self.store['min'] = 3 self.store['pow'] = 7 self.store.update(tstore) self.assertEqual(self.store['min'], 6) def test_values(self): self.store['max'] = 3 self.store['min'] = 6 self.store['pow'] = 7 slist = self.store.values() self.assertEqual(6 in slist, True) def test_keys(self): self.store['max'] = 3 self.store['min'] = 6 self.store['pow'] = 7 slist = self.store.keys() self.assertEqual('min' in slist, True)
class Personis_server: def __init__(self, modeldir=None, adminsfile=None, oauthconfig=None): self.modeldir = modeldir self.admins = yaml.load(file(adminsfile,'r')) self.oauth_clients = Shove('sqlite:///oauth_clients.dat') self.users = Shove('sqlite:///oauth_users.dat') self.bearers = Shove('sqlite:///oauth_bearers.dat') self.oauthconf = yaml.load(file(oauthconfig,'r')) def stopper(): print 'saving persistant data' self.oauth_clients.close() self.users.close() self.bearers.close() cherrypy.engine.subscribe('stop', stopper) @cherrypy.expose def list_clients(self): if cherrypy.session.get('user') == None: cherrypy.session['target_url'] = '/list_clients' raise cherrypy.HTTPRedirect('/login') if not cherrypy.session.get('user')['id'] in self.admins.keys(): raise cherrypy.HTTPError() base_path = os.path.dirname(os.path.abspath(__file__)) loader = TemplateLoader([base_path]) tmpl = loader.load('list_clients.html') for k, v in self.oauth_clients.items(): print k, v.friendly_name stream = tmpl.generate(clients=self.oauth_clients.values()) return stream.render('xhtml') @cherrypy.expose def list_clients_save(self, id, value, _method='get'): if cherrypy.session.get('user') == None: raise cherrypy.HTTPError() if not cherrypy.session.get('user')['id'] in self.admins.keys(): raise cherrypy.HTTPError() if id == "removeOneForMe": del(self.oauth_clients[value]) self.oauth_clients.sync() print "removed a client" raise cherrypy.HTTPRedirect('/list_clients') if id == "addOneForMe": clid = '' secret = '' for i in range(10): clid = clid + str(int(random.random()*100)) secret = secret + str(int(random.random()*100)) self.oauth_clients[clid] = oauth_client( client_id = clid, friendly_name= 'my client', secret= secret, redirect_uri='http://www.example.com/', icon='/static/images/icon.svg') self.oauth_clients.sync() print "added a client" raise cherrypy.HTTPRedirect('/list_clients') clid, field = id.split('|') print 'saving: ',clid, field, value oldc = self.oauth_clients[clid] oldc.__dict__[field] = value self.oauth_clients[clid] = oldc for k, v in self.oauth_clients.items(): print k, v.friendly_name self.oauth_clients.sync() return value @cherrypy.expose def authorize(self, client_id, redirect_uri, scope, access_type, response_type='code', approval_prompt='auto', state=None): body = cherrypy.request.body.fp.read() rurl = cherrypy.request.base+cherrypy.request.path_info cherrypy.session['client_id'] = client_id cli = self.oauth_clients[client_id] if state <> None: cherrypy.session['state'] = state if cli.redirect_uri <> redirect_uri: raise cherrypy.HTTPError() raise cherrypy.HTTPRedirect('/login') @cherrypy.expose def login(self): flow = OAuth2WebServerFlow(client_id=self.oauthconf['personis_client_id'], client_secret=self.oauthconf['personis_client_secret'], scope='https://www.googleapis.com/auth/userinfo.profile https://www.googleapis.com/auth/userinfo.email', user_agent='personis-server/1.0') callback = callback = cherrypy.request.base + '/logged_in' authorize_url = flow.step1_get_authorize_url(callback) cherrypy.session['flow'] = flow raise cherrypy.HTTPRedirect(authorize_url) @cherrypy.expose def logged_in(self, code): flow = cherrypy.session.get('flow') if not flow: raise IOError() credentials = flow.step2_exchange(cherrypy.request.params) http = httplib2.Http() http = credentials.authorize(http) cjson = credentials.to_json() cjson = json.loads(cjson) content = http.request('https://www.googleapis.com/oauth2/v1/userinfo?access_token='+cjson['access_token']) #print 'content', content usr = json.loads(content[1]) cherrypy.session['user'] = usr self.users[usr['id']] = [usr, credentials] self.users.sync() if not 'client_id' in cherrypy.session: raise cherrypy.HTTPRedirect(cherrypy.session['target_url']) cli = self.oauth_clients[cherrypy.session['client_id']] print 'loggedin session id',cherrypy.session.id # if no model for user, create one. if not os.path.exists(os.path.join(self.modeldir,usr['id'])): fn = os.path.join('tmp',usr['id']) #f1 = open('Modeldefs/user.template','r').read() #f = open(fn, 'w') #f.write(f1%(usr['given_name'],usr['family_name'], usr['gender'], usr['email'])) #f.close() mkmodel(model=usr['id'], mfile='Modeldefs/user.prod', modeldir=self.modeldir, user=usr['id'], password='') um = Personis_a.Access(model=usr['id'], modeldir=self.modeldir, user=usr['id'], password='') ev = Personis_base.Evidence(source="Create_Model", evidence_type="explicit", value=usr['given_name']) um.tell(context=["Personal"], componentid='firstname', evidence=ev) ev = Personis_base.Evidence(source="Create_Model", evidence_type="explicit", value=usr['family_name']) um.tell(context=["Personal"], componentid='lastname', evidence=ev) ev = Personis_base.Evidence(source="Create_Model", evidence_type="explicit", value=usr['gender']) um.tell(context=["Personal"], componentid='gender', evidence=ev) ev = Personis_base.Evidence(source="Create_Model", evidence_type="explicit", value=usr['email']) um.tell(context=["Personal"], componentid='email', evidence=ev) reslist = um.ask(context=["Personal"], view=['firstname','email']) Personis_util.printcomplist(reslist) # if it's mneme, then don't ask whether it's all ok. just do it. #if cli.secret == 'personis_client_secret_mneme': #raise cherrypy.HTTPRedirect('/allow') #otherwise, ask yes/no base_path = os.path.dirname(os.path.abspath(__file__)) loader = TemplateLoader([base_path]) tmpl = loader.load('appQuery.html') stream = tmpl.generate(name=usr['given_name'], app=cli.friendly_name, icon=cli.icon) return stream.render('xhtml') @cherrypy.expose def allow(self): usr = cherrypy.session.get('user') cli = self.oauth_clients[cherrypy.session['client_id']] val_key = '' for i in range(10): val_key = val_key+ str(int(random.random()*100)) rdi = cli.redirect_uri + '?' if 'state' in cherrypy.session: rdi = rdi + 'state='+cherrypy.session['state']+'&' rdi = rdi + 'code=' + val_key cherrypy.session['auth_code'] = val_key cli.auth_codes[val_key] = [time.time(),usr['id']] cli = self.oauth_clients[cherrypy.session.get('client_id')] redr = cli.redirect_uri um = Personis_a.Access(model=usr['id'], modeldir=self.modeldir, user=usr['id'], password='') cherrypy.session['um'] = um result = um.registerapp(app=cherrypy.session['client_id'], desc='', password='') raise cherrypy.HTTPRedirect(redr+'?code='+cherrypy.session.get('auth_code')) @cherrypy.expose def dissallow(self): cli = self.oauth_clients[cherrypy.session.get('client_id')] redr = cli.redirect_uri+'?error=access_denied' #print redr raise cherrypy.HTTPRedirect(redr) @cherrypy.expose def request_token(self, code, redirect_uri, client_id, client_secret, scope, grant_type): print code, client_id, client_secret cli = self.oauth_clients[client_id] if not code in cli.auth_codes: raise cherrypy.HTTPError() atoken = '' rtoken = '' for i in range(10): atoken = atoken + str(int(random.random()*100)) rtoken = rtoken + str(int(random.random()*100)) user = self.users[cli.auth_codes[code][1]][0] b = {} b['user'] = user b['credentials'] = self.users[cli.auth_codes[code][1]][1] print b self.bearers[atoken] = b #print user cli.request_tokens[atoken] = user['id'] cli.refresh_tokens[rtoken] = '' s = ''' { "access_token":"%s", "token_type":"bearer", "expires_in":3600, "refresh_token":"%s", "example_parameter":"example_value" } '''%(atoken,rtoken) del(self.users[cli.auth_codes[code][1]]) del(cli.auth_codes[code]) self.oauth_clients[client_id] = cli print s return s @cherrypy.expose def default(self, *args): print 'path', cherrypy.request.path_info print 'headers', cherrypy.request.headers jsonobj = cherrypy.request.body.fp.read() print 'body', jsonobj try: access_token = cherrypy.request.headers['Authorization'].split()[1] print self.bearers usr = self.bearers[access_token]['user'] except: raise cherrypy.HTTPError() try: pargs = json.loads(jsonobj) except: print "bad request - cannot decode json - possible access from web browser" return json.dumps("Personis User Model server. Not accessible using a web browser.") # dirty kludge to get around unicode for k,v in pargs.items(): if type(v) == type(u''): pargs[k] = str(v) if type(k) == type(u''): del pargs[k] pargs[str(k)] = v try: result = False if args[0] == 'mkmodel': # fixme need to implement security # and error handling Personis_base.MkModel(model=usr['id'], modeldir=self.modeldir, \ user=usr['id'], password='', description=pargs['description']) result = True else: um = Personis_a.Access(model=usr['id'], modeldir=self.modeldir, user=usr['id'], password='') if args[0] == 'access': result = True elif args[0] == 'tell': result = um.tell(context=pargs['context'], componentid=pargs['componentid'], evidence=Personis_base.Evidence(**pargs['evidence'])) elif args[0] == 'ask': reslist = um.ask(context=pargs['context'], view=pargs['view'], resolver=pargs['resolver'], \ showcontexts=pargs['showcontexts']) if pargs['showcontexts']: cobjlist, contexts, theviews, thesubs = reslist cobjlist = [c.__dict__ for c in cobjlist] for c in cobjlist: if c["evidencelist"]: c["evidencelist"] = [e for e in c["evidencelist"]] newviews = {} if theviews != None: for vname,v in theviews.items(): newviews[vname] = v.__dict__ else: newviews = None reslist = [cobjlist, contexts, newviews, thesubs] else: reslist = [c.__dict__ for c in reslist] for c in reslist: if c["evidencelist"]: c["evidencelist"] = [e for e in c["evidencelist"]] result = reslist elif args[0] == 'subscribe': result = um.subscribe(context=pargs['context'], view=pargs['view'], subscription=pargs['subscription']) elif args[0] == 'delete_sub': result = um.delete_sub(context=pargs['context'], componentid=pargs['componentid'], subname=pargs['subname']) elif args[0] == 'list_subs': result = um.list_subs(context=pargs['context'], componentid=pargs['componentid']) elif args[0] == 'export_model': result = um.export_model(context=pargs['context'], resolver=pargs['resolver']) elif args[0] == 'import_model': result = um.import_model(context=pargs['context'], partial_model=pargs['partial_model']) elif args[0] == 'set_goals': result = um.set_goals(context=pargs['context'], componentid=pargs['componentid'], goals=pargs['goals']) elif args[0] == 'registerapp': result = um.registerapp(app=pargs['app'], desc=pargs['description'], password=pargs['apppassword']) elif args[0] == 'deleteapp': result = um.deleteapp(app=pargs['app']) elif args[0] == 'getpermission': result = um.getpermission(context=pargs['context'], componentid=pargs['componentid'], app=pargs['app']) elif args[0] == 'setpermission': result = um.setpermission(context=pargs['context'], componentid=pargs['componentid'], app=pargs['app'], permissions=pargs['permissions']) elif args[0] == 'listapps': result = um.listapps() elif args[0] == 'mkcomponent': comp = Personis_base.Component(**pargs["componentobj"]) result = um.mkcomponent(pargs["context"], comp) elif args[0] == 'delcomponent': result = um.delcomponent(pargs["context"], pargs["componentid"]) elif args[0] == 'delcontext': result = um.delcontext(pargs["context"]) elif args[0] == 'setresolver': result = um.setresolver(pargs["context"], pargs["componentid"], pargs["resolver"]) elif args[0] == 'getresolvers': result = um.getresolvers() elif args[0] == 'mkview': viewobj = Personis_base.View(**pargs["viewobj"]) result = um.mkview(pargs["context"], viewobj) elif args[0] == 'delview': result = um.delview(pargs["context"], pargs["viewid"]) elif args[0] == 'mkcontext': contextobj = Personis_base.Context(**pargs["contextobj"]) result = um.mkcontext(pargs["context"], contextobj) elif args[0] == 'getcontext': result = um.getcontext(pargs["context"], pargs["getsize"]) # Repackage result code with error values IF there is a version string. if pargs.has_key("version"): new_result = {} new_result["result"] = "ok" new_result["val"] = result result = new_result except Exception, e: print "Exception:", e traceback.print_exc() if pargs.has_key("version"): new_result = {} #new_result["errorType"] = e.__class__.__name__ #new_result["errorData"] = e.__dict__.copy() #new_result["pythonPickel"] = cPickle.dumps(e) new_result["val"] = [e.__class__.__name__, e.__dict__.copy(), cPickle.dumps(e)] new_result["result"] = "error" result = new_result else: result = False return json.dumps(result)
class Store(object): def setUp(self): from shove import Shove self.store = Shove( self.initstring, optimize=False, compress=True, sync=0, ) def tearDown(self): self.store.close() def test__getitem__(self): self.store['max'] = 3 self.store.sync() self.assertEqual(self.store['max'], 3) def test__setitem__(self): self.store['max'] = 3 self.store['d'] = {'A': 1}, {'A': 1} self.store['d'] = {'AA': 1}, {'A': 1} self.store['d'] = {'AA': 1}, {'AA': 1} self.store.sync() self.assertEqual(self.store['max'], 3) def test__delitem__(self): self.store['max'] = 3 self.store.sync() del self.store['max'] self.store.sync() self.assertEqual('max' in self.store, False) def test_get(self): self.store['max'] = 3 self.store.sync() self.assertEqual(self.store.get('min'), None) def test__cmp__(self): from shove import Shove tstore = Shove() self.store['max'] = 3 self.store.sync() tstore['max'] = 3 self.assertEqual(self.store, tstore) def test__len__(self): self.store['max'] = 3 self.store['min'] = 6 self.store.sync() self.assertEqual(len(self.store), 2) def test_items(self): self.store['max'] = 3 self.store['min'] = 6 self.store['pow'] = 7 self.store.sync() slist = list(items(self.store)) self.assertEqual(('min', 6) in slist, True) def test_keys(self): self.store['max'] = 3 self.store['min'] = 6 self.store['pow'] = 7 self.store.sync() slist = list(keys(self.store)) self.assertEqual('min' in slist, True) def test_values(self): self.store['max'] = 3 self.store['min'] = 6 self.store['pow'] = 7 self.store.sync() slist = list(values(self.store)) self.assertEqual(6 in slist, True) def test_pop(self): self.store['max'] = 3 self.store['min'] = 6 self.store.sync() item = self.store.pop('min') self.store.sync() self.assertEqual(item, 6) def test_setdefault(self): self.store['max'] = 3 self.store['min'] = 6 self.store.sync() self.assertEqual(self.store.setdefault('pow', 8), 8) self.store.sync() self.assertEqual(self.store['pow'], 8) def test_update(self): from shove import Shove tstore = Shove() tstore['max'] = 3 tstore['min'] = 6 tstore['pow'] = 7 self.store['max'] = 2 self.store['min'] = 3 self.store['pow'] = 7 self.store.update(tstore) self.store.sync() self.assertEqual(self.store['min'], 6) def test_clear(self): self.store['max'] = 3 self.store['min'] = 6 self.store['pow'] = 7 self.store.sync() self.store.clear() self.store.sync() self.assertEqual(len(self.store), 0) def test_popitem(self): self.store['max'] = 3 self.store['min'] = 6 self.store['pow'] = 7 self.store.sync() item = self.store.popitem() self.store.sync() self.assertEqual(len(item) + len(self.store), 4) def test_close(self): self.store.close() self.assertEqual(self.store._store, None) self.assertEqual(self.store._buffer, None) self.assertEqual(self.store._cache, None)
class TestRedisStore(unittest.TestCase): def setUp(self): from shove import Shove self.store = Shove('redis://localhost:6379/0') def tearDown(self): self.store.clear() self.store.close() def test__getitem__(self): self.store['max'] = 3 self.assertEqual(self.store['max'], 3) def test__setitem__(self): self.store['max'] = 3 self.assertEqual(self.store['max'], 3) def test__delitem__(self): self.store['max'] = 3 del self.store['max'] self.assertEqual('max' in self.store, False) def test_get(self): self.store['max'] = 3 self.assertEqual(self.store.get('min'), None) def test__cmp__(self): from shove import Shove tstore = Shove() self.store['max'] = 3 tstore['max'] = 3 self.assertEqual(self.store, tstore) def test__len__(self): self.store['max'] = 3 self.store['min'] = 6 self.assertEqual(len(self.store), 2) def test_clear(self): self.store['max'] = 3 self.store['min'] = 6 self.store['pow'] = 7 self.store.clear() self.assertEqual(len(self.store), 0) def test_items(self): self.store['max'] = 3 self.store['min'] = 6 self.store['pow'] = 7 slist = list(self.store.items()) self.assertEqual(('min', 6) in slist, True) def test_iteritems(self): self.store['max'] = 3 self.store['min'] = 6 self.store['pow'] = 7 slist = list(self.store.iteritems()) self.assertEqual(('min', 6) in slist, True) def test_iterkeys(self): self.store['max'] = 3 self.store['min'] = 6 self.store['pow'] = 7 slist = list(self.store.iterkeys()) self.assertEqual('min' in slist, True) def test_itervalues(self): self.store['max'] = 3 self.store['min'] = 6 self.store['pow'] = 7 slist = list(self.store.itervalues()) self.assertEqual(6 in slist, True) def test_pop(self): self.store['max'] = 3 self.store['min'] = 6 item = self.store.pop('min') self.assertEqual(item, 6) def test_popitem(self): self.store['max'] = 3 self.store['min'] = 6 self.store['pow'] = 7 item = self.store.popitem() self.assertEqual(len(item) + len(self.store), 4) def test_setdefault(self): self.store['max'] = 3 self.store['min'] = 6 self.store['powl'] = 7 self.store.setdefault('pow', 8) self.assertEqual(self.store.setdefault('pow', 8), 8) self.assertEqual(self.store['pow'], 8) def test_update(self): from shove import Shove tstore = Shove() tstore['max'] = 3 tstore['min'] = 6 tstore['pow'] = 7 self.store['max'] = 2 self.store['min'] = 3 self.store['pow'] = 7 self.store.update(tstore) self.assertEqual(self.store['min'], 6) def test_values(self): self.store['max'] = 3 self.store['min'] = 6 self.store['pow'] = 7 slist = self.store.values() self.assertEqual(6 in slist, True) def test_keys(self): self.store['max'] = 3 self.store['min'] = 6 self.store['pow'] = 7 slist = self.store.keys() self.assertEqual('min' in slist, True)
class TestDbmStore(unittest.TestCase): def setUp(self): self.store = Shove('dbm://test.dbm', compress=True) def tearDown(self): self.store.close() os.remove('test.dbm') def test__getitem__(self): self.store['max'] = 3 self.assertEqual(self.store['max'], 3) def test__setitem__(self): self.store['max'] = 3 self.assertEqual(self.store['max'], 3) def test__delitem__(self): self.store['max'] = 3 del self.store['max'] self.assertEqual('max' in self.store, False) def test_get(self): self.store['max'] = 3 self.assertEqual(self.store.get('min'), None) def test__cmp__(self): tstore = Shove() self.store['max'] = 3 tstore['max'] = 3 self.assertEqual(self.store, tstore) def test__len__(self): self.store['max'] = 3 self.store['min'] = 6 self.assertEqual(len(self.store), 2) def test_close(self): self.store.close() self.assertEqual(self.store, None) def test_clear(self): self.store['max'] = 3 self.store['min'] = 6 self.store['pow'] = 7 self.store.clear() self.assertEqual(len(self.store), 0) def test_items(self): self.store['max'] = 3 self.store['min'] = 6 self.store['pow'] = 7 slist = list(self.store.items()) self.assertEqual(('min', 6) in slist, True) def test_iteritems(self): self.store['max'] = 3 self.store['min'] = 6 self.store['pow'] = 7 slist = list(self.store.iteritems()) self.assertEqual(('min', 6) in slist, True) def test_iterkeys(self): self.store['max'] = 3 self.store['min'] = 6 self.store['pow'] = 7 slist = list(self.store.iterkeys()) self.assertEqual('min' in slist, True) def test_itervalues(self): self.store['max'] = 3 self.store['min'] = 6 self.store['pow'] = 7 slist = list(self.store.itervalues()) self.assertEqual(6 in slist, True) def test_pop(self): self.store['max'] = 3 self.store['min'] = 6 item = self.store.pop('min') self.assertEqual(item, 6) def test_popitem(self): self.store['max'] = 3 self.store['min'] = 6 self.store['pow'] = 7 item = self.store.popitem() self.assertEqual(len(item) + len(self.store), 4) def test_setdefault(self): self.store['max'] = 3 self.store['min'] = 6 self.store['powl'] = 7 self.store.setdefault('pow', 8) self.assertEqual(self.store['pow'], 8) def test_update(self): tstore = Shove() tstore['max'] = 3 tstore['min'] = 6 tstore['pow'] = 7 self.store['max'] = 2 self.store['min'] = 3 self.store['pow'] = 7 self.store.update(tstore) self.assertEqual(self.store['min'], 6) def test_values(self): self.store['max'] = 3 self.store['min'] = 6 self.store['pow'] = 7 slist = self.store.values() self.assertEqual(6 in slist, True) def test_keys(self): self.store['max'] = 3 self.store['min'] = 6 self.store['pow'] = 7 slist = self.store.keys() self.assertEqual('min' in slist, True)
if pr and args.profile: # Finish profiling pr.disable() pr.dump_stats('library.prof') print '[PROFILE] Done' if pr and args.graph: # Convert profile to dot subprocess.Popen(['python', os.environ['GPROF2DOT'], '-fpstats', 'library.prof', '-olibrary.dot']) print '[DOT] Done' # Convert dot to png with open('library.png', 'w') as fp: p = subprocess.Popen(['dot', '-Tpng', 'library.dot'], stdout=fp) ret_code = p.wait() print '[PNG] Done (%s)' % ret_code if pr and args.graph and args.view: # Display image (in default viewer) path = os.path.abspath(os.path.join(os.path.dirname(__file__), 'library.png')) print '[VIEW] Opening image at "%s"' % path os.system("start %s" % path) # Close shove caches http_cache.close() matcher_cache.close() metadata_cache.close()
print '[PROFILE] Done' if pr and args.graph: # Convert profile to dot subprocess.Popen([ 'python', os.environ['GPROF2DOT'], '-fpstats', 'library.prof', '-olibrary.dot' ]) print '[DOT] Done' # Convert dot to png with open('library.png', 'w') as fp: p = subprocess.Popen(['dot', '-Tpng', 'library.dot'], stdout=fp) ret_code = p.wait() print '[PNG] Done (%s)' % ret_code if pr and args.graph and args.view: # Display image (in default viewer) path = os.path.abspath( os.path.join(os.path.dirname(__file__), 'library.png')) print '[VIEW] Opening image at "%s"' % path os.system("start %s" % path) # Close shove caches http_cache.close() matcher_cache.close() metadata_cache.close()