class Dao: def __init__(self, dbname): self.db = Base(dbname).open() self.db.set_string_format(unicode, 'utf-8') def regist_admin(self, data): return self.db.insert( generation = data.get('generation'), dept = data.get('dept'), name_sei = data.get('name_sei'), name_mei = data.get('name_mei'), mail = data.get('mail'), account = data.get('account'), passwd = data.get('crypt_passwd'), add_dt = datetime.now() ) def get_admins(self): result = self.db.select() return result def close(self): self.db.close() def __del__(self): self.db.close()
class DataBase(): def __init__(self): self.db=Base(mypath) self.sfield = {'शब्द': 'Words', 'पद': 'Speech', 'अर्थ': 'Meaning', 'पर्यायवाचि': 'Synonym', 'विपरीतार्थक': 'Antonym', 'अंग्रेजी': 'English'} self.db.open() def _get_attribute(self,args): ret_val = {} for k,f in self.sfield.items(): ret_val[k] = getattr(args, f) if ret_val[k] == None: ret_val.pop(k) return ret_val def select(self,value): self.record = self.db.select([f for f in self.sfield.itervalues()],Words=value) for v in self.record: self.g = self._get_attribute(v) return self.g def fields(self): return self.db.field_names def get_field_length(self, value): t = self.select(value) return len(self.g)
class RepDB: def __init__(self, path): self.path = path self.dbh_stored_blocks = Base(self.path) try: self.dbh_stored_blocks.create(('key', str), ('version', str)) except IOError: pass def open(self): self.dbh_stored_blocks.open() def add(self, oid, block_id, version): key = str((oid, block_id)) # lets see if we already have a key stored set = self.dbh_stored_blocks.select_for_update(['key', 'version'], key=key) if set == []: self.dbh_stored_blocks.insert(key, str(version)) else: set[0].update() def get(self, oid, block_id): key = str((oid, block_id)) result = self.dbh_stored_blocks.select(['key', 'version'], key=key) return result[0].version def update(self, oid, block_id, version): self.add(oid, block_id, version) def delete(self, oid, block_id): key = str((oid, block_id)) set = self.dbh_stored_blocks.select_for_update(['key', 'version'], key=key) self.dbh_stored_blocks.delete(set[0]) def close(self): self.dbh_stored_blocks.close() def getIterator(self): return RepDBIterator([record for record in self.dbh_stored_blocks])
class RepDB: def __init__(self, path): self.path = path self.dbh_stored_blocks = Base(self.path) try: self.dbh_stored_blocks.create(('key', str), ('version', str)) except IOError: pass def open(self): self.dbh_stored_blocks.open() def add(self, oid, block_id, version): key = str((oid, block_id)) # lets see if we already have a key stored set = self.dbh_stored_blocks.select_for_update(['key','version'],key=key) if set == []: self.dbh_stored_blocks.insert(key, str(version)) else: set[0].update() def get(self, oid, block_id): key = str((oid, block_id)) result = self.dbh_stored_blocks.select(['key','version'],key=key) return result[0].version def update(self, oid, block_id, version): self.add(oid, block_id, version) def delete(self, oid, block_id): key = str((oid, block_id)) set = self.dbh_stored_blocks.select_for_update(['key','version'],key=key) self.dbh_stored_blocks.delete(set[0]) def close(self): self.dbh_stored_blocks.close() def getIterator(self): return RepDBIterator([record for record in self.dbh_stored_blocks])
class DB: def __init__(self, storage_path): self.dbh_objects = Base(os.path.join(storage_path, 'objects')) self.dbh_blocks = Base(os.path.join(storage_path, 'blocks')) self.dbh_replicas = Base(os.path.join(storage_path, 'replicas')) self.dbh_tree = Base(os.path.join(storage_path, 'tree')) self.dbh_paths = Base(os.path.join(storage_path, 'paths')) self.dbh_id = Base(os.path.join(storage_path, 'id')) self.dbh_tags = Base(os.path.join(storage_path, 'tags')) self.storage_path = storage_path def __create_root(self): """ Check if the filesystem has a / and if not create it""" print "Initializing filesystem..." if self.get_file(path='/'): return print "Creating root..." f = FSObject(1, 1, '/', 0, 0, 0, 0) # lets see if we already have a key stored set = self.dbh_objects.select(['oid'], oid=str(f.oid)) if set == []: # we have create tree and paths first self.dbh_tree.insert(str(f.oid), str(f.parent)) self.dbh_paths.insert(str((f.parent, f.path))) self.dbh_objects.insert(str(f.oid), dumps(f), self.dbh_tree[len(self.dbh_tree) - 1], self.dbh_paths[len(self.dbh_paths) - 1]) #set the current oid for the id increment sequence set = self.dbh_id.select(['curr_oid']) if set == []: self.dbh_id.insert('1') def setup_fs_db(self): try: self.dbh_blocks.create(('key', str), ('blocks', str)) except IOError: self.dbh_blocks.open() try: self.dbh_replicas.create(('key', str), ('replicas', str)) except IOError: self.dbh_replicas.open() try: self.dbh_tree.create(('oid', str), ('parent', str)) except IOError: self.dbh_tree.open() try: self.dbh_tags.create(('oid', str), ('tag', str)) except IOError: self.dbh_tags.open() try: self.dbh_paths.create(('key', str)) except IOError: self.dbh_paths.open() try: self.dbh_id.create(('curr_oid', str)) except IOError: self.dbh_id.open() try: self.dbh_objects.create(('oid', str), ('fsobj', str), ('tree', self.dbh_tree), ('paths', self.dbh_paths)) except IOError: self.dbh_objects.open() self.__create_root() def get_path_oid(self, path): """Gets the parent filenode for path""" nodes = [] parent_path = path while 1: (parent_path, node) = os.path.split(parent_path) if node == '': nodes.insert(0, '/') break nodes.insert(0, node) parent_oid = 0 for node_name in nodes: key = str((parent_oid, node_name)) print "looking up: %s" % key # search for a match f = None for record in [record for record in self.dbh_objects]: if record.paths.key == key: f = loads(record.fsobj) break print "found it!" if not f: return 0 parent_oid = f.oid return parent_oid def insert_file(self, path, fsobj): #check first if there is a parent directory to store this file f = self.get_file(path=path) print "inserting file with path: " + path print fsobj if not f: print "ERR: [%s]" % os.path.split(fsobj.path)[0] raise FileSystemError('No parent directory to store: %s' % fsobj.path) #the parent of this object is the path fsobj.parent = f.oid set = self.dbh_id.select_for_update(['curr_oid']) curr_oid = int(set[0].curr_oid) + 1 fsobj.oid = curr_oid print "Inserting OID: %s" % fsobj # lets see if we already have a key stored result = self.dbh_objects.select(['oid', 'fsobj'], oid=str(fsobj.oid)) if result != []: raise FileSystemError('File already exists') else: # we have create tree and paths first self.dbh_tree.insert(str(fsobj.oid), str(fsobj.parent)) self.dbh_paths.insert(str((fsobj.parent, fsobj.path))) self.dbh_objects.insert(str(fsobj.oid), dumps(fsobj), self.dbh_tree[len(self.dbh_tree) - 1], self.dbh_paths[len(self.dbh_paths) - 1]) set[0].update(curr_oid=str(curr_oid)) return curr_oid def get_file(self, oid='', path=''): if oid: set = self.dbh_objects.select(['oid', 'fsobj'], oid=str(oid)) if set == []: f = None else: f = set[0].fsobj elif path: if path == '/': key = str((0, '/')) else: parent_oid = self.get_path_oid(os.path.split(path)[0]) node_name = os.path.split(path)[1] key = str((parent_oid, node_name)) # search for a match f = None for record in [record for record in self.dbh_objects]: print record.paths.key if record.paths.key == key: f = record.fsobj break else: f = None if f: f = loads(f) return f def get_children(self, oid): # lookup FSOBJECT with given oid set = self.dbh_objects.select(['oid', 'fsobj'], oid=str(oid)) if set == []: return [] file_array = [] # lookup objects with parent oid set = self.dbh_tree.select(['oid', 'parent'], parent=str(oid)) for i in set: obj = self.dbh_objects.select(['oid', 'fsobj'], oid=str(i.oid)) if obj != []: file_array.append(loads(obj[0].fsobj)) return file_array def debug_print_db(self, db): pass def print_object_db(self): self.debug_print_db(self.dbh_objects) def delete_dir(self, oid): pass def delete_file(self, oid): pass def rename_file(self, src, dest): pass def update_file(self, fsobj): set = self.dbh_objects.select_for_update(['oid', 'fsobj'], oid=str(fsobj.oid)) if set != []: set[0].update(fsobj=dumps(fsobj)) def add_block(self, block, serverid): f = self.get_file(oid=str(block.oid)) if not f: raise FileSystemError('add_block: Object %s does not exist' % block.oid) key = str( (long(block.oid), long(block.block_id))) #the key is both the oid and the block_id set1 = self.dbh_replicas.select_for_update(['key', 'replicas'], key=key) if set1 == []: replicas = FSReplicas(block.oid, block.block_id) else: replicas = loads(set1[0].replicas) f.blocks[block.block_id] = block.version set2 = self.dbh_blocks.select_for_update(['key', 'blocks'], key=key) if set2 == []: b = None else: b = set2[0].block if b: b = loads(b) diff = block.size - b.size else: diff = block.size f.size += diff # update or insert? if set1 == []: self.dbh_blocks.insert(key, dumps(block)) else: set1[0].update(blocks=dumps(block)) self.update_file(f) replicas.add(serverid, block.version) # update or insert? if set2 == []: self.dbh_replicas.insert(key, dumps(replicas)) else: set2[0].update(replicas=dumps(replicas)) def add_block_replica(self, block, serverid): f = self.get_file(str(block.oid)) if not f: raise FileSystemError( 'add_block_replica: Object %s does not exist' % block.oid) key = str((block.oid, block.block_id)) set = self.dbh_replicas.select_for_update(['key', 'replicas'], key=key) if set == []: replicas = FSReplicas(block.oid, block.block_id) else: replicas = loads(set[0].replicas) replicas.add(serverid, block.version) # update or insert? if set == []: self.dbh_replicas.insert(key, dumps(replicas)) else: set[0].update(replicas=dumps(replicas)) def get_block_replicas(self, oid, block_id): key = str((long(oid), long(block_id))) set = self.dbh_replicas.select(['key', 'replicas'], key=key) if set == []: return None return loads(set[0].replicas) def get_block(self, oid, block_id): key = str((long(oid), long(block_id))) set = self.dbh_blocks.select(['key', 'blocks'], key=key) if set == []: return None return loads(set[0].blocks) def print_replicas_db(self): self.debug_print_db(self.dbh_replicas) def close_fs_db(self): self.dbh_blocks.close() self.dbh_replicas.close() self.dbh_tree.close() self.dbh_id.close() self.dbh_paths.close() self.dbh_objects.close()
class TransactionsDB(object): BASE = 'banan/transactions' def __init__(self, conf): self.config = conf self._sessions = {} self.open() # Management def open(self): self.db = Base(TransactionsDB.BASE) try: self.db.open() except IOError: self.db.create(('amount' , float), ('amount_local', float), ('date' , date), ('account' , str), ('label' , str), ('currency' , str)) def close(self): self.db.close() def clearall(self): self.db.destroy() self.open() def insert(self, entry): self.db.insert(amount = entry['amount'], amount_local = entry['amount_local'], date = entry['date'], account = entry['account'], label = entry['label'], currency = entry['currency']) def feed(self, fpath, parser, skip_duplicates=True, overwrite=False, delete=False, dry_run=False): deleted = added = 0 for entry in parser.parse(fpath): if dry_run: print('%s %-40s\t%12.2f %s' % (entry['date'].isoformat(), entry['account'][:40], entry['amount'], entry['currency'])); continue if skip_duplicates or overwrite or delete: _dup = self.db(date=entry['date'], account=entry['account'], amount=entry['amount']) if _dup: if overwrite or delete: deleted += len(_dup) self.db.delete(_dup) else: continue if delete: continue entry['label'] = self.config.assign_label(entry) self.insert(entry) added += 1 if not dry_run: INFO(' added %i transactions' % added) INFO(' deleted %i transactions' % deleted) parser.post_process(self.db, added) def update_labels(self): # Load all records into memory. File will get corrupt if using the iterator. records = [rec for rec in self.db] for record in records: as_dict = dict((field, getattr(record, field)) for field in record.fields) label = self.config.assign_label(as_dict) if label != record.label: self.db.update(record, label=label) self.db.cleanup() # Queries get_amount = lambda rec: rec.amount_local def results_as_text(self, results): results.sort_by('date') idx = 0 record = results[idx] text_list = [] while True: text_list.append('%s %-40s\t%12.2f %s' % (record.date.isoformat(), unicode(record.account[:40], 'utf-8'), record.amount, record.currency)); try: idx += 1 record = results[idx] except IndexError: return text_list def assemble_data(self, sid, datatype, foreach, show, select): try: session = self._sessions.get(sid, {}) if session: if session['raw_query'] == (foreach, show, select): # Same query, return cached result return True, \ self._sessions[sid]['flot_' + show] if datatype == 'plot' else \ self._sessions[sid]['text'] # Helpers get_amount = lambda rec: rec.amount_local M = range(1,13) total = strlen = 0 data = {} text = {} query = 'date1 <= date < date2 and label == l' if foreach == 'label': if session: if session['raw_query'][0] == 'label' and session['raw_query'][2] == select: # Same query, but different presentation (sum or average) return True, \ self._sessions[sid]['flot_' + show] if datatype == 'plot' else \ self._sessions[sid]['text'] # New query dates = re.findall('[0-9]{6}', unquote_plus(select)) date1 = date2 = date(int(dates[0][2:]), int(dates[0][:2]), 1) if len(dates) == 2: date2 = date(int(dates[1][2:]), int(dates[1][:2]), 1) date2 = date(date2.year + (date2.month == 12), M[date2.month - 12], 1) for label in self.config.labels.iterkeys(): results = self.db.select(None, query, l = label, date1 = date1, date2 = date2) value = sum(map(get_amount, results)) if abs(value) > 1: data[label] = value if label not in self.config.cash_flow_ignore: total += value else: label += '*' text[label] = self.results_as_text(results) strlen = len(text[label][-1]) sumstr = '%12.2f %s' % (value, self.config.local_currency) text[label].append('-' * strlen) text[label].append(' ' * (strlen - len(sumstr)) + sumstr) ydelta = date2.year - date1.year mdelta = date2.month - date1.month delta = 12 * ydelta + mdelta session['flot_average'] = {} for key, val in data.iteritems(): session['flot_average'][key] = val/delta elif foreach in ('month', 'year'): # New query date1 = date2 = first = datetime.now() if foreach == 'month': first = date(date1.year - 1, date1.month, 1) date1 = date(date1.year - (date1.month == 1), M[date1.month - 2], 1) date2 = date(date2.year, date2.month, 1) else: first = date(date1.year - 9, 1, 1) date1 = date(date1.year, 1, 1) date2 = date(date2.year + 1, 1, 1) select = unquote_plus(select) while date1 >= first: results = self.db.select(None, query, l = select, date1 = date1, date2 = date2) value = sum(map(get_amount, results)) date2 = date1 if foreach == 'month': key = date1.strftime('%Y.%m') date1 = date(date2.year - (date2.month == 1), M[date2.month - 2], 1) else: key = str(date1.year) date1 = date(date2.year - 1, 1, 1) data[key] = value total += value if results: text[key] = self.results_as_text(results) strlen = len(text[key][-1]) sumstr = '%12.2f %s' % (value, self.config.local_currency) text[key].append('-' * strlen) text[key].append(' ' * (strlen - len(sumstr)) + sumstr) # All good, set new session attributes session['raw_query'] = (foreach, show, select) session['flot_sum'] = data session['text'] = text if session['text']: session['text']['***'] = ['-' * strlen, 'SUM: %12.2f %s' % (total, self.config.local_currency), '-' * strlen] self._sessions[sid] = session return True, session['flot_' + show] if datatype == 'plot' else session['text'] except Exception as e: return False, str(e)
10**random.randint(-307,307))) db.insert(name=random.choice(names)) # missing fields # insert as string db.set_string_format(unicode,'latin-1') db.set_string_format(date,'%d-%m-%y') db.insert_as_strings(name="testname",fr_name=random.choice(fr_names), age=10,size=123,birth="07-10-94") print db[len(db)-1].birth db.insert_as_strings("testname",random.choice(fr_names), 11,134,"09-10-94",1.0) print db[len(db)-1].birth # search between 2 dates print '\nBirth between 1960 and 1964' for r in db.select(None,birth=[date(1960,1,1),date(1964,12,13)]): print r.name,r.birth f = buzhug_files.FloatFile().to_block def all(v): print v,[ord(c) for c in f(v)] # search between 2 floats print '\nFloat between 0 and 1e50' print len(db.select(None,afloat=[0.0,1e50])) all(0.0) all(1e50) print 'lc' for r in [ r for r in db if 0.0 <= r.afloat <= 1e50 ]: all(r.afloat) print '\n select' for r in db.select(None,'x<=afloat<=y',x=0.0,y=1e50):
#!/bin/python2.7 from buzhug import Base import os db = Base(os.getcwd() + '/db/bookDB').open() res = db.select(reviewed=True) #select all reviewed for r in res: print r.text.encode('utf-8')
print "\n**************************" print "Welcome to Tar Heel Ranker v1.0. The goal of this project is to actively assist in the reviewing process of books that have no home. For more information contact Jesse Osiecki at [email protected]" print "standard usage: python tarheelreview.py -FLAGS" print "Flags are: r for redisbayes checker (Bayesian spam filter. Make sure Redis is running for this" print "g for gibberish detector using markov chains on letter transitions" print "m for markover -- markov chain filter using word transitions" print "a for author check. Outputs number of books a given author has in the Database that are already reviewed" print "s for rating check. Tests books current rating against a set value (4) and returns boolean if higher" print "recommended usage is -masg" print "NOTE: to end the program, simply press CTRL-D and the reviewed books will be noted in a file in the CWD" print "\n\n**************************" db = Base(os.getcwd() + '/db/bookDB').open() #open redis rb = redisbayes.RedisBayes(redis.StrictRedis(host='localhost', port=6379, db=0)) books = db.select(reviewed=False) #select all unreviewed markov = markover.Markov() def author_checker(aut): authored_books = db.select(author_id=aut, reviewed=True) return len(authored_books) def rating_checker(rat, lim): return rat >= lim redis_bayes = False gibb = False markover = False author_check = False rating_check = False l = len(sys.argv)
def main(): AestheticScorer.LoadModel() #Make the database. We need tables for events, locations, photos, people, and faces events = Base(os.getcwd()+"\\Databases\\events.db") events.create(('name',str),('firsttime',str),('lasttime',str),mode="override") locations = Base(os.getcwd()+"\\Databases\\locations.db") locations.create(('lat',float),('lon',float),('name',str),mode="override") photos = Base(os.getcwd()+"\\Databases\\photos.db") photos.create(('path',str),('timestamp',str),('aestheticscore',int),('locationid',int), ('eventid',int), ("width", int), ("height", int),mode="override") people = Base(os.getcwd()+"\\Databases\\people.db") people.create(('name',str),('age',int),mode="override") faces = Base(os.getcwd()+"\\Databases\\faces.db") faces.create(('photoid',int),('thumbnail',str),('personid',int),('x',int),('y',int),('w',int),('h',int),mode="override") # Walk through all the directories, making a list of photos, geotagging the lowest level subdirectories, and making # a master list of the photos and geotags photolist = [] geotaglist = [] print "geocoding directories" for dirname, dirnames, filenames in os.walk(os.getcwd()+"\\Images\\photos"): #geocode all the subdirectory names for subdirname in dirnames: n,lat,long = geotag.findlocation(subdirname) #if we have a problem geotagging, prompt the user for a different location name while n == "NONE": newname = raw_input("We couldn't find the location '" + subdirname + "'. Type in another place name to try.") n, lat, long = geotag.findlocation(subdirname) #once we have a valid location, insert it if it's not already in the database if not locations(name=n): locations.insert(float(lat), float(long), n) #add all the files to a file list, and go ahead and make a parallel geotags for filename in filenames: #print "filename is ",filename if filename[-3:] == "jpg" or filename[-3:] == "JPG": #find the id for that subdirname in the database so we can geotag it locname = dirname[dirname.rfind('\\') + 1:] #location = locations(name=dirname) photolist.append(os.path.join(dirname,filename)) geotaglist.append((os.path.join(dirname,filename),locname)) #make a list to identify which event each photo is in #print photolist print "getting events" eventLabels, uniqueEvents = timecluster.eventCluster(photolist) #print "events: " #print eventLabels print uniqueEvents #insert the events into the event database for label in uniqueEvents: events.insert(label[1],"","") #the events are already sorted according to photo names #now sort the geotags and photolist according to photo names as well, so we'll have parallel lists geotaglist.sort() photolist.sort() #now we can finally insert each photo, with a name, event, and geotag for i in range(len(photolist)): print i, photolist[i] width, height = Image.open(photolist[i]).size photos.insert(photolist[i],eventLabels[i][1], AestheticScorer.getScore(photolist[i]), locations(name=geotaglist[i][1])[0].__id__, eventLabels[i][0], int(width), int(height)) print "finding faces" #for all the images we just gathered, find the people and faces, and insert them into the database facelist = [] for file in photolist: facelist.append(detectfaces.detectFaceInImage(file)) faceimages, projections, imgs, minFaces = detectfaces.faceLBP(facelist)#detectfaces.facePCA(facelist) labels, nclusters = detectfaces.gMeansCluster(projections, minFaces) #detectfaces.visualizeResults(faceimages, labels, nclusters) #add the individuals we found in the photos into the people database i = 0 for i in range(0,nclusters): people.insert(str(i),0) #add the faces, linking them to the individuals faceindex = 0 photoindex = 0 for listing in facelist: facerects = listing[1:] for entry in facerects: faces.insert(photoindex,imgs[faceindex],people[labels[faceindex]].__id__,entry[0],entry[1],entry[2],entry[3]) faceindex = faceindex + 1 photoindex = photoindex + 1 print "updating events" #update the events table to include tthe start time eventtimes=[] for event in events.select(None).sort_by("+__id__"): orderedPhotos = photos.select(eventid=event.__id__).sort_by("+timestamp") events.update(event,firsttime=orderedPhotos[0].timestamp) events.update(event,lasttime=orderedPhotos[-1].timestamp) print "Done"
def run_test(thread_safe=False): if not thread_safe: db = Base(r'dummy') else: db = TS_Base('dummy') db.create(('name', str), ('fr_name', unicode), ('age', int), ('size', int, 300), ('birth', date, date(1994, 1, 14)), ('afloat', float, 1.0), ('birth_hour', dtime, dtime(10, 10, 10)), mode='override') # test float conversions if thread_safe is False: f = db._file["afloat"] def all(v): return [ord(c) for c in v] for i in range(10): afloat = random.uniform(-10**random.randint(-307, 307), 10**random.randint(-307, 307)) try: assert cmp(afloat, 0.0) == cmp(f.to_block(afloat), f.to_block(0.0)) except: print afloat print "afloat > 0.0 ?", afloat > 0.0 print "blocks ?", f.to_block(afloat) > f.to_block(0.0) print all(f.to_block(afloat)), all(f.to_block(0.0)) raise assert db.defaults["age"] == None assert db.defaults["size"] == 300 assert db.defaults["afloat"] == 1.0 assert db.defaults["birth_hour"] == dtime(10, 10, 10) assert db.defaults["birth"] == date(1994, 1, 14) for i in range(100): db.insert(name=random.choice(names), fr_name=unicode(random.choice(fr_names), 'latin-1'), age=random.randint(7, 47), size=random.randint(110, 175), birth=date(random.randint(1858, 1999), random.randint(1, 12), 10), afloat=random.uniform(-10**random.randint(-307, 307), 10**random.randint(-307, 307)), birth_hour=dtime(random.randint(0, 23), random.randint(0, 59), random.randint(0, 59))) assert len(db) == 100 assert isinstance(db[50].fr_name, unicode) print db[50].fr_name.encode('latin-1') db.open() # test if default values have not been modified after open() assert db.defaults["age"] == None assert db.defaults["size"] == 300 assert db.defaults["afloat"] == 1.0 assert db.defaults["birth_hour"] == dtime(10, 10, 10) assert db.defaults["birth"] == date(1994, 1, 14) for i in range(5): # insert a list db.insert( random.choice(names), unicode(random.choice(fr_names), 'latin-1'), random.randint(7, 47), random.randint(110, 175), date(random.randint(1958, 1999), random.randint(1, 12), 10), random.uniform(-10**random.randint(-307, 307), 10**random.randint(-307, 307)), dtime(random.randint(0, 23), random.randint(0, 59), random.randint(0, 59))) db.insert(name=random.choice(names)) # missing fields for field in db.field_names[2:]: if field == "name": continue try: assert getattr(db[-1], field) == db.defaults[field] except: print "attribute %s not set to default value %s" % (field, db[-1]) raise # insert as string db.set_string_format(unicode, 'latin-1') db.set_string_format(date, '%d-%m-%y') db.set_string_format(dtime, '%H-%M-%S') db.insert_as_strings(name="testname", fr_name=random.choice(fr_names), age=10, size=123, birth="07-10-95", birth_hour="20-53-3") assert db[-1].birth == date(1995, 10, 7) assert db[-1].name == "testname" assert db[-1].age == 10 assert db[-1].afloat == db.defaults["afloat"] db.insert_as_strings("testname", random.choice(fr_names), 11, 134, "09-12-94", 1.0, "5-6-13") assert db[len(db) - 1].birth == date(1994, 12, 9) assert db[-1].name == "testname" assert db[-1].age == 11 assert db[-1].size == 134 assert db[-1].afloat == 1.0 # search between 2 dates print '\nBirth between 1960 and 1970' for r in db.select(None, birth=[date(1960, 1, 1), date(1970, 12, 13)]): print r.name, r.birth print "sorted" for r in db.select(None, birth=[date(1960, 1, 1), date(1970, 12, 13)]).sort_by('+name-birth'): print r.name, r.birth f = buzhug_files.FloatFile().to_block def all(v): return [ord(c) for c in f(v)] # search between 2 floats # selection by list comprehension s1 = [r for r in db if 0.0 <= r.afloat <= 1e50] # selection by select s2 = db.select(None, 'x<=afloat<=y', x=0.0, y=1e50) # selction by select with interval s3 = db.select(None, afloat=[0.0, 1e50]) try: assert len(s1) == len(s2) == len(s3) except: print "%s records by list comprehension, " % len(s1) print "%s by select by formula," % len(s2) print "%s by select by interval" % len(s3) for r in s1: try: assert r in s2 except: print all(r.afloat) for r in s2: try: assert r in s1 except: print "in select but not in list comprehension", r raise r = db[0] assert r.__class__.db is db fr = random.choice(fr_names) s1 = [r for r in db if r.age == 30 and r.fr_name == unicode(fr, 'latin-1')] s2 = db.select(['name', 'fr_name'], age=30, fr_name=unicode(fr, 'latin-1')) assert len(s1) == len(s2) # different ways to count the number of items assert len(db) == sum([1 for r in db]) == len(db.select(['name'])) # check if version number is correctly incremented for i in range(5): recs = db.select_for_update(['name'], 'True') version = recs[0].__version__ recs[0].update() assert db[0].__version__ == version + 1 # check if cleanup doesn't change db length length_before = len(db) db.cleanup() assert len(db) == length_before # check if selection by select on __id__ returns the same as direct # access by id recs = db.select([], '__id__ == c', c=20) assert recs[0] == db[20] # check that has_key returns False for invalid hey assert not db.has_key(1000) # drop field db.drop_field('name') # check if field was actually removed from base definition and rows assert not "name" in db.fields assert not hasattr(db[20], "name") # add field db.add_field('name', str, default="marcel") # check if field was added with the correct default value assert "name" in db.fields assert hasattr(db[20], "name") assert db[20].name == "marcel" # change default value db.set_default("name", "julie") db.insert(age=20) assert db[-1].name == "julie" # delete a record db.delete([db[10]]) # check if record has been deleted try: print db[10] raise Exception, "Row 10 should have been deleted" except IndexError: pass assert 10 not in db assert len(db) == length_before # selections # selection by generator expression # age between 30 et 32 d_ids = [] for r in [r for r in db if 33 > r.age >= 30]: d_ids.append(r.__id__) length = len(db) # remove these items db.delete([r for r in db if 33 > r.age >= 30]) # check if correct number of records removed assert len(db) == length - len(d_ids) # check if all records have been removed assert not [r for r in db if 33 > r.age >= 30] # updates # select name = pierre s1 = db.select(['__id__', 'name', 'age', 'birth'], name='pierre') # make 'pierre' uppercase for record in db.select_for_update(None, 'name == x', x='pierre'): db.update(record, name=record.name.upper()) # check if attribute was correctly updated for rec in s1: assert db[rec.__id__] == "Pierre" # increment ages for record in db.select_for_update([], 'True'): age = record.age if not record.age is None: db.update(record, age=record.age + 1) # check assert db[record.__id__].age == age + 1 for record in [r for r in db]: age = record.age if not record.age is None: db.update(record, age=record.age + 1) # check assert db[record.__id__].age == age + 1 # change dates for record in db.select_for_update([], 'age>v', v=35): db.update(record, birth=date(random.randint(1958, 1999), random.randint(1, 12), 10)) db.commit() # check length after commit assert sum([1 for r in db]) == len(db) # insert new records for i in range(50): db.insert(name=random.choice(names), age=random.randint(7, 47), size=random.randint(110, 175)) # check that record 10 is still deleted try: print db[10] raise Exception, "Row 10 should have been deleted" except IndexError: pass print db.keys() print "has key 10 ?", db.has_key(10) assert 10 not in db #raw_input() # check that deleted_lines was cleared by commit() assert not db._pos.deleted_lines print db._del_rows.deleted_rows length = len(db) # before cleanup # physically remove the deleted items db.cleanup() # check that deleted_lines and deleted_rows are clean assert not db._pos.deleted_lines assert not db._del_rows.deleted_rows # check that record 10 is still deleted try: print db[10] raise Exception, "Row 10 should have been deleted" except IndexError: pass assert 10 not in db # check that length was not changed by cleanup assert len(db) == length assert len([r for r in db]) == length # age > 30 for r in db.select(['__id__', 'name', 'age'], 'name == c1 and age > c2', c1='pierre', c2=30): assert r.name == "pierre" assert r.age > 30 # name =="PIERRE" and age > 30 for r in db.select(['__id__', 'name', 'age', 'birth'], 'name == c1 and age > c2', c1='PIERRE', c2=30): assert r.name == 'PIERRE' assert r.age > 30 # test with != for r in db.select(['__id__'], 'name != c1', c1='claire'): assert r.name != 'claire' # age > id # with select s1 = db.select(['name', '__id__', 'age'], 'age > __id__') for r in s1: assert r.age > r.__id__ # with iter s2 = [r for r in db if r.age > r.__id__] for r in s2: assert r.age > r.__id__ assert len(s1) == len(s2) # birth > date(1978,1,1) # with select s1 = db.select(['name', '__id__', 'age'], 'birth > v', v=date(1978, 1, 1)) for r in s1: assert r.birth > date(1978, 1, 1) # with iter s2 = [r for r in db if r.birth and r.birth > date(1978, 1, 1)] for r in s2: assert r.birth > date(1978, 1, 1) assert len(s1) == len(s2) # test with floats for i in range(10): x = random.uniform(-10**random.randint(-307, 307), 10**random.randint(-307, 307)) s1 = [r for r in db if r.afloat > x] s2 = db.select(['name'], 'afloat > v', v=x) assert len(s1) == len(s2) # base with external link houses = Base('houses') houses.create(('address', str), ('flag', bool), ('resident', db, db[0]), mode="override") addresses = [ 'Giono', 'Proust', 'Mauriac', 'Gide', 'Bernanos', 'Racine', 'La Fontaine' ] ks = db.keys() for i in range(50): x = random.choice(ks) address = random.choice(addresses) houses.insert(address=address, flag=address[0] > "H", resident=db[x]) # houses with jean s1 = [] for h in houses: if h.resident.name == 'jean': s1.append(h) # by select : ??? #s2 = houses.select([],'resident.name == v',v='jean') # assert len(s1) == len(s2) h1 = Base('houses') h1.open() l1 = len(h1.select([], flag=True)) l2 = len(h1.select([], flag=False)) assert l1 + l2 == len(h1) class DictRecord(Record): def __getitem__(self, k): item = self names = k.split('.') for name in names: item = getattr(item, name) return item h1.set_record_class(DictRecord) print '\nrecord_class = DictRecord, h1[0]' print h1[0] print "\nResident name: %(resident.name)s\nAddress: %(address)s" % h1[0]
assert db[-1].name == "testname" assert db[-1].age == 10 assert db[-1].afloat == db.defaults["afloat"] db.insert_as_strings("testname",random.choice(fr_names), 11,134,"09-12-94",1.0, "5-6-13") assert db[len(db)-1].birth == date(1994,12,9) assert db[-1].name == "testname" assert db[-1].age == 11 assert db[-1].size == 134 assert db[-1].afloat == 1.0 # search between 2 dates print '\nBirth between 1960 and 1970' for r in db.select(None,birth=[date(1960,1,1),date(1970,12,13)]): print r.name,r.birth print "sorted" for r in db.select(None,birth=[date(1960,1,1),date(1970,12,13)]).sort_by('+name-birth'): print r.name,r.birth f = buzhug_files.FloatFile().to_block def all(v): return [ord(c) for c in f(v)] # search between 2 floats # selection by list comprehension s1 = [ r for r in db if 0.0 <= r.afloat <= 1e50 ] # selection by select
# insert as string db.set_string_format(unicode, 'latin-1') db.set_string_format(date, '%d-%m-%y') db.insert_as_strings(name="testname", fr_name=random.choice(fr_names), age=10, size=123, birth="07-10-94") print db[len(db) - 1].birth db.insert_as_strings("testname", random.choice(fr_names), 11, 134, "09-10-94", 1.0) print db[len(db) - 1].birth # search between 2 dates print '\nBirth between 1960 and 1964' for r in db.select(None, birth=[date(1960, 1, 1), date(1964, 12, 13)]): print r.name, r.birth f = buzhug_files.FloatFile().to_block def all(v): print v, [ord(c) for c in f(v)] # search between 2 floats print '\nFloat between 0 and 1e50' print len(db.select(None, afloat=[0.0, 1e50])) all(0.0) all(1e50) print 'lc'
def main(): events = Base(os.getcwd()+"\\Databases\\events.db") locations = Base(os.getcwd()+"\\Databases\\locations.db") photos = Base(os.getcwd()+"\\Databases\\photos.db") people = Base(os.getcwd()+"\\Databases\\people.db") faces = Base(os.getcwd()+"\\Databases\\faces.db") training = Base(os.getcwd()+"\\Databases\\training_images.db") features = Base(os.getcwd()+"\\Databases\\features.db") try: print "============ events ================" events.open() for field in events.fields: print field,events.fields[field] print "len",len(events),"\n\n" for record in events.select().sort_by("+firsttime"): print record elist =[ None for i in range(len(events))] print elist print "============ locations ================" locations.open() for field in locations.fields: print field,locations.fields[field] print "len",len(locations),"\n\n" #for record in locations: # print record print "============ photos ================" photos.open() for field in photos.fields: print field,photos.fields[field] print "len",len(photos),"\n\n" for record in photos: print record print "============ people ================" people.open() for field in people.fields: print field,people.fields[field] print "len",len(people),"\n\n" #for record in people: # print record print "=========== faces ==============" faces.open() for field in faces.fields: print field,faces.fields[field] print "len",len(faces),"\n\n" #for record in faces: # print record print "============ training ========" training.open() for field in training.fields: print field, training.fields[field] print "len",len(training),"\n\n" print "============ features ===========" features.open() for field in features.fields: print field, features.fields[field] print "len",len(features),"\n\n" except IOError as err: print "no database there:",err
time = m.group(1) yamlfile = yamlfile+time artist = m.group(2).strip() title = m.group(3).strip() dateformat="%Y-%m-%d" date = datetime.datetime.strftime(datetime.date.today(),dateformat) timestamp=datetime.datetime.strptime(date+' '+time,dateformat+" %H:%M:%S") #lastfmsubmitd doesnt accept timezones so convert to UTC zone = pytz.timezone('Europe/Amsterdam') localtimestamp = zone.localize(timestamp) utc=pytz.UTC timestamp = localtimestamp.astimezone(utc) all={'artist': artist,'title': title, 'length': 120,'time': '!timestamp '+timestamp.strftime('%Y-%m-%d %H:%M:%S')} recordsfromthisstamp = db.select(['date'],date=timestamp) #only update if it is a non-existing entry if len(recordsfromthisstamp)==0: record_id = db.insert(timestamp,artist,title) all={'artist': artist,'title': title, 'length': 120,'time': '!timestamp '+timestamp.strftime('%Y-%m-%d %H:%M:%S')} stream = file(yamlfile,'w') result=yaml.dump(all,default_flow_style=False) result=result.replace("'","") stream.write(result) stream.close() db.close()
#this is a commandline utility to quickly get book text for testing purposes from buzhug import Base import os import sys import json db = Base(os.getcwd() + '/db/bookDB') db.open() c=0 l = len(sys.argv) while c < l-1: c+=1 inp = sys.argv[c] dupProtect = True bookID = int(inp) records = db.select(ID = bookID) bookText = '' slug = '' for r in records: j = json.loads(r.json) if j.get('ID') == bookID: text = j.get('pages') for p in text: bookText += p['text'] + '\n' slug = r.slug if dupProtect: break print bookText db.close()
def run_test(thread_safe=False): if not thread_safe: db = Base(r'dummy') else: db = TS_Base('dummy') db.create(('name',str), ('fr_name',unicode), ('age',int), ('size',int,300), ('birth',date,date(1994,1,14)), ('afloat',float,1.0), ('birth_hour', dtime,dtime(10,10,10)), mode='override') # test float conversions if thread_safe is False: f = db._file["afloat"] def all(v): return [ord(c) for c in v] for i in range(10): afloat = random.uniform(-10**random.randint(-307,307), 10**random.randint(-307,307)) try: assert cmp(afloat,0.0) == cmp(f.to_block(afloat),f.to_block(0.0)) except: print afloat print "afloat > 0.0 ?",afloat>0.0 print "blocks ?",f.to_block(afloat)>f.to_block(0.0) print all(f.to_block(afloat)),all(f.to_block(0.0)) raise assert db.defaults["age"] == None assert db.defaults["size"] == 300 assert db.defaults["afloat"] == 1.0 assert db.defaults["birth_hour"] == dtime(10,10,10) assert db.defaults["birth"] == date(1994,1,14) for i in range(100): db.insert(name=random.choice(names), fr_name = unicode(random.choice(fr_names),'latin-1'), age=random.randint(7,47),size=random.randint(110,175), birth=date(random.randint(1858,1999),random.randint(1,12),10), afloat = random.uniform(-10**random.randint(-307,307), 10**random.randint(-307,307)), birth_hour = dtime(random.randint(0, 23), random.randint(0, 59), random.randint(0, 59))) assert len(db)==100 assert isinstance(db[50].fr_name,unicode) print db[50].fr_name.encode('latin-1') db.open() # test if default values have not been modified after open() assert db.defaults["age"] == None assert db.defaults["size"] == 300 assert db.defaults["afloat"] == 1.0 assert db.defaults["birth_hour"] == dtime(10,10,10) assert db.defaults["birth"] == date(1994,1,14) for i in range(5): # insert a list db.insert(random.choice(names), unicode(random.choice(fr_names),'latin-1'), random.randint(7,47),random.randint(110,175), date(random.randint(1958,1999),random.randint(1,12),10), random.uniform(-10**random.randint(-307,307), 10**random.randint(-307,307)), dtime(random.randint(0, 23), random.randint(0, 59), random.randint(0, 59))) db.insert(name=random.choice(names)) # missing fields for field in db.field_names[2:]: if field == "name": continue try: assert getattr(db[-1],field) == db.defaults[field] except: print "attribute %s not set to default value %s" %(field,db[-1]) raise # insert as string db.set_string_format(unicode,'latin-1') db.set_string_format(date,'%d-%m-%y') db.set_string_format(dtime,'%H-%M-%S') db.insert_as_strings(name="testname",fr_name=random.choice(fr_names), age=10,size=123,birth="07-10-95", birth_hour="20-53-3") assert db[-1].birth == date(1995,10,7) assert db[-1].name == "testname" assert db[-1].age == 10 assert db[-1].afloat == db.defaults["afloat"] db.insert_as_strings("testname",random.choice(fr_names), 11,134,"09-12-94",1.0, "5-6-13") assert db[len(db)-1].birth == date(1994,12,9) assert db[-1].name == "testname" assert db[-1].age == 11 assert db[-1].size == 134 assert db[-1].afloat == 1.0 # search between 2 dates print '\nBirth between 1960 and 1970' for r in db.select(None,birth=[date(1960,1,1),date(1970,12,13)]): print r.name,r.birth print "sorted" for r in db.select(None,birth=[date(1960,1,1),date(1970,12,13)]).sort_by('+name-birth'): print r.name,r.birth f = buzhug_files.FloatFile().to_block def all(v): return [ord(c) for c in f(v)] # search between 2 floats # selection by list comprehension s1 = [ r for r in db if 0.0 <= r.afloat <= 1e50 ] # selection by select s2 = db.select(None,'x<=afloat<=y',x=0.0,y=1e50) # selction by select with interval s3 = db.select(None,afloat=[0.0,1e50]) try: assert len(s1) == len(s2) == len(s3) except: print "%s records by list comprehension, " %len(s1) print "%s by select by formula," %len(s2) print "%s by select by interval" %len(s3) for r in s1: try: assert r in s2 except: print all(r.afloat) for r in s2: try: assert r in s1 except: print "in select but not in list comprehension",r raise r = db[0] assert r.__class__.db is db fr=random.choice(fr_names) s1 = [ r for r in db if r.age == 30 and r.fr_name == unicode(fr,'latin-1')] s2 = db.select(['name','fr_name'],age=30,fr_name = unicode(fr,'latin-1')) assert len(s1)==len(s2) # different ways to count the number of items assert len(db) == sum([1 for r in db]) == len(db.select(['name'])) # check if version number is correctly incremented for i in range(5): recs = db.select_for_update(['name'],'True') version = recs[0].__version__ recs[0].update() assert db[0].__version__ == version + 1 # check if cleanup doesn't change db length length_before = len(db) db.cleanup() assert len(db) == length_before # check if selection by select on __id__ returns the same as direct # access by id recs = db.select([],'__id__ == c',c=20) assert recs[0] == db[20] # check that has_key returns False for invalid hey assert not db.has_key(1000) # drop field db.drop_field('name') # check if field was actually removed from base definition and rows assert not "name" in db.fields assert not hasattr(db[20],"name") # add field db.add_field('name',str,default="marcel") # check if field was added with the correct default value assert "name" in db.fields assert hasattr(db[20],"name") assert db[20].name == "marcel" # change default value db.set_default("name","julie") db.insert(age=20) assert db[-1].name == "julie" # delete a record db.delete([db[10]]) # check if record has been deleted try: print db[10] raise Exception,"Row 10 should have been deleted" except IndexError: pass assert 10 not in db assert len(db) == length_before # selections # selection by generator expression # age between 30 et 32 d_ids = [] for r in [r for r in db if 33> r.age >= 30]: d_ids.append(r.__id__) length = len(db) # remove these items db.delete([r for r in db if 33> r.age >= 30]) # check if correct number of records removed assert len(db) == length - len(d_ids) # check if all records have been removed assert not [r for r in db if 33> r.age >= 30] # updates # select name = pierre s1 = db.select(['__id__','name','age','birth'],name='pierre') # make 'pierre' uppercase for record in db.select_for_update(None,'name == x',x='pierre'): db.update(record,name = record.name.upper()) # check if attribute was correctly updated for rec in s1: assert db[rec.__id__] == "Pierre" # increment ages for record in db.select_for_update([],'True'): age = record.age if not record.age is None: db.update(record,age = record.age+1) # check assert db[record.__id__].age == age + 1 for record in [r for r in db]: age = record.age if not record.age is None: db.update(record,age = record.age+1) # check assert db[record.__id__].age == age + 1 # change dates for record in db.select_for_update([],'age>v',v=35): db.update(record,birth = date(random.randint(1958,1999), random.randint(1,12),10)) db.commit() # check length after commit assert sum([1 for r in db]) == len(db) # insert new records for i in range(50): db.insert(name=random.choice(names), age=random.randint(7,47),size=random.randint(110,175)) # check that record 10 is still deleted try: print db[10] raise Exception,"Row 10 should have been deleted" except IndexError: pass print db.keys() print "has key 10 ?",db.has_key(10) assert 10 not in db #raw_input() # check that deleted_lines was cleared by commit() assert not db._pos.deleted_lines print db._del_rows.deleted_rows length = len(db) # before cleanup # physically remove the deleted items db.cleanup() # check that deleted_lines and deleted_rows are clean assert not db._pos.deleted_lines assert not db._del_rows.deleted_rows # check that record 10 is still deleted try: print db[10] raise Exception,"Row 10 should have been deleted" except IndexError: pass assert 10 not in db # check that length was not changed by cleanup assert len(db) == length assert len([ r for r in db]) == length # age > 30 for r in db.select(['__id__','name','age'], 'name == c1 and age > c2', c1 = 'pierre',c2 = 30): assert r.name == "pierre" assert r.age > 30 # name =="PIERRE" and age > 30 for r in db.select(['__id__','name','age','birth'], 'name == c1 and age > c2', c1 = 'PIERRE',c2 = 30): assert r.name == 'PIERRE' assert r.age > 30 # test with != for r in db.select(['__id__'],'name != c1',c1='claire'): assert r.name != 'claire' # age > id # with select s1 = db.select(['name','__id__','age'],'age > __id__') for r in s1: assert r.age > r.__id__ # with iter s2 = [ r for r in db if r.age > r.__id__ ] for r in s2: assert r.age > r.__id__ assert len(s1) == len(s2) # birth > date(1978,1,1) # with select s1 = db.select(['name','__id__','age'],'birth > v',v=date(1978,1,1)) for r in s1: assert r.birth > date(1978,1,1) # with iter s2 = [ r for r in db if r.birth and r.birth > date(1978,1,1) ] for r in s2: assert r.birth > date(1978,1,1) assert len(s1) == len(s2) # test with floats for i in range(10): x = random.uniform(-10**random.randint(-307,307), 10**random.randint(-307,307)) s1 = [ r for r in db if r.afloat > x ] s2 = db.select(['name'],'afloat > v',v=x) assert len(s1)==len(s2) # base with external link houses = Base('houses') houses.create(('address',str),('flag',bool),('resident',db,db[0]),mode="override") addresses = ['Giono','Proust','Mauriac','Gide','Bernanos','Racine', 'La Fontaine'] ks = db.keys() for i in range(50): x = random.choice(ks) address = random.choice(addresses) houses.insert(address=address,flag = address[0]>"H",resident=db[x]) # houses with jean s1 = [] for h in houses: if h.resident.name == 'jean': s1.append(h) # by select : ??? #s2 = houses.select([],'resident.name == v',v='jean') # assert len(s1) == len(s2) h1 = Base('houses') h1.open() l1 = len(h1.select([],flag=True)) l2 = len(h1.select([],flag=False)) assert l1 + l2 == len(h1) class DictRecord(Record): def __getitem__(self, k): item = self names = k.split('.') for name in names: item = getattr(item, name) return item h1.set_record_class(DictRecord) print '\nrecord_class = DictRecord, h1[0]' print h1[0] print "\nResident name: %(resident.name)s\nAddress: %(address)s" % h1[0]
class DB: def __init__(self, storage_path): self.dbh_objects = Base(os.path.join(storage_path, 'objects')) self.dbh_blocks = Base(os.path.join(storage_path, 'blocks')) self.dbh_replicas = Base(os.path.join(storage_path, 'replicas')) self.dbh_tree = Base(os.path.join(storage_path, 'tree')) self.dbh_paths = Base(os.path.join(storage_path, 'paths')) self.dbh_id = Base(os.path.join(storage_path, 'id')) self.dbh_tags = Base(os.path.join(storage_path, 'tags')) self.storage_path = storage_path def __create_root(self): """ Check if the filesystem has a / and if not create it""" print "Initializing filesystem..." if self.get_file(path='/'): return print "Creating root..." f = FSObject(1,1,'/',0,0,0,0) # lets see if we already have a key stored set = self.dbh_objects.select(['oid'],oid=str(f.oid)) if set == []: # we have create tree and paths first self.dbh_tree.insert(str(f.oid), str(f.parent)) self.dbh_paths.insert(str((f.parent, f.path))) self.dbh_objects.insert(str(f.oid), dumps(f), self.dbh_tree[len(self.dbh_tree)-1], self.dbh_paths[len(self.dbh_paths)-1]) #set the current oid for the id increment sequence set = self.dbh_id.select(['curr_oid']) if set == []: self.dbh_id.insert('1') def setup_fs_db(self): try: self.dbh_blocks.create(('key', str), ('blocks', str)) except IOError: self.dbh_blocks.open() try: self.dbh_replicas.create(('key', str), ('replicas', str)) except IOError: self.dbh_replicas.open() try: self.dbh_tree.create(('oid', str), ('parent', str)) except IOError: self.dbh_tree.open() try: self.dbh_tags.create(('oid', str), ('tag', str)) except IOError: self.dbh_tags.open() try: self.dbh_paths.create(('key', str)) except IOError: self.dbh_paths.open() try: self.dbh_id.create(('curr_oid', str)) except IOError: self.dbh_id.open() try: self.dbh_objects.create(('oid', str), ('fsobj', str), ('tree', self.dbh_tree), ('paths', self.dbh_paths)) except IOError: self.dbh_objects.open() self.__create_root() def get_path_oid(self, path): """Gets the parent filenode for path""" nodes = [] parent_path = path while 1: (parent_path,node) = os.path.split(parent_path) if node == '': nodes.insert(0,'/') break nodes.insert(0,node) parent_oid = 0 for node_name in nodes: key = str((parent_oid, node_name)) print "looking up: %s" % key # search for a match f = None for record in [record for record in self.dbh_objects]: if record.paths.key == key: f = loads(record.fsobj) break print "found it!" if not f: return 0 parent_oid = f.oid return parent_oid def insert_file(self, path, fsobj): #check first if there is a parent directory to store this file f = self.get_file(path=path) print "inserting file with path: "+path print fsobj if not f: print "ERR: [%s]" % os.path.split(fsobj.path)[0] raise FileSystemError('No parent directory to store: %s' % fsobj.path) #the parent of this object is the path fsobj.parent = f.oid set = self.dbh_id.select_for_update(['curr_oid']) curr_oid = int(set[0].curr_oid) + 1 fsobj.oid = curr_oid print "Inserting OID: %s" % fsobj # lets see if we already have a key stored result = self.dbh_objects.select(['oid','fsobj'],oid=str(fsobj.oid)) if result != []: raise FileSystemError('File already exists') else: # we have create tree and paths first self.dbh_tree.insert(str(fsobj.oid), str(fsobj.parent)) self.dbh_paths.insert(str((fsobj.parent, fsobj.path))) self.dbh_objects.insert(str(fsobj.oid), dumps(fsobj), self.dbh_tree[len(self.dbh_tree)-1], self.dbh_paths[len(self.dbh_paths)-1]) set[0].update(curr_oid=str(curr_oid)) return curr_oid def get_file(self, oid='', path=''): if oid: set = self.dbh_objects.select(['oid', 'fsobj'], oid=str(oid)) if set == []: f = None else: f = set[0].fsobj elif path: if path == '/': key = str((0,'/')) else: parent_oid = self.get_path_oid(os.path.split(path)[0]) node_name = os.path.split(path)[1] key = str((parent_oid, node_name)) # search for a match f = None for record in [record for record in self.dbh_objects]: print record.paths.key if record.paths.key == key: f = record.fsobj break else: f = None if f: f = loads(f) return f def get_children(self, oid): # lookup FSOBJECT with given oid set = self.dbh_objects.select(['oid', 'fsobj'], oid=str(oid)) if set == []: return [] file_array = [] # lookup objects with parent oid set = self.dbh_tree.select(['oid', 'parent'], parent=str(oid)) for i in set: obj = self.dbh_objects.select(['oid', 'fsobj'], oid=str(i.oid)) if obj != []: file_array.append(loads(obj[0].fsobj)) return file_array def debug_print_db(self, db): pass def print_object_db(self): self.debug_print_db(self.dbh_objects) def delete_dir(self,oid): pass def delete_file(self, oid): pass def rename_file(self,src,dest): pass def update_file(self, fsobj): set = self.dbh_objects.select_for_update(['oid', 'fsobj'], oid=str(fsobj.oid)) if set != []: set[0].update(fsobj=dumps(fsobj)) def add_block(self, block, serverid): f = self.get_file(oid=str(block.oid)) if not f: raise FileSystemError('add_block: Object %s does not exist' % block.oid) key = str((long(block.oid),long(block.block_id))) #the key is both the oid and the block_id set1 = self.dbh_replicas.select_for_update(['key', 'replicas'], key=key) if set1 == []: replicas = FSReplicas(block.oid, block.block_id) else: replicas = loads(set1[0].replicas) f.blocks[block.block_id] = block.version set2 = self.dbh_blocks.select_for_update(['key', 'blocks'], key=key) if set2 == []: b = None else: b = set2[0].block if b: b = loads(b) diff = block.size - b.size else: diff = block.size f.size += diff # update or insert? if set1 == []: self.dbh_blocks.insert(key, dumps(block)) else: set1[0].update(blocks=dumps(block)) self.update_file(f) replicas.add(serverid, block.version) # update or insert? if set2 == []: self.dbh_replicas.insert(key,dumps(replicas)) else: set2[0].update(replicas=dumps(replicas)) def add_block_replica(self, block, serverid): f = self.get_file(str(block.oid)) if not f: raise FileSystemError('add_block_replica: Object %s does not exist' % block.oid) key = str((block.oid, block.block_id)) set = self.dbh_replicas.select_for_update(['key', 'replicas'], key=key) if set == []: replicas = FSReplicas(block.oid, block.block_id) else: replicas = loads(set[0].replicas) replicas.add(serverid, block.version) # update or insert? if set == []: self.dbh_replicas.insert(key,dumps(replicas)) else: set[0].update(replicas=dumps(replicas)) def get_block_replicas(self, oid, block_id): key = str((long(oid), long(block_id))) set = self.dbh_replicas.select(['key', 'replicas'], key=key) if set == []: return None return loads(set[0].replicas) def get_block(self, oid, block_id): key = str((long(oid), long(block_id))) set = self.dbh_blocks.select(['key', 'blocks'], key=key) if set == []: return None return loads(set[0].blocks) def print_replicas_db(self): self.debug_print_db(self.dbh_replicas) def close_fs_db(self): self.dbh_blocks.close() self.dbh_replicas.close() self.dbh_tree.close() self.dbh_id.close() self.dbh_paths.close() self.dbh_objects.close()
class bookDBHandler: def __init__(self): self.userdir = os.getcwd() self.bookDB = Base(self.userdir + '/db/bookDB').open() #this selects the whole db, for now self.resultset = self.bookDB.select() #for loading the db in memory self.membooks = [] def loadDBIn(self): for r in self.resultset: #load specific book's json from bookDB jsonObj = json.loads(r.json) dbID = r.__id__ script_review_status = r.script_review_status #put all relavant info into convienient fields. #Im sure there is a quicker way involving commas #with Python, will look at later status = jsonObj.get('status') rating_count = jsonObj.get('rating_count') #slug = jsonObj.get('slug') #language = jsonObj.get('language') author = jsonObj.get('author') rating_value = jsonObj.get('rating_value') title = jsonObj.get('title') modified = jsonObj.get('modified') #bust = jsonObj.get('bust') reviewed = jsonObj.get('reviewed') audience = jsonObj.get('audience') tags = jsonObj.get('tags') created = jsonObj.get('created') link = jsonObj.get('link') author_id = jsonObj.get('author_id') iD = jsonObj.get('ID') typex = jsonObj.get('type') pages = jsonObj.get('pages') categories = jsonObj.get('categories') rating_total = jsonObj.get('rating_total') slug = r.slug text = [] #do the same for the pages for p in pages: text.append(p['text']) #nice little book object #print dbID #break elBook = book(dbID, status, rating_count, author, rating_value, title, modified, reviewed, audience, tags, created, link, author_id, iD, typex, pages, categories, rating_total, text, script_review_status, slug) self.membooks.append(elBook) def saveDB(self): #for now the only thing that #should be saved is the script_reviewed_status for b in self.membooks: self.resultset[b.dbID].script_review_status = b.script_review_status def closeDB(self): self.bookDB.cleanup() self.bookDB.close()