def welcome(): db = Base('alf.db') db.create('name','pwd',mode="open") #override user = request.form['Username'] passwd = request.form['password1'] db.insert(name=user,pwd=passwd) db.commit() return 'welcome ' + user
class KopeteLog(): def __init__(self, directory=None): if not directory: directory=os.path.join(os.path.expanduser("~"), ".kde/share/apps/kopete/logs") self.messages = Base('kopete.db') # Database stuff - Initializing... self.messages.create('protocol', 'date', 'time', 'msgfrom', 'msgto', 'sender', 'inbound', 'nick', 'message', mode='override') for file in self.searchLogs(directory): self.feedDatabase(file) def searchLogs(self, dir): logfiles = [] for (basepath, dirnames, filenames) in os.walk(dir): for child in filenames: if child.endswith (".xml"): logfiles.append(os.path.join(basepath, child)) return logfiles def feedDatabase(self, filepath): if 'WlmProtocol' in filepath: protocol = 'wlm' elif 'ICQProtocol' in filepath: protocol = 'icq' elif 'JabberProtocol' in filepath: protocol = 'jabber' else: protocol = 'unknown' xmllog = parse(filepath) for head in xmllog.getiterator('head'): for date in head.getiterator('date'): month=date.attrib['month'] year=date.attrib['year'] for contact in head.getiterator('contact'): if contact.attrib.has_key('type'): if contact.attrib['type'] == 'myself': contactfrom = contact.attrib['contactId'] else: contactto = contact.attrib['contactId'] for msg in xmllog.getiterator('msg'): nick = msg.attrib['nick'] time = msg.attrib['time'] inbound = msg.attrib['in'] message = msg.text sender = msg.attrib['from'] date = datetime.strptime("%s;%s;%s" % (year, month, msg.attrib['time']) , "%Y;%m;%d %H:%M:%S") self.messages.insert( protocol=protocol, date=date.strftime("%Y%m%d"), time=date.strftime("%H:%M:%S"), msgfrom=contactfrom, msgto=contactto, sender=sender, inbound=inbound, nick=nick, message=message)
class Queue(object): """ Simple queue with PyDbLite backend. """ def __init__(self, queue_type=0, config=None): if config is not None: # copy config pass else: self.commit = False self.db_filename = "/tmp/queue.pydb" self.mode = "override" self.queue = Base(self.db_filename) def create_queue(self): self.queue.create('id', 'item', mode = self.mode) self.queue.create_index('id') return None def push(self, item): self.queue.insert(self.length(), item) return None def pop(self): if not self.is_empty(): id = self.length() - 1 r = self.queue._id[id] self.queue.delete(r) return r else: return None def list(self): return self.queue.records def length(self): return len(self.queue) def is_empty(self): return self.length() == 0 def commit(self): if self.commit is True: self.queue.commit() return None
def makeDB(read, write, startTime = "2010-01-01 00:00:00", \ endTime = "2010-01-01 00:10:00"): db = Base(write) startTime = calc.datetonumber(startTime) endTime = calc.datetonumber(endTime) #Day comes from day of the week. It is a number from 0 to 6. #0 = Monday 6 = Sunday. db.create('sensor', 'date', 'weekday', 'index', mode="override") db.open() allData = {} for i in range(len(bbdata.allSensors)): s = bbdata.allSensors[i] data = [] print "Parsing sensor " + str(s) try: sString = read + "sensor" + str(s) + ".txt" f = open(sString).readlines() oldD = None for timeLine in f: tmp = timeLine.split() tmp = tmp[1] + " " + tmp[2] #tmp = tmp[0] + " " + tmp[1] d = datetime.datetime.strptime(tmp, "%Y-%m-%d %H:%M:%S") foo = calc.datetonumber(d) if foo >= startTime and foo <= endTime: data.append(calc.datetonumber(d)) if d.toordinal() != oldD: #Add to database db.insert(s, d.toordinal(), d.weekday(), len(data) - 1) oldD = d.toordinal() print " " + str(d) except Exception, e: print "Except:" + str(e) pass allData[s] = data
def makeDB(read, write, startTime = "2010-01-01 00:00:00", \ endTime = "2010-01-01 00:10:00"): db = Base(write) startTime = calc.datetonumber(startTime) endTime = calc.datetonumber(endTime) #Day comes from day of the week. It is a number from 0 to 6. #0 = Monday 6 = Sunday. db.create('sensor', 'date', 'weekday', 'index', mode="override") db.open() allData = {} for i in range(len(bbdata.allSensors)): s = bbdata.allSensors[i] data = [] print "Parsing sensor " + str(s) try: sString = read + "sensor" + str(s) + ".txt" f = open(sString).readlines() oldD = None for timeLine in f: tmp = timeLine.split() tmp = tmp[1] + " " + tmp[2] #tmp = tmp[0] + " " + tmp[1] d = datetime.datetime.strptime(tmp, "%Y-%m-%d %H:%M:%S") foo = calc.datetonumber(d) if foo >= startTime and foo <= endTime: data.append(calc.datetonumber(d)) if d.toordinal() != oldD: #Add to database db.insert(s, d.toordinal(), d.weekday(), len(data) - 1) oldD = d.toordinal() print " " + str(d) except Exception, e: print "Except:" + str(e) pass allData[s] = data
driver.find_element_by_id('fm1').submit() driver.find_element_by_id('tabLink_u1240l1s214').click() driver.find_element_by_id('formMenu:linknotes1').click() driver.find_element_by_id('_id137Pluto_108_u1240l1n228_50520_:tabledip:0:_id158Pluto_108_u1240l1n228_50520_').click() page = driver.find_element_by_id('_id111Pluto_108_u1240l1n228_50520_:tableel:tbody_element') i = 0 for item in page.text.splitlines( ): if item.endswith('20'): line = item.split(' ',1)[1].lstrip() note = line.rsplit(' ', 1)[1] field = line.rsplit(' ', 1)[0] courseindb = db("course")==field if (len(courseindb) == 0): db.insert(grade=note, course=field) graph.post(path='486181564779150/feed/', message='Nouvelle note : ' + field) db.commit() Freq = 2500 # Set Frequency To 2500 Hertz Dur = 1000 # Set Duration To 1000 ms == 1 second winsound.Beep(Freq,Dur) print "A new grade is available " + field + " : " + note else: for rec in courseindb: #only one if (rec["grade"] != note): #FB update Freq = 2500 # Set Frequency To 2500 Hertz Dur = 1000 # Set Duration To 1000 ms == 1 second winsound.Beep(Freq,Dur) print "A grade has just been updated for " + field + " : " + note
os.remove('blog') db = Base('blog').create('parent', 'title', 'author', 'text', 'date') db.create_index('parent') nbthreads = 200 for i in range(nbthreads): # generate thread author = 'pierre' title = sentence(10, 10) text = sentence(100, 10) date = datetime.datetime(random.randint(2004, 2006), random.randint(1, 12), random.randint(1, 28), random.randint(0, 23), random.randint(0, 59), random.randint(0, 59)) thread_id = db.insert(parent=-1, author=author, title=title, text=text, date=date) # generate comments nbcomments = random.randint(0, 5) for i in range(nbcomments): author = word(10) text = sentence(50, 10) tdelta = datetime.date(2007, 1, 1) - date.date() c_date = date + datetime.timedelta(random.randint(1, tdelta.days)) c_date = datetime.datetime(c_date.year, c_date.month, c_date.day, random.randint(0, 23), random.randint(0, 59), random.randint(0, 59)) db.insert(parent=thread_id,
def createBase(): """for path in ('chansons','recueils','dialectes', 'genres','chansons_par_dialecte', 'chansons_par_recueil'): try: os.remove(path+'.pdl') except OSError: pass""" txt_path = os.path.join(os.path.dirname(os.path.dirname(CWD)), "base.txt") chansons=open(txt_path).readlines() db_genres = Base(REL('genres.pdl')).create('nom',mode="override") db_chansons = Base(REL('chansons.pdl')).create('url','breton', 'francais','prix','genre',mode="override") db_recueils = Base(REL('recueils.pdl')).create('nom',mode="override") db_dialectes = Base(REL('dialectes.pdl')).create('nom',mode="override") db_ch_par_dial = Base(REL('chansons_par_dialecte.pdl')).create( 'chanson','dialecte',mode="override") db_ch_par_rec = Base(REL('chansons_par_recueil.pdl')).create( 'chanson','recueil',mode="override") l_chansons=[] id_chanson = 0 l_recueils=[] l_dialectes=[] l_genres=[] chansonsParRecueil=[] chansonsParGenre=[] chansonsParDialecte=[] for line in chansons: [url,breton,francais,recueils,genre,dialectes,enreg]=line.strip().split("#") if not genre in l_genres: l_genres.append(genre) id_genre = l_genres.index(genre) prix=random.randrange(200,400) l_chansons.append([url,breton,francais,prix,id_genre]) recs=recueils.split(";") for rec in recs: if not rec in l_recueils: l_recueils.append(rec) id_recueil = l_recueils.index(rec) chansonsParRecueil.append([id_chanson,id_recueil]) dials=dialectes.split(";") for dial in dials: if not dial in l_dialectes: l_dialectes.append(dial) id_dialecte = l_dialectes.index(dial) chansonsParDialecte.append([id_chanson, id_dialecte]) id_chanson += 1 for g in l_genres: db_genres.insert(nom=g) for d in l_dialectes: db_dialectes.insert(nom=d) for r in l_recueils: db_recueils.insert(nom=r) for ch in l_chansons: db_chansons.insert(**dict(zip(db_chansons.fields,ch))) for ch_d in chansonsParDialecte: db_ch_par_dial.insert(chanson = ch_d[0], dialecte = ch_d[1]) for ch_r in chansonsParRecueil: db_ch_par_rec.insert(chanson = ch_r[0], recueil = ch_r[1]) for db in (db_genres,db_dialectes,db_recueils,db_chansons, db_ch_par_dial,db_ch_par_rec): db.commit()
class TagOrganizer(Organizer): def __init__(self, cache, category=None): self.tags = None self.category = category Organizer.__init__(self, cache, False) def reset(self): if not self.tags: self.tags = Base(DB_FILE_TAGS) self.tags.create('realpath', 'category', 'tag', mode = 'override') self.tags.create_index('realpath') self.tags.create_index('category') Organizer.reset(self) def updatecache(self): self._generatetags() Organizer.updatecache(self) def _deletefromcache(self, path): realpath = self.realpath(path) logger.debug("_deletefromcache(%s)" % realpath) for tag in self.tags.get_index('realpath')[realpath]: self.tags.delete(tag) def deletefromcache(self, path): self._deletefromcache(path) Organizer.deletefromcache(self, path) def addtocache(self, path): self._deletefromcache(path) self.generatetags(self.realpath(path)) Organizer.addtocache(self, path) def generatepaths(self, realpath): for record in self.tags.get_index('realpath')[realpath]: yield os.path.join(os.sep, record['tag'], os.path.basename(realpath)) def dirlist(self, path): if path == '/': return self.taglist(self.category) else: return [] ############################################ # Tag functions def _generatetags(self): for filename in filter(util.ignoretag, #IGNORE:W0141 self.cache.filelist()): self.generatetags(filename) def generatetags(self, filename): pass def tag(self, realpath, category, tag): logger.debug('tag(%s, %s, %s)' % (realpath, category, tag)) if not tag == None and not tag == '': self.tags.insert(realpath, category, tag) def filelistbytags(self, category, tags): self.refreshcache() for record in self.tags.get_index('category')[category]: if record['tag'] in tags: yield os.path.basename(record['realpath']) def taglist(self, category): self.refreshcache() return util.unique([record['tag'] for record in self.tags.get_index('category')[category]])
class Organizer(Cacheable): """ This is the base class for organizers """ def __init__(self, cache, recursive=True): Cacheable.__init__(self) self.cache = cache self.recursive = recursive self.transformed = None # Do not call reset here, it is called from fs.py when the fs is # already started def reset(self): if not self.transformed: self.transformed = Base(DB_TRANSFORMED) self.transformed.create('realpath', 'path', 'dirname', mode='override') self.transformed.create_index('realpath') self.transformed.create_index('path') self.transformed.create_index('dirname') self.cache.reset() Cacheable.reset(self) def updatecache(self): self.generateallpaths() def deletefromcache(self, path): realpath = self.realpath(path) logger.debug("deletefromcache(%s)" % realpath) self.cache.deletefromcache(realpath) for item in self.transformed.get_index('realpath')[realpath]: self.transformed.delete(item) def addtocache(self, path): if not self.transformed.get_index('path')[path]: realpath = self.realpath(path) self.cache.addtocache(realpath) self.addfile(realpath) ############################################ # Overwritable functions def dirlist(self, path): #IGNORE:W0613 """ Returns a list of (non-existent, generated, virtual) directories for a given path. Default implementation. """ return [] def generatepaths(self, realpath): """ Generates paths for a given real path. A file can have more than one transformed path. Default implementation. """ yield util.addtrailingslash( util.removeroot(realpath, self.cache.filter.root)) def generaterealpath(self, path): """ Generates a real path for a inexistent path. Default implementation. """ return os.path.join(self.cache.filter.root, path[1:]) ############################################ # General functions def generateallpaths(self): """ Generates paths for all the files given by the cache and stores them in self.transformed """ for realpath in self.cache.filelist(): if self.recursive: # Add all sub-directories first currentpath = self.cache.filter.root for pathpart in util.pathparts( util.removeroot(realpath, self.cache.filter.root)): currentpath = os.path.join(currentpath, pathpart) self.addfile(currentpath) else: self.addfile(realpath) def addfile(self, realpath): """ Stores a file in self.transformed if not there already and returns the paths for that file in the proxy file system """ logger.debug('addfile(%s)' % realpath) if not util.ignoretag(util.removeroot(realpath, self.cache.filter.root)): return [] self.refreshcache() transformed = self.transformed.get_index('realpath')[realpath] if transformed: return (record['path'] for record in transformed) else: paths = [] for path in self.paths(realpath): while self.transformed.get_index('path')[path]: path = self.increasefilename(path) dirname = os.path.dirname(path) logger.debug('addfile(%s, %s, %s)' % (realpath, path, dirname)) self.transformed.insert(realpath=realpath, path=path, dirname=dirname) paths.append(path) return paths def increasefilename(self, filename): """ Returns a new filename in sequence. Called if the current filename already exists. This default implementation adds a "(1)" to the end if not present or increases that number by one. """ root, ext = os.path.splitext(filename) num = 1 matches = _INCREASE_REGEX.match(root) if not matches is None: num = int(matches.group(2)) + 1 filename = matches.group(1) return '%s(%i)%s' % (root, num, ext) ############################################ # General functions that read the cache def filelist(self, path): """ Returns a list of directories and filenames in a list from cache """ logger.debug('filelist(%s)' % path) self.refreshcache() for dirname in self.dirlist(path): yield dirname for record in self.transformed.get_index('dirname')[path]: yield os.path.basename(record['path']) def paths(self, realpath): """ Generates or returns paths from cache for a given real path """ self.refreshcache() paths = self.transformed.get_index('realpath')[realpath] if paths: return (path['path'] for path in paths) else: return (path for path in self.generatepaths(realpath)) def realpath(self, path): """ Returns the real path for a file given the path in the file system. """ logger.debug('realpath(%s)' % path) self.refreshcache() realpaths = [ r['realpath'] for r in self.transformed.get_index('path')[path] ] realpath = None if realpaths: realpath = realpaths[0] elif path == '/': realpath = self.cache.filter.root elif path == util.addtrailingslash(util.ORIGINAL_DIR): realpath = '.' elif util.isspecial(path, 'original', True): realpath = os.path.join('.', os.sep.join(util.pathparts(path)[2:])) elif util.isspecial(path, 'root', True): realpath = os.path.join(self.cache.filter.root, os.sep.join(util.pathparts(path)[2:])) elif util.isspecial(path, 'commands'): realpath = '.' elif util.iscommand(path): realpath = getserver().tempfile.name else: realpath = self.generaterealpath(path) logger.debug('realpath(%s) = %s' % (path, realpath)) return realpath ############################################ # File system functions def getattr(self, path): dirname = os.path.dirname(path) if util.removeroot(path, os.sep) in self.dirlist(dirname): return self.cache.getattr(self.realpath(dirname)) else: return self.cache.getattr(self.realpath(path)) def readdir(self, path, offset): #IGNORE:W0613 for filename in util.getbasefilelist(): yield fuse.Direntry(filename) for filename in self._filelist(path): yield fuse.Direntry(filename) def _filelist(self, path): filelist = [] if path == util.addtrailingslash(util.ORIGINAL_DIR): filelist = ['original', 'root', 'commands'] elif util.isspecial(path, 'root', True): filelist = os.listdir(self.realpath(path)) elif util.isspecial(path, 'original', True): filelist = os.listdir(self.realpath(path)) elif util.isspecial(path, 'commands'): filelist = CommandHandler.COMMANDS else: filelist = self.filelist(path) for filename in filelist: yield filename
class EventPassportOffice: #what do we need in init? #pressure run ID number #acoustic ID number #(btw marking those separate is a bad idea on the operators part) def __init__(self): self.EventPassport = Base('EventPassport/EventPassport.pdl') #check if the DB exists. If Yes, open, if not #create it: if not self.EventPassport.exists(): self.genPDL() else: self.EventPassport.open() self.CleanEvents = CleanEvents.CleanData() def genPDL(self): #Create the PDL file for database self.EventPassport.create('EventID','Temperature','Pressure','Time', 'RunNumber','Path', 'RunType', mode = "open") #RunNumber is defined as RunNumberAcoustic #Runtype can be neutron or alpha def genPassport(self, Path, RunNumberAcoustic, RunNumberPressure, RunType_WS): FilteredData = self.CleanEvents.MatchEvent_PressurePiezo(Path, str(RunNumberAcoustic), str(RunNumberPressure)) #Get the last EventID recs = [ Record['EventID'] for Record in self.EventPassport if Record['RunNumber'] == RunNumberAcoustic] if len(recs) == 0: EID = str(RunNumberAcoustic)+"0001" EID = int(EID) else: EID = max(recs)+1 #check if we have a duplicate! for DataPoint in FilteredData: timestamp = DataPoint[1] #Check if we have a dupe/conflict x = [Event for Event in self.EventPassport if Event['Time']-timedelta(seconds=2)<=timestamp<=Event['Time']+timedelta(seconds=2)] if len(x) == 0: self.EventPassport.insert(EventID = EID ,Temperature = DataPoint[3],Pressure = DataPoint[2],Time = DataPoint[1], RunNumber = RunNumberAcoustic, Path = DataPoint[0], RunType = RunType_WS) EID += 1 print("Inserting Entry ...") else: print "Duplicate entry found at: "+str(DataPoint[1])+" Event ID: "+str(x[0]['EventID']) self.EventPassport.commit() def CheckPassport_RunNumber(self, RunNumberQry): return self.EventPassport(RunNumber = RunNumberQry) def CheckPassport_Temperature(self, HighTemp, LowTemp): return self.EventPassport(HighTemp>Temperature>LowTemp) def CheckPassport_Time(self, fromTime, toTime): recs = [ r for r in self.EventPassport if fromTime < r['Time'] < toTime] return recs def SizeofPassportDB(self): return len(self.EventPassport) def CheckPassport_Runtype(self, runtype_WS): return self.EventPassport(RunType = runtype_WS) def CheckPassport_eventID(self, EventID_WS): return self.EventPassport(EventID = EventID_WS) def _deleteEvent(self, RecID_WS): del self.EventPassport[RecID_WS] self.EventPassport.commit()
class PressureVeto: #We need the run number for init. We will use PyDBLite #so we need to gen the db first. There will be another #function for that. The reason we use this is because #of native python compatibility def __init__(self, RunNumber): #property self.RunNumber assigned. #This is typecasted to string for manipulation self.RunNumber = str(RunNumber) #property self.PyDB -> Database for pressures self.PyDB = Base('pressures/'+self.RunNumber+'.dbl') #check if the DB exists. If Yes, open, if not #create it: if not self.PyDB.exists(): self.genPDL() else: self.PyDB.open() #Define the time iteration between bubbles minimum threshold #Remember, each iteration is 1/10th second! #Iter must be integer! minSecondsBetweenBubbles = 4 self.minIterBetweenBubbles = int(minSecondsBetweenBubbles*10) #Funtion to generate PyDBLite database #I will deliberately not give this MySQL abilities #since I dont want my data wiped out by "mistake" #The human veto has to be in here somewhere. def genPDL(self): #Create the PDL file for database self.PyDB.create('id','temp','pressure','time', mode = "override") #import CSV for CSV file ops. Import ONLY if needed, so its here. import csv #filename in CSV file. Assumption -> RunID.csv fname_csv = self.RunNumber+".csv" PTcsv = csv.reader(open(fname_csv)) #convert CSV to PyDB line by line for line in PTcsv: self.PyDB.insert(id = int(line[0]),temp=float(line[1]), pressure=float(line[2]), time=datetime.strptime(line[3], "%Y-%m-%d %H:%M:%S")) #Commit the database self.PyDB.commit() #Print a confirmation print "Creating PyDB complete." #this function finds the "peaks" in the pressures. #Criterion: Peaks are above 30 PSI def findBubbleTimings(self): '''Finds the bubble timings In -> Pressure data Out -> Timings (datetime.datetime) Assumptions -> Bubble PSI > 30 PSI ''' #Select records with pressure > 30.0 PSI recs = [r for r in self.PyDB] #Make an iterator of this list RecIter = itertools.islice(recs, None) #Declare memory space for: #Valid Bubbles #Temporary Storage #Last Record's ID (to stop Iterator) ValidBubbles = [] _VBubbleAmpTemporaryStorage = [] RecLastID = recs[-1:][0]['__id__'] #Go record by record: for record in RecIter: #If pressure > 30: if record['pressure'] >= 30.0: #Assign the temporary memory with present pressure, time _VBubbleAmpTemporaryStorage = [record['pressure'], record['time'], record['temp']] #Number of steps to iter so we dont go beyond the last rec stepsTillLastRec = RecLastID - record['__id__'] stepsIter = self.minIterBetweenBubbles if ( stepsTillLastRec > self.minIterBetweenBubbles) else stepsTillLastRec #Investigate for next minIterBetweenBubbles for a maxima for i in xrange(stepsIter): #Progress iterator by 1 record = RecIter.next() #is present iteration > memory stored variable? Yes: Store it, No: Continue _VBubbleAmpTemporaryStorage = [record['pressure'], record['time'], record['temp']] if record['pressure']>=_VBubbleAmpTemporaryStorage else _VBubbleAmpTemporaryStorage #The local maxima is found, store it as good data, continue searching ValidBubbles.append(_VBubbleAmpTemporaryStorage) #clear the temporary space _VBubbleAmpTemporaryStorage = [] #Return the time cut! return ValidBubbles
class BARTSIDEE_CONFIG: def __init__(self): self.path_temp = mc.GetTempDir() self.path_module = os.path.join(CWD, "modules") if not os.path.exists(self.path_module): os.makedirs(self.path_module) sys.path.append(self.path_module) self.app_version = VERSION self.db_version = 11 self.initDB() GA.debug = self.get("debug") def initDB(self): self.db = Base("maindb") self.db.create("id", "data", mode="open") self.searchdb = Base("searchdb") self.searchdb.create("module", "id", "timestamp", "data", mode="open") self.cache = tools.storage() try: if len(self.db) < 3: self.default() except: self.default() records = self.db(id="version") if records[0]["data"] < self.db_version: self.default() def get(self, key): records = self.db(id=key) if records: return records[0]["data"] else: return False def set(self, key, data): records = self.db(id=key) if records: self.db.update(records, data=data) else: self.db.insert(id=key, data=data) self.db.commit() def default(self): self.clearDB() try: pointer = os.path.join(CWD, "settings.json") file = open(pointer, "r") defaults = json.load(file) file.close() except: print traceback.format_exc() defaults = {} for key in defaults.keys(): self.db.insert(str(key), defaults[key]) self.db.insert("version", self.db_version) self.db.commit() def clearDB(self): try: records = self.db() self.db.delete(records) except: self.db.create("id", "data", mode="override") self.db.commit() def clearCache(self): self.cache.empty() def clearSearch(self): try: records = self.searchdb() self.searchdb.delete(records) except: self.searchdb.create("module", "id", "timestamp", "data", mode="override") self.searchdb.commit() def clearAll(self): self.default() self.clearCache() self.clearSearch()
class TagOrganizer(Organizer): def __init__(self, cache, category=None): self.tags = None self.category = category Organizer.__init__(self, cache, False) def reset(self): if not self.tags: self.tags = Base(DB_FILE_TAGS) self.tags.create('realpath', 'category', 'tag', mode='override') self.tags.create_index('realpath') self.tags.create_index('category') Organizer.reset(self) def updatecache(self): self._generatetags() Organizer.updatecache(self) def _deletefromcache(self, path): realpath = self.realpath(path) logger.debug("_deletefromcache(%s)" % realpath) for tag in self.tags.get_index('realpath')[realpath]: self.tags.delete(tag) def deletefromcache(self, path): self._deletefromcache(path) Organizer.deletefromcache(self, path) def addtocache(self, path): self._deletefromcache(path) self.generatetags(self.realpath(path)) Organizer.addtocache(self, path) def generatepaths(self, realpath): for record in self.tags.get_index('realpath')[realpath]: yield os.path.join(os.sep, record['tag'], os.path.basename(realpath)) def dirlist(self, path): if path == '/': return self.taglist(self.category) else: return [] ############################################ # Tag functions def _generatetags(self): for filename in filter( util.ignoretag, #IGNORE:W0141 self.cache.filelist()): self.generatetags(filename) def generatetags(self, filename): pass def tag(self, realpath, category, tag): logger.debug('tag(%s, %s, %s)' % (realpath, category, tag)) if not tag == None and not tag == '': self.tags.insert(realpath, category, tag) def filelistbytags(self, category, tags): self.refreshcache() for record in self.tags.get_index('category')[category]: if record['tag'] in tags: yield os.path.basename(record['realpath']) def taglist(self, category): self.refreshcache() return util.unique([ record['tag'] for record in self.tags.get_index('category')[category] ])
from PyDbLite import Base db = Base() db.create("Acc", "Gyro") db.insert(Acc="136", Gyro="162") print(db)
db.create_index("parent") nbthreads = 200 for i in range(nbthreads): # generate thread author = "pierre" title = sentence(10, 10) text = sentence(100, 10) date = datetime.datetime( random.randint(2004, 2006), random.randint(1, 12), random.randint(1, 28), random.randint(0, 23), random.randint(0, 59), random.randint(0, 59), ) thread_id = db.insert(parent=-1, author=author, title=title, text=text, date=date) # generate comments nbcomments = random.randint(0, 5) for i in range(nbcomments): author = word(10) text = sentence(50, 10) tdelta = datetime.date(2007, 1, 1) - date.date() c_date = date + datetime.timedelta(random.randint(1, tdelta.days)) c_date = datetime.datetime( c_date.year, c_date.month, c_date.day, random.randint(0, 23), random.randint(0, 59), random.randint(0, 59) ) db.insert(parent=thread_id, author=author, title=title, text=text, date=c_date) db.commit()
class KopeteLog(): def __init__(self, directory=None): if not directory: directory = os.path.join(os.path.expanduser("~"), ".kde/share/apps/kopete/logs") self.messages = Base('kopete.db') # Database stuff - Initializing... self.messages.create('protocol', 'date', 'time', 'msgfrom', 'msgto', 'sender', 'inbound', 'nick', 'message', mode='override') for file in self.searchLogs(directory): self.feedDatabase(file) def searchLogs(self, dir): logfiles = [] for (basepath, dirnames, filenames) in os.walk(dir): for child in filenames: if child.endswith(".xml"): logfiles.append(os.path.join(basepath, child)) return logfiles def feedDatabase(self, filepath): if 'WlmProtocol' in filepath: protocol = 'wlm' elif 'ICQProtocol' in filepath: protocol = 'icq' elif 'JabberProtocol' in filepath: protocol = 'jabber' else: protocol = 'unknown' xmllog = parse(filepath) for head in xmllog.getiterator('head'): for date in head.getiterator('date'): month = date.attrib['month'] year = date.attrib['year'] for contact in head.getiterator('contact'): if contact.attrib.has_key('type'): if contact.attrib['type'] == 'myself': contactfrom = contact.attrib['contactId'] else: contactto = contact.attrib['contactId'] for msg in xmllog.getiterator('msg'): nick = msg.attrib['nick'] time = msg.attrib['time'] inbound = msg.attrib['in'] message = msg.text sender = msg.attrib['from'] date = datetime.strptime( "%s;%s;%s" % (year, month, msg.attrib['time']), "%Y;%m;%d %H:%M:%S") self.messages.insert(protocol=protocol, date=date.strftime("%Y%m%d"), time=date.strftime("%H:%M:%S"), msgfrom=contactfrom, msgto=contactto, sender=sender, inbound=inbound, nick=nick, message=message)
class Organizer(Cacheable): """ This is the base class for organizers """ def __init__(self, cache, recursive=True): Cacheable.__init__(self) self.cache = cache self.recursive = recursive self.transformed = None # Do not call reset here, it is called from fs.py when the fs is # already started def reset(self): if not self.transformed: self.transformed = Base(DB_TRANSFORMED) self.transformed.create('realpath', 'path', 'dirname', mode='override') self.transformed.create_index('realpath') self.transformed.create_index('path') self.transformed.create_index('dirname') self.cache.reset() Cacheable.reset(self) def updatecache(self): self.generateallpaths() def deletefromcache(self, path): realpath = self.realpath(path) logger.debug("deletefromcache(%s)" % realpath) self.cache.deletefromcache(realpath) for item in self.transformed.get_index('realpath')[realpath]: self.transformed.delete(item) def addtocache(self, path): if not self.transformed.get_index('path')[path]: realpath = self.realpath(path) self.cache.addtocache(realpath) self.addfile(realpath) ############################################ # Overwritable functions def dirlist(self, path): #IGNORE:W0613 """ Returns a list of (non-existent, generated, virtual) directories for a given path. Default implementation. """ return [] def generatepaths(self, realpath): """ Generates paths for a given real path. A file can have more than one transformed path. Default implementation. """ yield util.addtrailingslash(util.removeroot(realpath, self.cache.filter.root)) def generaterealpath(self, path): """ Generates a real path for a inexistent path. Default implementation. """ return os.path.join(self.cache.filter.root, path[1:]) ############################################ # General functions def generateallpaths(self): """ Generates paths for all the files given by the cache and stores them in self.transformed """ for realpath in self.cache.filelist(): if self.recursive: # Add all sub-directories first currentpath = self.cache.filter.root for pathpart in util.pathparts(util.removeroot(realpath, self.cache.filter.root)): currentpath = os.path.join(currentpath, pathpart) self.addfile(currentpath) else: self.addfile(realpath) def addfile(self, realpath): """ Stores a file in self.transformed if not there already and returns the paths for that file in the proxy file system """ logger.debug('addfile(%s)' % realpath) if not util.ignoretag(util.removeroot(realpath, self.cache.filter.root)): return [] self.refreshcache() transformed = self.transformed.get_index('realpath')[realpath] if transformed: return (record['path'] for record in transformed) else: paths = [] for path in self.paths(realpath): while self.transformed.get_index('path')[path]: path = self.increasefilename(path) dirname = os.path.dirname(path) logger.debug('addfile(%s, %s, %s)' % (realpath, path, dirname)) self.transformed.insert(realpath=realpath, path=path, dirname=dirname) paths.append(path) return paths def increasefilename(self, filename): """ Returns a new filename in sequence. Called if the current filename already exists. This default implementation adds a "(1)" to the end if not present or increases that number by one. """ root, ext = os.path.splitext(filename) num = 1 matches = _INCREASE_REGEX.match(root) if not matches is None: num = int(matches.group(2)) + 1 filename = matches.group(1) return '%s(%i)%s' % (root, num, ext) ############################################ # General functions that read the cache def filelist(self, path): """ Returns a list of directories and filenames in a list from cache """ logger.debug('filelist(%s)' % path) self.refreshcache() for dirname in self.dirlist(path): yield dirname for record in self.transformed.get_index('dirname')[path]: yield os.path.basename(record['path']) def paths(self, realpath): """ Generates or returns paths from cache for a given real path """ self.refreshcache() paths = self.transformed.get_index('realpath')[realpath] if paths: return (path['path'] for path in paths) else: return (path for path in self.generatepaths(realpath)) def realpath(self, path): """ Returns the real path for a file given the path in the file system. """ logger.debug('realpath(%s)' % path) self.refreshcache() realpaths = [r['realpath'] for r in self.transformed.get_index('path')[path]] realpath = None if realpaths: realpath = realpaths[0] elif path == '/': realpath = self.cache.filter.root elif path == util.addtrailingslash(util.ORIGINAL_DIR): realpath = '.' elif util.isspecial(path, 'original', True): realpath = os.path.join('.', os.sep.join(util.pathparts(path)[2:])) elif util.isspecial(path, 'root', True): realpath = os.path.join(self.cache.filter.root, os.sep.join(util.pathparts(path)[2:])) elif util.isspecial(path, 'commands'): realpath = '.' elif util.iscommand(path): realpath = getserver().tempfile.name else: realpath = self.generaterealpath(path) logger.debug('realpath(%s) = %s' % (path, realpath)) return realpath ############################################ # File system functions def getattr(self, path): dirname = os.path.dirname(path) if util.removeroot(path, os.sep) in self.dirlist(dirname): return self.cache.getattr(self.realpath(dirname)) else: return self.cache.getattr(self.realpath(path)) def readdir(self, path, offset): #IGNORE:W0613 for filename in util.getbasefilelist(): yield fuse.Direntry(filename) for filename in self._filelist(path): yield fuse.Direntry(filename) def _filelist(self, path): filelist = [] if path == util.addtrailingslash(util.ORIGINAL_DIR): filelist = ['original', 'root', 'commands'] elif util.isspecial(path, 'root', True): filelist = os.listdir(self.realpath(path)) elif util.isspecial(path, 'original', True): filelist = os.listdir(self.realpath(path)) elif util.isspecial(path, 'commands'): filelist = CommandHandler.COMMANDS else: filelist = self.filelist(path) for filename in filelist: yield filename
def createBase(): for path in ('chansons','recueils','dialectes', 'genres','chansons_par_dialecte', 'chansons_par_recueil'): try: os.remove(path+'.skl') except OSError: pass db_genres = Base('genres.skl').create('nom') db_chansons = Base('chansons.skl').create('url','breton', 'francais','prix','genre') db_recueils = Base('recueils.skl').create('nom') db_dialectes = Base('dialectes.skl').create('nom') db_ch_par_dial = Base('chansons_par_dialecte.skl').create( 'chanson','dialecte') db_ch_par_rec = Base('chansons_par_recueil.skl').create( 'chanson','recueil') chansons=open("base.txt").readlines() l_chansons=[] id_chanson = 0 l_recueils=[] l_dialectes=[] l_genres=[] chansonsParRecueil=[] chansonsParGenre=[] chansonsParDialecte=[] for line in chansons: [url,breton,francais,recueils,genre,dialectes,enreg]=line.strip().split("#") if not genre in l_genres: l_genres.append(genre) id_genre = l_genres.index(genre) prix=random.randrange(200,400) l_chansons.append([url,breton,francais,prix,id_genre]) recs=recueils.split(";") for rec in recs: if not rec in l_recueils: l_recueils.append(rec) id_recueil = l_recueils.index(rec) chansonsParRecueil.append([id_chanson,id_recueil]) dials=dialectes.split(";") for dial in dials: if not dial in l_dialectes: l_dialectes.append(dial) id_dialecte = l_dialectes.index(dial) chansonsParDialecte.append([id_chanson, id_dialecte]) id_chanson += 1 for g in l_genres: db_genres.insert(nom=g) for d in l_dialectes: db_dialectes.insert(nom=d) for r in l_recueils: db_recueils.insert(nom=r) for ch in l_chansons: print ch db_chansons.insert(**dict(zip(db_chansons.fields,ch))) for ch_d in chansonsParDialecte: db_ch_par_dial.insert(chanson = ch_d[0], dialecte = ch_d[1]) for ch_r in chansonsParRecueil: db_ch_par_rec.insert(chanson = ch_r[0], recueil = ch_r[1]) for db in (db_genres,db_dialectes,db_recueils,db_chansons, db_ch_par_dial,db_ch_par_rec): db.commit()