Пример #1
0
		for a in soup:
			url ="http://www.dpchallenge.com"+str(a['href'])
			if(a.string != None):
				title=unicodedata.normalize('NFKD',a.string).encode('ascii','ignore')
				imagesDB.insert(title, str(url), 0.0, "")

	records = imagesDB().sort_by("+__id__")
	for record in imagesDB():
		print "working on image ",record.__id__
		try:
			imagePage = urllib2.urlopen(str(record.url)).read()
			img_soup=BeautifulSoup(imagePage)
	
			for child in img_soup.find(id="img_container").contents:
					if child.has_key('alt') and child['alt']==record.title: 
						imagesDB.update(record,url=str(child['src']))
						break
	
			tag = img_soup.find(text=re.compile("all users"))
			record.update(score=float(tag.next_element))
	
			img_url = record.url
			img_req = Request(img_url)
			ext=os.path.splitext(img_url)[1]
			try:
					f = urlopen(img_req)
					imgFile='%0*d' % (6,record.__id__)
					imgFile=os.getcwd()+"\\Images\\training\\" + imgFile + ext
	
					local_file = open(imgFile,"wb")
					local_file.write(f.read())
Пример #2
0
def main():
	AestheticScorer.LoadModel()
	
	#Make the database. We need tables for events, locations, photos, people, and faces
	events = Base(os.getcwd()+"\\Databases\\events.db")
	events.create(('name',str),('firsttime',str),('lasttime',str),mode="override")

	locations = Base(os.getcwd()+"\\Databases\\locations.db")
	locations.create(('lat',float),('lon',float),('name',str),mode="override")

	photos = Base(os.getcwd()+"\\Databases\\photos.db")
	photos.create(('path',str),('timestamp',str),('aestheticscore',int),('locationid',int), ('eventid',int), ("width", int), ("height", int),mode="override")

	people = Base(os.getcwd()+"\\Databases\\people.db")
	people.create(('name',str),('age',int),mode="override")

	faces = Base(os.getcwd()+"\\Databases\\faces.db")
	faces.create(('photoid',int),('thumbnail',str),('personid',int),('x',int),('y',int),('w',int),('h',int),mode="override")

	# Walk through all the directories, making a list of photos, geotagging the lowest level subdirectories, and making
	# a master list of the photos and geotags

	photolist = []
	geotaglist = []
	print "geocoding directories"
	for dirname, dirnames, filenames in os.walk(os.getcwd()+"\\Images\\photos"):
		#geocode all the subdirectory names
		for subdirname in dirnames:
			n,lat,long =  geotag.findlocation(subdirname)

			#if we have a problem geotagging, prompt the user for a different location name
			while n == "NONE":
				newname = raw_input("We couldn't find the location '" + subdirname + "'. Type in another place name to try.")
				n, lat, long = geotag.findlocation(subdirname)

			#once we have a valid location, insert it if it's not already in the database
			if not locations(name=n):
				locations.insert(float(lat), float(long), n)

		#add all the files to a file list, and go ahead and make a parallel geotags
		for filename in filenames:
			#print "filename is ",filename
			if filename[-3:] == "jpg" or filename[-3:] == "JPG":
				#find the id for that subdirname in the database so we can geotag it
				locname = dirname[dirname.rfind('\\') + 1:]
				#location = locations(name=dirname)
				photolist.append(os.path.join(dirname,filename))
				geotaglist.append((os.path.join(dirname,filename),locname))

	#make a list to identify which event each photo is in

	#print photolist
	print "getting events"
	eventLabels, uniqueEvents = timecluster.eventCluster(photolist)
	#print "events: "
	#print eventLabels
	print uniqueEvents

	#insert the events into the event database
	for label in uniqueEvents:
		events.insert(label[1],"","")

	#the events are already sorted according to photo names
	
	
	
	#now sort the geotags and photolist according to photo names as well, so we'll have parallel lists
	geotaglist.sort()
	photolist.sort()
	
	#now we can finally insert each photo, with a name, event, and geotag
	for i in range(len(photolist)):
		print i, photolist[i]
		width, height = Image.open(photolist[i]).size
		photos.insert(photolist[i],eventLabels[i][1],
		 AestheticScorer.getScore(photolist[i]),
		 locations(name=geotaglist[i][1])[0].__id__,
		 eventLabels[i][0], int(width), int(height))
	print "finding faces"
	#for all the images we just gathered, find the people and faces, and insert them into the database
	facelist = []
	for file in photolist:
		facelist.append(detectfaces.detectFaceInImage(file))

	faceimages, projections, imgs, minFaces = detectfaces.faceLBP(facelist)#detectfaces.facePCA(facelist)
	labels, nclusters = detectfaces.gMeansCluster(projections, minFaces)
	#detectfaces.visualizeResults(faceimages, labels, nclusters)

	#add the individuals we found in the photos into the people database
	i = 0
	for i in range(0,nclusters):
		people.insert(str(i),0)

	#add the faces, linking them to the individuals
	faceindex = 0
	photoindex = 0
	for listing in facelist:
		facerects = listing[1:]
		for entry in facerects:
			faces.insert(photoindex,imgs[faceindex],people[labels[faceindex]].__id__,entry[0],entry[1],entry[2],entry[3])
			faceindex = faceindex + 1
		photoindex = photoindex + 1
	
	print "updating events"
	#update the events table to include tthe start time
	eventtimes=[]
	for event in events.select(None).sort_by("+__id__"):
		orderedPhotos = photos.select(eventid=event.__id__).sort_by("+timestamp")
		events.update(event,firsttime=orderedPhotos[0].timestamp)
		events.update(event,lasttime=orderedPhotos[-1].timestamp)
	
		
		
	print "Done"
Пример #3
0
class TransactionsDB(object):

    BASE = 'banan/transactions'

    def __init__(self, conf):
        self.config = conf
        self._sessions = {}
        self.open()

    # Management
    def open(self):
        self.db = Base(TransactionsDB.BASE)
        try:
            self.db.open()
        except IOError:
            self.db.create(('amount'      , float), 
                           ('amount_local', float),
                           ('date'        , date),
                           ('account'     , str),
                           ('label'       , str),
                           ('currency'    , str))

    def close(self):
        self.db.close()
            
    def clearall(self):
        self.db.destroy()
        self.open()
        
    def insert(self, entry):
        self.db.insert(amount       = entry['amount'],
                       amount_local = entry['amount_local'],
                       date         = entry['date'],
                       account      = entry['account'],
                       label        = entry['label'],
                       currency     = entry['currency'])

    def feed(self, fpath, parser, skip_duplicates=True, overwrite=False, delete=False, dry_run=False):

        deleted = added = 0
        for entry in parser.parse(fpath):
            if dry_run:
                print('%s %-40s\t%12.2f %s' % (entry['date'].isoformat(),
                                               entry['account'][:40],
                                               entry['amount'], 
                                               entry['currency'])); continue

            if skip_duplicates or overwrite or delete:
                _dup = self.db(date=entry['date'], account=entry['account'], amount=entry['amount'])
                if _dup:
                    if overwrite or delete:
                        deleted += len(_dup)
                        self.db.delete(_dup)
                    else:
                        continue
                if delete:
                    continue

            entry['label'] = self.config.assign_label(entry)
            self.insert(entry)
            added += 1
        
        if not dry_run:
            INFO('  added %i transactions' % added)
            INFO('  deleted %i transactions' % deleted)
            parser.post_process(self.db, added)

    def update_labels(self):
        
        # Load all records into memory. File will get corrupt if using the iterator.
        records = [rec for rec in self.db]
        for record in records:
            as_dict = dict((field, getattr(record, field)) for field in record.fields)
            label = self.config.assign_label(as_dict)
            if label != record.label:
                self.db.update(record, label=label)

        self.db.cleanup()


    # Queries
    get_amount = lambda rec: rec.amount_local    

    def results_as_text(self, results):

        results.sort_by('date')
        idx = 0
        record = results[idx]
        text_list = []
        while True:
            text_list.append('%s   %-40s\t%12.2f %s' %
                             (record.date.isoformat(), 
                              unicode(record.account[:40], 'utf-8'),
                              record.amount,
                              record.currency)); 
            try:
                idx += 1
                record = results[idx]
            except IndexError:
                return text_list

    
    def assemble_data(self, sid, datatype, foreach, show, select):

        try:

            session = self._sessions.get(sid, {})
            if session:
                if session['raw_query'] == (foreach, show, select):
                    # Same query, return cached result
                    return True, \
                        self._sessions[sid]['flot_' + show] if datatype == 'plot' else \
                        self._sessions[sid]['text']

            # Helpers
            get_amount = lambda rec: rec.amount_local
            M = range(1,13)
            total = strlen = 0

            data = {}
            text = {}
            query = 'date1 <= date < date2 and label == l'

            if foreach == 'label':
                
                if session:
                    if session['raw_query'][0] == 'label' and session['raw_query'][2] == select:
                        # Same query, but different presentation (sum or average)
                        return True, \
                            self._sessions[sid]['flot_' + show] if datatype == 'plot' else \
                            self._sessions[sid]['text']

                # New query
                dates = re.findall('[0-9]{6}', unquote_plus(select))
                date1 = date2 = date(int(dates[0][2:]), int(dates[0][:2]), 1)
                if len(dates) == 2:
                    date2 = date(int(dates[1][2:]), int(dates[1][:2]), 1)
                date2 = date(date2.year + (date2.month == 12), M[date2.month - 12], 1)

                for label in self.config.labels.iterkeys():

                    results = self.db.select(None, query, l = label, date1 = date1, date2 = date2)
                    value = sum(map(get_amount, results))

                    if abs(value) > 1:
                        data[label] = value

                        if label not in self.config.cash_flow_ignore:
                            total += value
                        else:
                            label += '*'

                        text[label] = self.results_as_text(results)
                        strlen = len(text[label][-1])
                        sumstr = '%12.2f %s' % (value, self.config.local_currency)
                        text[label].append('-' * strlen)
                        text[label].append(' ' * (strlen - len(sumstr)) + sumstr)

                ydelta = date2.year - date1.year
                mdelta = date2.month - date1.month
                delta = 12 * ydelta + mdelta

                session['flot_average'] = {}
                for key, val in data.iteritems():
                    session['flot_average'][key] = val/delta


            elif foreach in ('month', 'year'):
                
                # New query
                date1 = date2 = first = datetime.now()
                if foreach == 'month':
                    first = date(date1.year - 1, date1.month, 1)
                    date1 = date(date1.year - (date1.month == 1), M[date1.month - 2], 1)
                    date2 = date(date2.year, date2.month, 1)
                else:
                    first = date(date1.year - 9, 1, 1)
                    date1 = date(date1.year, 1, 1)
                    date2 = date(date2.year + 1, 1, 1)

                select = unquote_plus(select)

                while date1 >= first:

                    results = self.db.select(None, query, l = select, date1 = date1, date2 = date2)
                    value = sum(map(get_amount, results))

                    date2 = date1 
                    if foreach == 'month':
                        key = date1.strftime('%Y.%m') 
                        date1 = date(date2.year - (date2.month == 1), M[date2.month - 2], 1)
                    else:
                        key = str(date1.year)
                        date1 = date(date2.year - 1, 1, 1)

                    data[key] = value
                    total += value
                    if results:
                        text[key] = self.results_as_text(results)
                        strlen = len(text[key][-1])
                        sumstr = '%12.2f %s' % (value, self.config.local_currency)
                        text[key].append('-' * strlen)
                        text[key].append(' ' * (strlen - len(sumstr)) + sumstr)


            # All good, set new session attributes
            session['raw_query'] = (foreach, show, select)
            session['flot_sum'] = data
            session['text'] = text

            if session['text']:
                session['text']['***'] = ['-' * strlen,
                                          'SUM: %12.2f %s' % (total, self.config.local_currency),
                                          '-' * strlen]

            self._sessions[sid] = session
            return True, session['flot_' + show] if datatype == 'plot' else session['text']

        except Exception as e:
            return False, str(e)
Пример #4
0
print 'enumerate on _pos',i

print "after delete"
for r in [r for r in db if 33> r.age >= 30]:
    print r.name,r.age
print '-------------'

print '\nname = pierre'
for r in db.select(['__id__','name','age','birth'],name='pierre'):
    print r
print '-------------'

# make 'pierre' uppercase
print '\nmake pierre uppercase'
for record in db.select_for_update(None,'name == x',x='pierre'):
    db.update(record,name = record.name.upper())
    print db[record.__id__]

# increment ages
print '\nincrement ages'
for record in db.select_for_update([],'True'):
    if not record.age is None:
        db.update(record,age = record.age+1)
print db[5]

for record in [r for r in db]:
    if not record.age is None:
        db.update(record,age = record.age+1)
print db[5]

print "\nChange dates"
Пример #5
0
def run_test(thread_safe=False):

    if not thread_safe:
        db = Base(r'dummy') 
    else:
        db = TS_Base('dummy')

    db.create(('name',str), ('fr_name',unicode),
        ('age',int),
        ('size',int,300),
        ('birth',date,date(1994,1,14)),
        ('afloat',float,1.0),
        ('birth_hour', dtime,dtime(10,10,10)),
        mode='override')

    # test float conversions
    if thread_safe is False:
        f = db._file["afloat"]
        def all(v):
            return [ord(c) for c in v]

        for i in range(10):
            afloat = random.uniform(-10**random.randint(-307,307),
                    10**random.randint(-307,307))
            try:
                assert cmp(afloat,0.0) == cmp(f.to_block(afloat),f.to_block(0.0))
            except:
                print afloat
                print "afloat > 0.0 ?",afloat>0.0
                print "blocks ?",f.to_block(afloat)>f.to_block(0.0)
                print all(f.to_block(afloat)),all(f.to_block(0.0))
                raise

    assert db.defaults["age"] == None
    assert db.defaults["size"] == 300
    assert db.defaults["afloat"] == 1.0
    assert db.defaults["birth_hour"] == dtime(10,10,10)
    assert db.defaults["birth"] == date(1994,1,14)

    for i in range(100):
        db.insert(name=random.choice(names),
             fr_name = unicode(random.choice(fr_names),'latin-1'),
             age=random.randint(7,47),size=random.randint(110,175),
             birth=date(random.randint(1858,1999),random.randint(1,12),10),
             afloat = random.uniform(-10**random.randint(-307,307),
                10**random.randint(-307,307)),
             birth_hour = dtime(random.randint(0, 23), random.randint(0, 59), random.randint(0, 59)))

    assert len(db)==100
    assert isinstance(db[50].fr_name,unicode)
    print db[50].fr_name.encode('latin-1')

    db.open()
    # test if default values have not been modified after open()
    assert db.defaults["age"] == None
    assert db.defaults["size"] == 300
    assert db.defaults["afloat"] == 1.0
    assert db.defaults["birth_hour"] == dtime(10,10,10)
    assert db.defaults["birth"] == date(1994,1,14)

    for i in range(5):
        # insert a list
        db.insert(random.choice(names),
             unicode(random.choice(fr_names),'latin-1'),
             random.randint(7,47),random.randint(110,175),
             date(random.randint(1958,1999),random.randint(1,12),10),
             random.uniform(-10**random.randint(-307,307),
                10**random.randint(-307,307)),
             dtime(random.randint(0, 23), random.randint(0, 59), random.randint(0, 59)))
        db.insert(name=random.choice(names)) # missing fields
        for field in db.field_names[2:]:
            if field == "name":
                continue
            try:
                assert getattr(db[-1],field) == db.defaults[field]
            except:
                print "attribute %s not set to default value %s" %(field,db[-1])
                raise

    # insert as string
    db.set_string_format(unicode,'latin-1')
    db.set_string_format(date,'%d-%m-%y')
    db.set_string_format(dtime,'%H-%M-%S')
    db.insert_as_strings(name="testname",fr_name=random.choice(fr_names),
        age=10,size=123,birth="07-10-95", birth_hour="20-53-3")

    assert db[-1].birth == date(1995,10,7)
    assert db[-1].name == "testname"
    assert db[-1].age == 10
    assert db[-1].afloat == db.defaults["afloat"]

    db.insert_as_strings("testname",random.choice(fr_names),
        11,134,"09-12-94",1.0, "5-6-13")

    assert db[len(db)-1].birth == date(1994,12,9)
    assert db[-1].name == "testname"
    assert db[-1].age == 11
    assert db[-1].size == 134
    assert db[-1].afloat == 1.0

    # search between 2 dates
    print '\nBirth between 1960 and 1970'
    for r in db.select(None,birth=[date(1960,1,1),date(1970,12,13)]):
        print r.name,r.birth

    print "sorted"
    for r in db.select(None,birth=[date(1960,1,1),date(1970,12,13)]).sort_by('+name-birth'):
        print r.name,r.birth

    f = buzhug_files.FloatFile().to_block
    def all(v):
        return [ord(c) for c in f(v)]

    # search between 2 floats

    # selection by list comprehension
    s1 = [ r for r in db if 0.0 <= r.afloat <= 1e50 ]
    # selection by select
    s2 = db.select(None,'x<=afloat<=y',x=0.0,y=1e50)
    # selction by select with interval
    s3 = db.select(None,afloat=[0.0,1e50])

    try:
        assert len(s1) == len(s2) == len(s3)
    except:
        print "%s records by list comprehension, " %len(s1)
        print "%s by select by formula," %len(s2)
        print "%s by select by interval" %len(s3)

    for r in s1:
        try:
            assert r in s2
        except:
            print all(r.afloat)

    for r in s2:
        try:
            assert r in s1
        except:
            print "in select but not in list comprehension",r
            raise
    r = db[0]
    assert r.__class__.db is db

    fr=random.choice(fr_names)
    s1 = [ r for r in db if r.age == 30 and r.fr_name == unicode(fr,'latin-1')]
    s2 = db.select(['name','fr_name'],age=30,fr_name = unicode(fr,'latin-1'))

    assert len(s1)==len(s2)

    # different ways to count the number of items
    assert len(db)  == sum([1 for r in db]) == len(db.select(['name']))

    # check if version number is correctly incremented
    for i in range(5):
        recs = db.select_for_update(['name'],'True')
        version = recs[0].__version__
        recs[0].update()
        assert db[0].__version__ == version + 1

    # check if cleanup doesn't change db length
    length_before = len(db)
    db.cleanup()
    assert len(db) == length_before

    # check if selection by select on __id__ returns the same as direct
    # access by id
    recs = db.select([],'__id__ == c',c=20)
    assert recs[0] == db[20]

    # check that has_key returns False for invalid hey
    assert not db.has_key(1000)

    # drop field
    db.drop_field('name')
    # check if field was actually removed from base definition and rows
    assert not "name" in db.fields
    assert not hasattr(db[20],"name")

    # add field
    db.add_field('name',str,default="marcel")
    # check if field was added with the correct default value
    assert "name" in db.fields
    assert hasattr(db[20],"name")
    assert db[20].name == "marcel"

    # change default value
    db.set_default("name","julie")
    db.insert(age=20)
    assert db[-1].name == "julie"

    # delete a record

    db.delete([db[10]])
    # check if record has been deleted
    try:
        print db[10]
        raise Exception,"Row 10 should have been deleted"
    except IndexError:
        pass

    assert 10 not in db
    assert len(db) == length_before

    # selections    

    # selection by generator expression
    # age between 30 et 32
    d_ids = []
    for r in [r for r in db if 33> r.age >= 30]:
        d_ids.append(r.__id__)

    length = len(db)
    # remove these items
    db.delete([r for r in db if 33> r.age >= 30])
    # check if correct number of records removed
    assert len(db) == length - len(d_ids)
    # check if all records have been removed
    assert not [r for r in db if 33> r.age >= 30]

    # updates
    # select name = pierre
    s1 = db.select(['__id__','name','age','birth'],name='pierre')
    # make 'pierre' uppercase
    for record in db.select_for_update(None,'name == x',x='pierre'):
        db.update(record,name = record.name.upper())
    # check if attribute was correctly updated
    for rec in s1:
        assert db[rec.__id__] == "Pierre"

    # increment ages
    for record in db.select_for_update([],'True'):
        age = record.age
        if not record.age is None:
            db.update(record,age = record.age+1)
            # check
            assert db[record.__id__].age == age + 1

    for record in [r for r in db]:
        age = record.age
        if not record.age is None:
            db.update(record,age = record.age+1)
            # check
            assert db[record.__id__].age == age + 1

    # change dates
    for record in db.select_for_update([],'age>v',v=35):
        db.update(record,birth = date(random.randint(1958,1999),
                                random.randint(1,12),10))

    db.commit()

    # check length after commit
    assert sum([1 for r in db]) == len(db)

    # insert new records
    for i in range(50):
        db.insert(name=random.choice(names),
             age=random.randint(7,47),size=random.randint(110,175))

    # check that record 10 is still deleted
    try:
        print db[10]
        raise Exception,"Row 10 should have been deleted"
    except IndexError:
        pass

    print db.keys()
    print "has key 10 ?",db.has_key(10)
    assert 10 not in db
    #raw_input()

    # check that deleted_lines was cleared by commit()
    assert not db._pos.deleted_lines
    print db._del_rows.deleted_rows

    length = len(db) # before cleanup

    # physically remove the deleted items    
    db.cleanup()
    # check that deleted_lines and deleted_rows are clean
    assert not db._pos.deleted_lines
    assert not db._del_rows.deleted_rows

    # check that record 10 is still deleted
    try:
        print db[10]
        raise Exception,"Row 10 should have been deleted"
    except IndexError:
        pass

    assert 10 not in db

    # check that length was not changed by cleanup
    assert len(db) == length
    assert len([ r for r in db]) == length

    # age > 30
    for r in db.select(['__id__','name','age'],
        'name == c1 and age > c2',
        c1 = 'pierre',c2 = 30):
        assert r.name == "pierre"
        assert r.age > 30

    # name =="PIERRE" and age > 30
    for r in db.select(['__id__','name','age','birth'],
                'name == c1 and age > c2',
                c1 = 'PIERRE',c2 = 30):
        assert r.name == 'PIERRE'
        assert r.age > 30

    # test with !=
    for r in db.select(['__id__'],'name != c1',c1='claire'):
        assert r.name != 'claire'

    # age > id
    # with select
    s1 = db.select(['name','__id__','age'],'age > __id__')
    for r in s1:
        assert r.age > r.__id__
    # with iter
    s2 = [ r for r in db if r.age > r.__id__ ]
    for r in s2:
        assert r.age > r.__id__

    assert len(s1) == len(s2)

    # birth > date(1978,1,1)
    # with select
    s1 = db.select(['name','__id__','age'],'birth > v',v=date(1978,1,1))
    for r in s1:
        assert r.birth > date(1978,1,1)
    # with iter

    s2 = [ r for r in db if r.birth and r.birth > date(1978,1,1) ]
    for r in s2:
        assert r.birth > date(1978,1,1)

    assert len(s1) == len(s2)

    # test with floats
    for i in range(10):
        x = random.uniform(-10**random.randint(-307,307),
                10**random.randint(-307,307))
        s1 = [ r for r in db if r.afloat > x ]
        s2 = db.select(['name'],'afloat > v',v=x)
        assert len(s1)==len(s2)

    # base with external link
    houses = Base('houses')
    houses.create(('address',str),('flag',bool),('resident',db,db[0]),mode="override")

    addresses = ['Giono','Proust','Mauriac','Gide','Bernanos','Racine',
        'La Fontaine']
    ks = db.keys()
    for i in range(50):
        x = random.choice(ks)
        address = random.choice(addresses)
        houses.insert(address=address,flag = address[0]>"H",resident=db[x])

    # houses with jean
    s1 = []
    for h in houses:
        if h.resident.name == 'jean':
            s1.append(h)

    # by select : ???
    #s2 = houses.select([],'resident.name == v',v='jean')
    # assert len(s1) == len(s2)

    h1 = Base('houses')
    h1.open()

    l1 = len(h1.select([],flag=True))
    l2 = len(h1.select([],flag=False))
    assert l1 + l2 == len(h1)

    class DictRecord(Record):
        def __getitem__(self, k):
            item = self
            names = k.split('.')
            for name in names:
                item = getattr(item, name)
            return item

    h1.set_record_class(DictRecord)
    print '\nrecord_class = DictRecord, h1[0]'
    print h1[0]
    print "\nResident name: %(resident.name)s\nAddress: %(address)s" % h1[0]
def run_test(thread_safe=False):

    if not thread_safe:
        db = Base(r'dummy')
    else:
        db = TS_Base('dummy')

    db.create(('name', str), ('fr_name', unicode), ('age', int),
              ('size', int, 300), ('birth', date, date(1994, 1, 14)),
              ('afloat', float, 1.0), ('birth_hour', dtime, dtime(10, 10, 10)),
              mode='override')

    # test float conversions
    if thread_safe is False:
        f = db._file["afloat"]

        def all(v):
            return [ord(c) for c in v]

        for i in range(10):
            afloat = random.uniform(-10**random.randint(-307, 307),
                                    10**random.randint(-307, 307))
            try:
                assert cmp(afloat, 0.0) == cmp(f.to_block(afloat),
                                               f.to_block(0.0))
            except:
                print afloat
                print "afloat > 0.0 ?", afloat > 0.0
                print "blocks ?", f.to_block(afloat) > f.to_block(0.0)
                print all(f.to_block(afloat)), all(f.to_block(0.0))
                raise

    assert db.defaults["age"] == None
    assert db.defaults["size"] == 300
    assert db.defaults["afloat"] == 1.0
    assert db.defaults["birth_hour"] == dtime(10, 10, 10)
    assert db.defaults["birth"] == date(1994, 1, 14)

    for i in range(100):
        db.insert(name=random.choice(names),
                  fr_name=unicode(random.choice(fr_names), 'latin-1'),
                  age=random.randint(7, 47),
                  size=random.randint(110, 175),
                  birth=date(random.randint(1858, 1999), random.randint(1, 12),
                             10),
                  afloat=random.uniform(-10**random.randint(-307, 307),
                                        10**random.randint(-307, 307)),
                  birth_hour=dtime(random.randint(0,
                                                  23), random.randint(0, 59),
                                   random.randint(0, 59)))

    assert len(db) == 100
    assert isinstance(db[50].fr_name, unicode)
    print db[50].fr_name.encode('latin-1')

    db.open()
    # test if default values have not been modified after open()
    assert db.defaults["age"] == None
    assert db.defaults["size"] == 300
    assert db.defaults["afloat"] == 1.0
    assert db.defaults["birth_hour"] == dtime(10, 10, 10)
    assert db.defaults["birth"] == date(1994, 1, 14)

    for i in range(5):
        # insert a list
        db.insert(
            random.choice(names), unicode(random.choice(fr_names), 'latin-1'),
            random.randint(7, 47), random.randint(110, 175),
            date(random.randint(1958, 1999), random.randint(1, 12), 10),
            random.uniform(-10**random.randint(-307, 307),
                           10**random.randint(-307, 307)),
            dtime(random.randint(0, 23), random.randint(0, 59),
                  random.randint(0, 59)))
        db.insert(name=random.choice(names))  # missing fields
        for field in db.field_names[2:]:
            if field == "name":
                continue
            try:
                assert getattr(db[-1], field) == db.defaults[field]
            except:
                print "attribute %s not set to default value %s" % (field,
                                                                    db[-1])
                raise

    # insert as string
    db.set_string_format(unicode, 'latin-1')
    db.set_string_format(date, '%d-%m-%y')
    db.set_string_format(dtime, '%H-%M-%S')
    db.insert_as_strings(name="testname",
                         fr_name=random.choice(fr_names),
                         age=10,
                         size=123,
                         birth="07-10-95",
                         birth_hour="20-53-3")

    assert db[-1].birth == date(1995, 10, 7)
    assert db[-1].name == "testname"
    assert db[-1].age == 10
    assert db[-1].afloat == db.defaults["afloat"]

    db.insert_as_strings("testname", random.choice(fr_names), 11, 134,
                         "09-12-94", 1.0, "5-6-13")

    assert db[len(db) - 1].birth == date(1994, 12, 9)
    assert db[-1].name == "testname"
    assert db[-1].age == 11
    assert db[-1].size == 134
    assert db[-1].afloat == 1.0

    # search between 2 dates
    print '\nBirth between 1960 and 1970'
    for r in db.select(None, birth=[date(1960, 1, 1), date(1970, 12, 13)]):
        print r.name, r.birth

    print "sorted"
    for r in db.select(None, birth=[date(1960, 1, 1),
                                    date(1970, 12,
                                         13)]).sort_by('+name-birth'):
        print r.name, r.birth

    f = buzhug_files.FloatFile().to_block

    def all(v):
        return [ord(c) for c in f(v)]

    # search between 2 floats

    # selection by list comprehension
    s1 = [r for r in db if 0.0 <= r.afloat <= 1e50]
    # selection by select
    s2 = db.select(None, 'x<=afloat<=y', x=0.0, y=1e50)
    # selction by select with interval
    s3 = db.select(None, afloat=[0.0, 1e50])

    try:
        assert len(s1) == len(s2) == len(s3)
    except:
        print "%s records by list comprehension, " % len(s1)
        print "%s by select by formula," % len(s2)
        print "%s by select by interval" % len(s3)

    for r in s1:
        try:
            assert r in s2
        except:
            print all(r.afloat)

    for r in s2:
        try:
            assert r in s1
        except:
            print "in select but not in list comprehension", r
            raise
    r = db[0]
    assert r.__class__.db is db

    fr = random.choice(fr_names)
    s1 = [r for r in db if r.age == 30 and r.fr_name == unicode(fr, 'latin-1')]
    s2 = db.select(['name', 'fr_name'], age=30, fr_name=unicode(fr, 'latin-1'))

    assert len(s1) == len(s2)

    # different ways to count the number of items
    assert len(db) == sum([1 for r in db]) == len(db.select(['name']))

    # check if version number is correctly incremented
    for i in range(5):
        recs = db.select_for_update(['name'], 'True')
        version = recs[0].__version__
        recs[0].update()
        assert db[0].__version__ == version + 1

    # check if cleanup doesn't change db length
    length_before = len(db)
    db.cleanup()
    assert len(db) == length_before

    # check if selection by select on __id__ returns the same as direct
    # access by id
    recs = db.select([], '__id__ == c', c=20)
    assert recs[0] == db[20]

    # check that has_key returns False for invalid hey
    assert not db.has_key(1000)

    # drop field
    db.drop_field('name')
    # check if field was actually removed from base definition and rows
    assert not "name" in db.fields
    assert not hasattr(db[20], "name")

    # add field
    db.add_field('name', str, default="marcel")
    # check if field was added with the correct default value
    assert "name" in db.fields
    assert hasattr(db[20], "name")
    assert db[20].name == "marcel"

    # change default value
    db.set_default("name", "julie")
    db.insert(age=20)
    assert db[-1].name == "julie"

    # delete a record

    db.delete([db[10]])
    # check if record has been deleted
    try:
        print db[10]
        raise Exception, "Row 10 should have been deleted"
    except IndexError:
        pass

    assert 10 not in db
    assert len(db) == length_before

    # selections

    # selection by generator expression
    # age between 30 et 32
    d_ids = []
    for r in [r for r in db if 33 > r.age >= 30]:
        d_ids.append(r.__id__)

    length = len(db)
    # remove these items
    db.delete([r for r in db if 33 > r.age >= 30])
    # check if correct number of records removed
    assert len(db) == length - len(d_ids)
    # check if all records have been removed
    assert not [r for r in db if 33 > r.age >= 30]

    # updates
    # select name = pierre
    s1 = db.select(['__id__', 'name', 'age', 'birth'], name='pierre')
    # make 'pierre' uppercase
    for record in db.select_for_update(None, 'name == x', x='pierre'):
        db.update(record, name=record.name.upper())
    # check if attribute was correctly updated
    for rec in s1:
        assert db[rec.__id__] == "Pierre"

    # increment ages
    for record in db.select_for_update([], 'True'):
        age = record.age
        if not record.age is None:
            db.update(record, age=record.age + 1)
            # check
            assert db[record.__id__].age == age + 1

    for record in [r for r in db]:
        age = record.age
        if not record.age is None:
            db.update(record, age=record.age + 1)
            # check
            assert db[record.__id__].age == age + 1

    # change dates
    for record in db.select_for_update([], 'age>v', v=35):
        db.update(record,
                  birth=date(random.randint(1958, 1999), random.randint(1, 12),
                             10))

    db.commit()

    # check length after commit
    assert sum([1 for r in db]) == len(db)

    # insert new records
    for i in range(50):
        db.insert(name=random.choice(names),
                  age=random.randint(7, 47),
                  size=random.randint(110, 175))

    # check that record 10 is still deleted
    try:
        print db[10]
        raise Exception, "Row 10 should have been deleted"
    except IndexError:
        pass

    print db.keys()
    print "has key 10 ?", db.has_key(10)
    assert 10 not in db
    #raw_input()

    # check that deleted_lines was cleared by commit()
    assert not db._pos.deleted_lines
    print db._del_rows.deleted_rows

    length = len(db)  # before cleanup

    # physically remove the deleted items
    db.cleanup()
    # check that deleted_lines and deleted_rows are clean
    assert not db._pos.deleted_lines
    assert not db._del_rows.deleted_rows

    # check that record 10 is still deleted
    try:
        print db[10]
        raise Exception, "Row 10 should have been deleted"
    except IndexError:
        pass

    assert 10 not in db

    # check that length was not changed by cleanup
    assert len(db) == length
    assert len([r for r in db]) == length

    # age > 30
    for r in db.select(['__id__', 'name', 'age'],
                       'name == c1 and age > c2',
                       c1='pierre',
                       c2=30):
        assert r.name == "pierre"
        assert r.age > 30

    # name =="PIERRE" and age > 30
    for r in db.select(['__id__', 'name', 'age', 'birth'],
                       'name == c1 and age > c2',
                       c1='PIERRE',
                       c2=30):
        assert r.name == 'PIERRE'
        assert r.age > 30

    # test with !=
    for r in db.select(['__id__'], 'name != c1', c1='claire'):
        assert r.name != 'claire'

    # age > id
    # with select
    s1 = db.select(['name', '__id__', 'age'], 'age > __id__')
    for r in s1:
        assert r.age > r.__id__
    # with iter
    s2 = [r for r in db if r.age > r.__id__]
    for r in s2:
        assert r.age > r.__id__

    assert len(s1) == len(s2)

    # birth > date(1978,1,1)
    # with select
    s1 = db.select(['name', '__id__', 'age'], 'birth > v', v=date(1978, 1, 1))
    for r in s1:
        assert r.birth > date(1978, 1, 1)
    # with iter

    s2 = [r for r in db if r.birth and r.birth > date(1978, 1, 1)]
    for r in s2:
        assert r.birth > date(1978, 1, 1)

    assert len(s1) == len(s2)

    # test with floats
    for i in range(10):
        x = random.uniform(-10**random.randint(-307, 307),
                           10**random.randint(-307, 307))
        s1 = [r for r in db if r.afloat > x]
        s2 = db.select(['name'], 'afloat > v', v=x)
        assert len(s1) == len(s2)

    # base with external link
    houses = Base('houses')
    houses.create(('address', str), ('flag', bool), ('resident', db, db[0]),
                  mode="override")

    addresses = [
        'Giono', 'Proust', 'Mauriac', 'Gide', 'Bernanos', 'Racine',
        'La Fontaine'
    ]
    ks = db.keys()
    for i in range(50):
        x = random.choice(ks)
        address = random.choice(addresses)
        houses.insert(address=address, flag=address[0] > "H", resident=db[x])

    # houses with jean
    s1 = []
    for h in houses:
        if h.resident.name == 'jean':
            s1.append(h)

    # by select : ???
    #s2 = houses.select([],'resident.name == v',v='jean')
    # assert len(s1) == len(s2)

    h1 = Base('houses')
    h1.open()

    l1 = len(h1.select([], flag=True))
    l2 = len(h1.select([], flag=False))
    assert l1 + l2 == len(h1)

    class DictRecord(Record):
        def __getitem__(self, k):
            item = self
            names = k.split('.')
            for name in names:
                item = getattr(item, name)
            return item

    h1.set_record_class(DictRecord)
    print '\nrecord_class = DictRecord, h1[0]'
    print h1[0]
    print "\nResident name: %(resident.name)s\nAddress: %(address)s" % h1[0]
Пример #7
0
print 'enumerate on _pos', i

print "after delete"
for r in [r for r in db if 33 > r.age >= 30]:
    print r.name, r.age
print '-------------'

print '\nname = pierre'
for r in db.select(['__id__', 'name', 'age', 'birth'], name='pierre'):
    print r
print '-------------'

# make 'pierre' uppercase
print '\nmake pierre uppercase'
for record in db.select_for_update(None, 'name == x', x='pierre'):
    db.update(record, name=record.name.upper())
    print db[record.__id__]

# increment ages
print '\nincrement ages'
for record in db.select_for_update([], 'True'):
    if not record.age is None:
        db.update(record, age=record.age + 1)
print db[5]

for record in [r for r in db]:
    if not record.age is None:
        db.update(record, age=record.age + 1)
print db[5]

print "\nChange dates"