def _backup_database_and_make_progress_dialog (self, xmlbackup):
self._backup_database_file()
from gourmet.dialog_extras import ProgressDialog
subl = self.database_change_message%{'backupfile':self.backupfile}
subl += "\n"
subl += _("In case anything goes wrong, a backup copy of your recipe database is being exported to %s")%xmlbackup
self.pd = ProgressDialog(
title=_("Transferring data to new database format"),
label=self.database_change_title,
sublabel=subl,
pause=False,
stop=False)
self.pd.show()
def _backup_database_and_make_progress_dialog (self, xmlbackup):
self._backup_database_file()
from gourmet.dialog_extras import ProgressDialog
subl = self.database_change_message%{'backupfile':self.backupfile}
subl += "\n"
subl += _("In case anything goes wrong, a backup copy of your recipe database is being exported to %s")%xmlbackup
self.pd = ProgressDialog(
title=_("Transferring data to new database format"),
label=self.database_change_title,
sublabel=subl,
pause=False,
stop=False)
self.pd.show()
class RecData (rdatabase.RecData):
"""A class to keep recipe data in. This class basically is a wrapper for interactions
with metakit (stored in self.db). Ideally, interactions with metakit should be abstracted
through this class so that I could easily change to a different database backend."""
database_change_title = _('Database format has changed')
database_change_message = _('%(progname)s %(version)s has changed the format of its database. Your database will no longer work with older versions of %(progname)s. A backup has been saved in %(backupfile)s')%{
'version':gourmet.version.version,
'progname':gourmet.version.appname,
'backupfile':"%(backupfile)s"}
def __init__ (self, file=os.path.join(gglobals.gourmetdir,'recipes.mk')):
debug('RecData.__init__()',3)
self.changed=False
self.file = file
self.backed_up = False
self.import_when_done = None
rdatabase.RecData.__init__(self)
# Core set-up methods
def initialize_connection (self):
debug('using file: %s'%self.file,1)
self.file = os.path.expandvars(os.path.expanduser(self.file))
mydir = os.path.split(self.file)[0]
# create the directory if it does not yet exist
if not os.path.exists(mydir):
os.mkdir(mydir)
self.db = metakit.storage(str(self.file),1) #filename must be string, not unicode
self.contentview=self.db.contents()
#self.load()
self.db.autocommit()
def setup_tables (self):
# we check for old, incompatible table names
# and fix them before calling our regular setup stuff
debug('setup_tables called!',1)
self.move_old_tables()
# set up our top id
# This is unique to metakit and not part of the normal setup_tables routine
# since other DBs will presumably have auto-increment built into them.
self.increment_vw = self.db.getas('incrementer[view:S,field:S,n:I]')
self.vw_to_name = {}
self.increment_dict = {}
#self.top_id_vw.append({'id':1})
#self.top_id_row = self.top_id_vw[0]
rdatabase.RecData.setup_tables(self)
# If we've dumped our data, we want to re-import it!
if self.import_when_done:
old_db,ifi = self.import_when_done
from gourmet.importers.gxml2_importer import converter
converter(
ifi, self, threaded=False,
progress=lambda p,m: self.pd.set_progress(p*0.5+0.5,m)
)
n = 0
tot = 3
for tabl,desc in [('shopcats_table',self.SHOPCATS_TABLE_DESC),
('shopcatsorder_table',self.SHOPCATSORDER_TABLE_DESC),
('pantry_table',self.PANTRY_TABLE_DESC)]:
self.copy_table(
old_db,
tabl,
table_cols=[i[0] for i in desc[1]],
prog=lambda p,m: self.pd.set_progress(p/tot+(n*p/tot),m),
convert_pickles=True
)
n+=1
self.pd.set_progress(1.0,'Database successfully converted!')
del old_db
def setup_table (self, name, data, key=None):
"""Setup a metakit view (table) for generic table description (see superclass rdatabase)."""
debug('setup_table(name=%(name)s,data=%(data)s,key=%(key)s)'%locals(),1)
getstring = name + "["
# We want to make our "key" the first item in the database
if key:
key_index = [x[0] for x in data].index(key)
data = [data[key_index]] + data[0:key_index] + data[key_index+1:]
for col,typ,flags in data:
if 'AUTOINCREMENT' in flags:
row = self.fetch_one(self.increment_vw,**{'view':name,
'field':col}
)
if not row:
self.increment_vw.append(view=name,field=col,n=1)
row = self.increment_vw[-1]
if not self.increment_dict.has_key(name):
self.increment_dict[name]={}
self.increment_dict[name][col]=row
getstring += "%s:%s,"%(col,self.type_to_metakit_type(typ))
if name=='recipe':
getstring = getstring+'categoryname:S,'
getstring = getstring[0:-1] + "]"
debug('Metakit: getting view: %s'%getstring,5)
vw = self.db.getas(getstring)
debug('Got metakit database',5)
if key:
rhsh = self.db.getas("__%s_hash__[_H:I,_R:I]"%name)
vw = vw.hash(rhsh,1)
# Make sure our increment fields are right...
if self.increment_dict.has_key(name):
self.vw_to_name[vw]=name
for field,row in self.increment_dict[name].items():
try:
svw=vw.sort(vw.id)
if len(svw)>svw[-1].id:
print """WTF: increment dicts are foobared. If you see this message, please
submit a bug report with the terminal output included.
"""
metakit.dump(svw)
self.increment_dict[name][field].n = svw[-1].id
except IndexError:
pass
return vw
def type_to_metakit_type (self, typ):
"""Convert a generic database type to a metakit property description."""
if typ.find('char') >= 0:
return 's'
if typ.find('text') >= 0:
return 's'
if typ.find('bool') >= 0:
return 'I'
if typ == 'unicode': return 's'
if typ == 'float': return "F"
if typ == 'int': return "I"
if typ == 'binary': return 'B'
else:
raise Exception("Can't Understand TYPE %s" % typ)
def save (self):
"""Commit our metakit database to file."""
debug('saving database to file %s'%self.file,0)
debug('there are %s recipes in the database'%len(self.recipe_table),0)
if self.changed:
self.db.commit()
self.changed=False
def load (self, file=None):
if file:
self.file = file
debug('loading database from file %s'%self.file,0)
fo = open(self.file,'rb')
self.db.load(fo)
self.db.autocommit()
fo.close()
# Search functions
def search (self, table, colname, regexp, exact=0, use_regexp=True, recurse=True):
"""Handed a table, a column name, and a regular expression, search
for an item. Alternatively, the regular expression can just be a value."""
debug('search %(table)s, %(colname)s, %(regexp)s, %(exact)s, %(use_regexp)s, %(recurse)s'%locals(),5)
if (recurse
and
self.normalizations.has_key(colname)
and
isinstance(table,rdatabase.NormalizedView)
):
nsrch = self.search(self.normalizations[colname],colname,regexp,exact,use_regexp,recurse=False)
if not nsrch: return []
nsrch = nsrch.rename(colname,'text')
nsrch = nsrch.rename('id',colname)
rvw = table.join(nsrch,getattr(table.__view__,colname))
return rvw
if type(regexp)==type(""):
regexp=str(regexp)
if exact and not use_regexp: return table.select(**{colname:regexp})
if not use_regexp: regexp = re.escape(regexp)
if exact:
indexvw = table.filter(lambda r: re.match(regexp, "%s"%getattr(r,colname)))
else:
indexvw = table.filter(lambda r: re.search(regexp,"%s"%getattr(r,colname),re.I))
if indexvw:
resultvw = table.remapwith(indexvw)
resultvw = resultvw.unique()
return resultvw
return []
def ings_search (self, ings, keyed=None, recipe_table=None, use_regexp=True, exact=False):
"""Handed a list of regexps, return a list of recipes containing all
items."""
for i in ings:
recipe_table = self.ing_search(i,keyed=keyed,recipe_table=recipe_table,exact=exact,use_regexp=use_regexp)
return recipe_table
def joined_search (self, table1, table2, search_by, search_str,
use_regexp=True, exact=False, join_on='id'):
table2 = table2.join(table1,getattr(table1,join_on))
vw = self.search(table2, search_by, search_str, use_regexp=use_regexp, exact=exact)
if not vw: return []
result_ids = vw.counts(getattr(vw,join_on),
'joinedcount')
return table1.join(result_ids,getattr(result_ids,join_on))
def filter (self, table, func):
ivw = table.filter(func)
return table.remapwith(ivw)
# convenience function
def delete_by_criteria (self, table, criteria):
"""Delete table by criteria"""
cur = table.select(**criteria)
if cur:
table.remove(table.indices(cur))
# Our versions of convenience functions for adding/modifying
# recipe stuff
def do_add_rec (self, rdict):
self.remove_unicode(rdict)
debug('adding recipe: %s'%rdict,5)
r=rdatabase.RecData.do_add_rec(self,rdict)
self.changed=True
return r
def do_modify_rec (self, rec, dic):
# This is a bit ugly, but we need to grab the rec object
# afresh for changes to "stick".
rec = self.get_rec(rec.id)
for k,v in dic.items():
if hasattr(rec,k):
self.changed=True
debug('do_modify_rec: setattr %s %s->%s'%(rec,k,v),10)
setattr(rec,k,v)
else:
debug("Warning: rec has no attribute %s (tried to set to %s)" %(k,v),1)
debug('running hooks',3)
self.run_hooks(self.modify_hooks,rec)
self.changed=True
## delete this code when we've figured out wtf is going on with this not sticking
#for attr in dic.keys():
# debug('modified recipe %s->%s'%(attr,getattr(rec,attr)),1)
return rec
def do_add_ing (self, ingdic):
"""Add ingredient to ingredients_table based on ingdict and return
ingredient object. Ingdict contains:
id: recipe_id
unit: unit
item: description
key: keyed descriptor
alternative: not yet implemented (alternative)
#optional: yes|no
optional: True|False (boolean)
position: INTEGER [position in list]
refid: id of reference recipe. If ref is provided, everything
else is irrelevant except for amount.
"""
self.remove_unicode(ingdic)
if ingdic.has_key('amount') and not ingdic['amount']: del ingdic['amount']
self.ingredients_table.append(ingdic)
if self.add_ing_hooks: self.run_hooks(self.add_ing_hooks, self.ingredients_table[-1])
self.changed=True
return self.ingredients_table[-1]
def delete_ing (self, ing):
self.ingredients_table.delete(ing.__index__)
self.changed=True
# Convenience functions
def fetch_one (self, table, *args, **kwargs):
indx=table.find(*args,**kwargs)
if indx >= 0:
return table[indx]
else:
return None
def remove_unicode (self, mydict):
for k,v in mydict.items():
if v.__class__ == u'hello'.__class__:
mydict[k]=v.encode('utf8','replace')
if k.__class__ == u'hello'.__class__:
v = mydict[k]
mydict.__delitem__(k)
mydict[k.encode('utf8','replace')] = v
def increment_field (self, table, field):
if type(table)!=str:
table = self.vw_to_name[table]
self.increment_dict[table][field].n += 1
return self.increment_dict[table][field].n
# Backup / Restructuring methods -- these are special methods to
# help us with our restructuring of the database - updating from
# old versions.
def move_old_tables (self):
self._move_row(table='ingredients',old=('group','text'),new=('inggroup','text'))
self._move_row(table='ingredients',old=('key','char(200)'),new=('ingkey','char(200)'))
self._move_row(table='shopcats',old=('key','char(50)'),new=('shopkey','char(50)'))
self._move_row(table='density',old=('key','char(50)'),new=('dkey','char(50)'))
self._move_row(table='convtable',old=('key','char(50)'),new=('ckey','char(50)'))
self._move_row(table='crossunitdict',old=('key','char(50)'),new=('cukey','char(50)'))
self._move_row(table='unitdict',old=('key','char(50)'),new=('ukey','char(50)'))
# If our recipe or ingredient tables are of the wrong type,
# we're simply going to dump the recipe portion of our DB into
# a file and reload it. This is ugly, but changing row types
# is simply too tricky in metakit and I've had it with
# segfaults and data corruption! At the very least, this
# method ensures the user has an XML copy of their data if the
# DB gets screwed up.
if (
(hasattr(self.contentview[0],'ingredients') and
(self._row_type('ingredients','optional') != 'i' or
self._row_type('ingredients','id')!='i' or
self._row_type('ingredients','refid')!='i')
)
or
(hasattr(self.contentview[0],'recipe') and
(self._row_type('recipe','id')!='i'
or self._row_type('recipe','rating')!='i'
or self._row_type('recipe','cooktime')!='i' # To be implemented in the future
or self._row_type('recipe','preptime')!='i'
or self._row_type('recipe','cuisine')!='i' # of if we're not normalized
or (hasattr(self.contentview[0],'recipe') and # of ir we still have 1-category-only
hasattr(self.contentview[0].recipe,'category')
)
)
)
):
self.clean_recs_table_and_dump_data()
def copy_table (self, old_db, table_name, table_cols,
prog=None,convert_pickles=False):
"""Copy columns of table from old database to ourselves.
Old database should be an instance of RecDataOld. Any format
changes that need to happen must happen magically, not here."""
oldvw = getattr(old_db,table_name)
tot = len(oldvw)
newvw = getattr(self,table_name)
if convert_pickles:
import pickle
def unpickle (o):
try: return pickle.loads(o)
except: return o
for n,row in enumerate(oldvw):
if n % 10: prog(float(n)/tot, "Copying other data...%s"%table_name)
if convert_pickles:
newvw.append(
dict([(col,unpickle(getattr(row,col))) for col in table_cols])
)
else:
newvw.append(
dict([(col,getattr(row,col)) for col in table_cols])
)
def clean_recs_table_and_dump_data (self):
"""Clean out our recipe table and dump our data for later recovery.
We return an old version of the DB and a file with our recipes.
"""
# Backup our DB
self._backup_database_file()
# Get ourselves out of memory
subrm = RecDataOldDB(self.backupfile)
from gourmet.exporters import gxml2_exporter
#print 'Our old DB looked like this: '
#print subrm.db.description()
#print 'And had'
#print len(subrm.recipe_table),'recipes'
#print 'and '
#print len(subrm.ingredients_table),'ingredients'
# dump our recipe db to a backup file
dumpfile = os.path.join(
os.path.split(self.file)[0],
'recipe_backup_%s.grmt'%(time.strftime('%m-%d-%y'))
)
self._backup_database_and_make_progress_dialog(dumpfile)
ofi = file(dumpfile,'w')
gxml2_exporter.recipe_table_to_xml(
subrm,
subrm.recipe_table,
ofi,
one_file=True,
progress_func=lambda p,m: self.pd.set_progress(p*0.5,m)
).run()
ofi.close()
# Now we drop our tables...
#self.db.getas('ingredients')
#self.db.getas('recipe')
# And commit
#self.db.commit()
# Move our current file out of the way so we can start a new
# one, deleting reference to the file so Windows won't
# complain
del self.db
shutil.move(self.file,self.file+'.trash')
# Restart our connection
self.initialize_connection()
# and later on, we'd better import file
self.import_when_done = subrm,dumpfile
def _change_row_types (self, changes):
"""Change row named 'name' from type 'old' to type 'new'
changes = [(table, name, old, new, converter),...]
We have to make all changes at once or bad things will happen.
"""
# This is quite a complex little problem in pymetakit. I got
# help from Brian Kelley <*****@*****.**> over the
# metakit mailing list on this one. Basically, in order to
# change the datatype of a column we have to add a dummy
# column, drop the old column, add the old column back, and
# copy the information over from our dummy. This is made even
# stranger by the fact that to drop a column in metakit, you
# have to call getas() without the column and then call
# commit() and nuke any reference to the db.
# if we don't have this table yet, then we don't need to do anything
DUMMIES = {}
change_dic = {}
default_descs = {}
dummy_descs = {}
for table,name,old,new,converter in changes:
if not hasattr(self.contentview[0],table):
continue
if not default_descs.has_key(table):
default_descs[table]=self.db.description(table)
if not dummy_descs.has_key(table):
dummy_descs[table]=default_descs[table]
if self._row_type(table,name) == self.type_to_metakit_type(new).lower():
continue
self._backup_database()
# we create a view with our old self and a new temporary self
DUMMIES[name] = 'TMP%s'%name
if not change_dic.has_key(table):
change_dic[table]={}
change_dic[table][name]={}
change_dic[table][name]['old']=old
change_dic[table][name]['new']=new
change_dic[table][name]['converter']=converter
self._move_row(table,(name,old),(DUMMIES[name],old))
# We're going to modify our default setup arguments to
# drop the proper column and add the new one in its sted
# before committing. This allows us to "drop" the old
# version of column 'name'
dummy_descs[table] = re.sub("(,|^)%s:"%re.escape(name),
r"\1%s:"%re.escape(DUMMIES[name]),
dummy_descs[table])
if DUMMIES:
# Drop our old columns...
for table,dummy_desc in dummy_descs.items():
self.db.getas("%s[%s]"%(table,dummy_desc))
debug('dropping columns by committing database',3)
self.db.commit()
debug('deleting reference to our db',3)
del self.db
debug('reinitialize our connection to a new db',3)
#self.initialize_connection() # reinitialize ourselves
#with our new DUMMYNAME column
self.db = metakit.storage(self.file,1)
self.contentview = self.db.contents()
# now we get our new self as a new datatype and copy our
# new information over...
# Loop through the changes we have to make
for table,cd in change_dic.items():
for name,change in cd.items():
newvw = self.setup_table(table,[(name,change['new']),(DUMMIES[name],change['old'])])
vw = newvw.filter(lambda x: getattr(x,DUMMIES[name]))
to_move_vw = newvw.remapwith(vw)
debug('converting attributes',4)
for n,r in enumerate(to_move_vw):
# convert our attributes
setattr(r,name,change['converter'](getattr(r,DUMMIES[name])))
# erase our temporary/holder attribute
setattr(r,DUMMIES[name],None)
debug('moved attribute %s times'%n,3)
default_descs[table] = re.sub(
"(,|^)%s:%s"%(name,self.type_to_metakit_type(change['old'])),
r"\1%s:%s"%(name,self.type_to_metakit_type(change['new'])),
default_descs[table]
)
for table,finished_desc in default_descs.items():
self.db.getas("%s[%s]"%(table,finished_desc)) #setup our table with the right attrs
self.db.commit() # and drop our dummy column
def _move_row (self, table, old, new, converter=None):
"""Move data from old (propertyname, type) to new (propertyname, type).
This is designed for backwards compatability (to allow
for other database backends)."""
debug('_move_row(table=%(table)s old=%(old)s new=%(new)s converter=%(converter)s'%locals(),1)
if not hasattr(self.contentview[0],table) or not hasattr(getattr(self.contentview[0],table),old[0]):
debug('Old property %s doesn\'t exist'%old[0],9)
return
tmpantry_table = self.setup_table(table, [new,old])
vw = tmpantry_table.filter(lambda x: getattr(x,old[0]))
to_move_vw = tmpantry_table.remapwith(vw)
to_move = len(to_move_vw)
if to_move > 0:
self._backup_database()
for r in to_move_vw:
if converter:
setattr(r,new[0],converter(getattr(r,old[0])))
else:
setattr(r,new[0],getattr(r,old[0]))
setattr(r,old[0],None)
def _backup_database_file (self):
"""Create a backup copy of our database tagged with ".OLD" in case we muck things up."""
if not self.backed_up:
backupfile = self.file + '.OLD'
n = 1
while os.path.exists(backupfile):
backupfile = re.split('[0-9]*[.]OLD',backupfile)[0]
backupfile = backupfile + str(n) + '.OLD'
n += 1
shutil.copy(self.file, backupfile)
self.backupfile = backupfile
self.backed_up = True
def _backup_database (self):
if not self.backed_up:
self._backup_database_file()
print """
Your database will not work with older
versions of GOURMET.
A backup has been saved in %s"""%self.backupfile
import gourmet.dialog_extras, gourmet.version
gourmet.dialog_extras.show_message(
label=self.database_change_title,
sublabel=self.database_change_message%{'backupfile':self.backupfile},
)
def _backup_database_and_make_progress_dialog (self, xmlbackup):
self._backup_database_file()
from gourmet.dialog_extras import ProgressDialog
subl = self.database_change_message%{'backupfile':self.backupfile}
subl += "\n"
subl += _("In case anything goes wrong, a backup copy of your recipe database is being exported to %s")%xmlbackup
self.pd = ProgressDialog(
title=_("Transferring data to new database format"),
label=self.database_change_title,
sublabel=subl,
pause=False,
stop=False)
self.pd.show()
def _row_type (self, table, name):
"""Return the row type for the table named name"""
if not hasattr(self.contentview[0],table): return
prop = getattr(getattr(self.contentview[0],table),name) # get ourselves the property object
return prop.type.lower()
class RecData(rdatabase.RecData):
"""A class to keep recipe data in. This class basically is a wrapper for interactions
with metakit (stored in self.db). Ideally, interactions with metakit should be abstracted
through this class so that I could easily change to a different database backend."""
database_change_title = _('Database format has changed')
database_change_message = _(
'%(progname)s %(version)s has changed the format of its database. Your database will no longer work with older versions of %(progname)s. A backup has been saved in %(backupfile)s'
) % {
'version': gourmet.version.version,
'progname': gourmet.version.appname,
'backupfile': "%(backupfile)s"
}
def __init__(self, file=os.path.join(gglobals.gourmetdir, 'recipes.mk')):
debug('RecData.__init__()', 3)
self.changed = False
self.file = file
self.backed_up = False
self.import_when_done = None
rdatabase.RecData.__init__(self)
# Core set-up methods
def initialize_connection(self):
debug('using file: %s' % self.file, 1)
self.file = os.path.expandvars(os.path.expanduser(self.file))
mydir = os.path.split(self.file)[0]
# create the directory if it does not yet exist
if not os.path.exists(mydir):
os.mkdir(mydir)
self.db = metakit.storage(str(self.file),
1) #filename must be string, not unicode
self.contentview = self.db.contents()
#self.load()
self.db.autocommit()
def setup_tables(self):
# we check for old, incompatible table names
# and fix them before calling our regular setup stuff
debug('setup_tables called!', 1)
self.move_old_tables()
# set up our top id
# This is unique to metakit and not part of the normal setup_tables routine
# since other DBs will presumably have auto-increment built into them.
self.increment_vw = self.db.getas('incrementer[view:S,field:S,n:I]')
self.vw_to_name = {}
self.increment_dict = {}
#self.top_id_vw.append({'id':1})
#self.top_id_row = self.top_id_vw[0]
rdatabase.RecData.setup_tables(self)
# If we've dumped our data, we want to re-import it!
if self.import_when_done:
old_db, ifi = self.import_when_done
from gourmet.importers.gxml2_importer import converter
converter(
ifi,
self,
threaded=False,
progress=lambda p, m: self.pd.set_progress(p * 0.5 + 0.5, m))
n = 0
tot = 3
for tabl, desc in [('shopcats_table', self.SHOPCATS_TABLE_DESC),
('shopcatsorder_table',
self.SHOPCATSORDER_TABLE_DESC),
('pantry_table', self.PANTRY_TABLE_DESC)]:
self.copy_table(old_db,
tabl,
table_cols=[i[0] for i in desc[1]],
prog=lambda p, m: self.pd.set_progress(
p / tot + (n * p / tot), m),
convert_pickles=True)
n += 1
self.pd.set_progress(1.0, 'Database successfully converted!')
del old_db
def setup_table(self, name, data, key=None):
"""Setup a metakit view (table) for generic table description (see superclass rdatabase)."""
debug(
'setup_table(name=%(name)s,data=%(data)s,key=%(key)s)' % locals(),
1)
getstring = name + "["
# We want to make our "key" the first item in the database
if key:
key_index = [x[0] for x in data].index(key)
data = [data[key_index]] + data[0:key_index] + data[key_index + 1:]
for col, typ, flags in data:
if 'AUTOINCREMENT' in flags:
row = self.fetch_one(self.increment_vw, **{
'view': name,
'field': col
})
if not row:
self.increment_vw.append(view=name, field=col, n=1)
row = self.increment_vw[-1]
if not self.increment_dict.has_key(name):
self.increment_dict[name] = {}
self.increment_dict[name][col] = row
getstring += "%s:%s," % (col, self.type_to_metakit_type(typ))
if name == 'recipe':
getstring = getstring + 'categoryname:S,'
getstring = getstring[0:-1] + "]"
debug('Metakit: getting view: %s' % getstring, 5)
vw = self.db.getas(getstring)
debug('Got metakit database', 5)
if key:
rhsh = self.db.getas("__%s_hash__[_H:I,_R:I]" % name)
vw = vw.hash(rhsh, 1)
# Make sure our increment fields are right...
if self.increment_dict.has_key(name):
self.vw_to_name[vw] = name
for field, row in self.increment_dict[name].items():
try:
svw = vw.sort(vw.id)
if len(svw) > svw[-1].id:
print """WTF: increment dicts are foobared. If you see this message, please
submit a bug report with the terminal output included.
"""
metakit.dump(svw)
self.increment_dict[name][field].n = svw[-1].id
except IndexError:
pass
return vw
def type_to_metakit_type(self, typ):
"""Convert a generic database type to a metakit property description."""
if typ.find('char') >= 0:
return 's'
if typ.find('text') >= 0:
return 's'
if typ.find('bool') >= 0:
return 'I'
if typ == 'unicode': return 's'
if typ == 'float': return "F"
if typ == 'int': return "I"
if typ == 'binary': return 'B'
else:
raise Exception("Can't Understand TYPE %s" % typ)
def save(self):
"""Commit our metakit database to file."""
debug('saving database to file %s' % self.file, 0)
debug('there are %s recipes in the database' % len(self.recipe_table),
0)
if self.changed:
self.db.commit()
self.changed = False
def load(self, file=None):
if file:
self.file = file
debug('loading database from file %s' % self.file, 0)
fo = open(self.file, 'rb')
self.db.load(fo)
self.db.autocommit()
fo.close()
# Search functions
def search(self,
table,
colname,
regexp,
exact=0,
use_regexp=True,
recurse=True):
"""Handed a table, a column name, and a regular expression, search
for an item. Alternatively, the regular expression can just be a value."""
debug(
'search %(table)s, %(colname)s, %(regexp)s, %(exact)s, %(use_regexp)s, %(recurse)s'
% locals(), 5)
if (recurse and self.normalizations.has_key(colname)
and isinstance(table, rdatabase.NormalizedView)):
nsrch = self.search(self.normalizations[colname],
colname,
regexp,
exact,
use_regexp,
recurse=False)
if not nsrch: return []
nsrch = nsrch.rename(colname, 'text')
nsrch = nsrch.rename('id', colname)
rvw = table.join(nsrch, getattr(table.__view__, colname))
return rvw
if type(regexp) == type(""):
regexp = str(regexp)
if exact and not use_regexp: return table.select(**{colname: regexp})
if not use_regexp: regexp = re.escape(regexp)
if exact:
indexvw = table.filter(
lambda r: re.match(regexp, "%s" % getattr(r, colname)))
else:
indexvw = table.filter(
lambda r: re.search(regexp, "%s" % getattr(r, colname), re.I))
if indexvw:
resultvw = table.remapwith(indexvw)
resultvw = resultvw.unique()
return resultvw
return []
def ings_search(self,
ings,
keyed=None,
recipe_table=None,
use_regexp=True,
exact=False):
"""Handed a list of regexps, return a list of recipes containing all
items."""
for i in ings:
recipe_table = self.ing_search(i,
keyed=keyed,
recipe_table=recipe_table,
exact=exact,
use_regexp=use_regexp)
return recipe_table
def joined_search(self,
table1,
table2,
search_by,
search_str,
use_regexp=True,
exact=False,
join_on='id'):
table2 = table2.join(table1, getattr(table1, join_on))
vw = self.search(table2,
search_by,
search_str,
use_regexp=use_regexp,
exact=exact)
if not vw: return []
result_ids = vw.counts(getattr(vw, join_on), 'joinedcount')
return table1.join(result_ids, getattr(result_ids, join_on))
def filter(self, table, func):
ivw = table.filter(func)
return table.remapwith(ivw)
# convenience function
def delete_by_criteria(self, table, criteria):
"""Delete table by criteria"""
cur = table.select(**criteria)
if cur:
table.remove(table.indices(cur))
# Our versions of convenience functions for adding/modifying
# recipe stuff
def do_add_rec(self, rdict):
self.remove_unicode(rdict)
debug('adding recipe: %s' % rdict, 5)
r = rdatabase.RecData.do_add_rec(self, rdict)
self.changed = True
return r
def do_modify_rec(self, rec, dic):
# This is a bit ugly, but we need to grab the rec object
# afresh for changes to "stick".
rec = self.get_rec(rec.id)
for k, v in dic.items():
if hasattr(rec, k):
self.changed = True
debug('do_modify_rec: setattr %s %s->%s' % (rec, k, v), 10)
setattr(rec, k, v)
else:
debug(
"Warning: rec has no attribute %s (tried to set to %s)" %
(k, v), 1)
debug('running hooks', 3)
self.run_hooks(self.modify_hooks, rec)
self.changed = True
## delete this code when we've figured out wtf is going on with this not sticking
#for attr in dic.keys():
# debug('modified recipe %s->%s'%(attr,getattr(rec,attr)),1)
return rec
def do_add_ing(self, ingdic):
"""Add ingredient to ingredients_table based on ingdict and return
ingredient object. Ingdict contains:
id: recipe_id
unit: unit
item: description
key: keyed descriptor
alternative: not yet implemented (alternative)
#optional: yes|no
optional: True|False (boolean)
position: INTEGER [position in list]
refid: id of reference recipe. If ref is provided, everything
else is irrelevant except for amount.
"""
self.remove_unicode(ingdic)
if ingdic.has_key('amount') and not ingdic['amount']:
del ingdic['amount']
self.ingredients_table.append(ingdic)
if self.add_ing_hooks:
self.run_hooks(self.add_ing_hooks, self.ingredients_table[-1])
self.changed = True
return self.ingredients_table[-1]
def delete_ing(self, ing):
self.ingredients_table.delete(ing.__index__)
self.changed = True
# Convenience functions
def fetch_one(self, table, *args, **kwargs):
indx = table.find(*args, **kwargs)
if indx >= 0:
return table[indx]
else:
return None
def remove_unicode(self, mydict):
for k, v in mydict.items():
if v.__class__ == u'hello'.__class__:
mydict[k] = v.encode('utf8', 'replace')
if k.__class__ == u'hello'.__class__:
v = mydict[k]
mydict.__delitem__(k)
mydict[k.encode('utf8', 'replace')] = v
def increment_field(self, table, field):
if type(table) != str:
table = self.vw_to_name[table]
self.increment_dict[table][field].n += 1
return self.increment_dict[table][field].n
# Backup / Restructuring methods -- these are special methods to
# help us with our restructuring of the database - updating from
# old versions.
def move_old_tables(self):
self._move_row(table='ingredients',
old=('group', 'text'),
new=('inggroup', 'text'))
self._move_row(table='ingredients',
old=('key', 'char(200)'),
new=('ingkey', 'char(200)'))
self._move_row(table='shopcats',
old=('key', 'char(50)'),
new=('shopkey', 'char(50)'))
self._move_row(table='density',
old=('key', 'char(50)'),
new=('dkey', 'char(50)'))
self._move_row(table='convtable',
old=('key', 'char(50)'),
new=('ckey', 'char(50)'))
self._move_row(table='crossunitdict',
old=('key', 'char(50)'),
new=('cukey', 'char(50)'))
self._move_row(table='unitdict',
old=('key', 'char(50)'),
new=('ukey', 'char(50)'))
# If our recipe or ingredient tables are of the wrong type,
# we're simply going to dump the recipe portion of our DB into
# a file and reload it. This is ugly, but changing row types
# is simply too tricky in metakit and I've had it with
# segfaults and data corruption! At the very least, this
# method ensures the user has an XML copy of their data if the
# DB gets screwed up.
if ((hasattr(self.contentview[0], 'ingredients') and
(self._row_type('ingredients', 'optional') != 'i'
or self._row_type('ingredients', 'id') != 'i'
or self._row_type('ingredients', 'refid') != 'i')) or
(hasattr(self.contentview[0], 'recipe') and
(self._row_type('recipe', 'id') != 'i'
or self._row_type('recipe', 'rating') != 'i'
or self._row_type('recipe', 'cooktime') !=
'i' # To be implemented in the future
or self._row_type('recipe', 'preptime') != 'i' or self._row_type(
'recipe', 'cuisine') != 'i' # of if we're not normalized
or (hasattr(self.contentview[0], 'recipe')
and # of ir we still have 1-category-only
hasattr(self.contentview[0].recipe, 'category'))))):
self.clean_recs_table_and_dump_data()
def copy_table(self,
old_db,
table_name,
table_cols,
prog=None,
convert_pickles=False):
"""Copy columns of table from old database to ourselves.
Old database should be an instance of RecDataOld. Any format
changes that need to happen must happen magically, not here."""
oldvw = getattr(old_db, table_name)
tot = len(oldvw)
newvw = getattr(self, table_name)
if convert_pickles:
import pickle
def unpickle(o):
try:
return pickle.loads(o)
except:
return o
for n, row in enumerate(oldvw):
if n % 10:
prog(float(n) / tot, "Copying other data...%s" % table_name)
if convert_pickles:
newvw.append(
dict([(col, unpickle(getattr(row, col)))
for col in table_cols]))
else:
newvw.append(
dict([(col, getattr(row, col)) for col in table_cols]))
def clean_recs_table_and_dump_data(self):
"""Clean out our recipe table and dump our data for later recovery.
We return an old version of the DB and a file with our recipes.
"""
# Backup our DB
self._backup_database_file()
# Get ourselves out of memory
subrm = RecDataOldDB(self.backupfile)
from gourmet.exporters import gxml2_exporter
#print 'Our old DB looked like this: '
#print subrm.db.description()
#print 'And had'
#print len(subrm.recipe_table),'recipes'
#print 'and '
#print len(subrm.ingredients_table),'ingredients'
# dump our recipe db to a backup file
dumpfile = os.path.join(
os.path.split(self.file)[0],
'recipe_backup_%s.grmt' % (time.strftime('%m-%d-%y')))
self._backup_database_and_make_progress_dialog(dumpfile)
ofi = file(dumpfile, 'w')
gxml2_exporter.recipe_table_to_xml(
subrm,
subrm.recipe_table,
ofi,
one_file=True,
progress_func=lambda p, m: self.pd.set_progress(p * 0.5, m)).run()
ofi.close()
# Now we drop our tables...
#self.db.getas('ingredients')
#self.db.getas('recipe')
# And commit
#self.db.commit()
# Move our current file out of the way so we can start a new
# one, deleting reference to the file so Windows won't
# complain
del self.db
shutil.move(self.file, self.file + '.trash')
# Restart our connection
self.initialize_connection()
# and later on, we'd better import file
self.import_when_done = subrm, dumpfile
def _change_row_types(self, changes):
"""Change row named 'name' from type 'old' to type 'new'
changes = [(table, name, old, new, converter),...]
We have to make all changes at once or bad things will happen.
"""
# This is quite a complex little problem in pymetakit. I got
# help from Brian Kelley <*****@*****.**> over the
# metakit mailing list on this one. Basically, in order to
# change the datatype of a column we have to add a dummy
# column, drop the old column, add the old column back, and
# copy the information over from our dummy. This is made even
# stranger by the fact that to drop a column in metakit, you
# have to call getas() without the column and then call
# commit() and nuke any reference to the db.
# if we don't have this table yet, then we don't need to do anything
DUMMIES = {}
change_dic = {}
default_descs = {}
dummy_descs = {}
for table, name, old, new, converter in changes:
if not hasattr(self.contentview[0], table):
continue
if not default_descs.has_key(table):
default_descs[table] = self.db.description(table)
if not dummy_descs.has_key(table):
dummy_descs[table] = default_descs[table]
if self._row_type(table,
name) == self.type_to_metakit_type(new).lower():
continue
self._backup_database()
# we create a view with our old self and a new temporary self
DUMMIES[name] = 'TMP%s' % name
if not change_dic.has_key(table):
change_dic[table] = {}
change_dic[table][name] = {}
change_dic[table][name]['old'] = old
change_dic[table][name]['new'] = new
change_dic[table][name]['converter'] = converter
self._move_row(table, (name, old), (DUMMIES[name], old))
# We're going to modify our default setup arguments to
# drop the proper column and add the new one in its sted
# before committing. This allows us to "drop" the old
# version of column 'name'
dummy_descs[table] = re.sub("(,|^)%s:" % re.escape(name),
r"\1%s:" % re.escape(DUMMIES[name]),
dummy_descs[table])
if DUMMIES:
# Drop our old columns...
for table, dummy_desc in dummy_descs.items():
self.db.getas("%s[%s]" % (table, dummy_desc))
debug('dropping columns by committing database', 3)
self.db.commit()
debug('deleting reference to our db', 3)
del self.db
debug('reinitialize our connection to a new db', 3)
#self.initialize_connection() # reinitialize ourselves
#with our new DUMMYNAME column
self.db = metakit.storage(self.file, 1)
self.contentview = self.db.contents()
# now we get our new self as a new datatype and copy our
# new information over...
# Loop through the changes we have to make
for table, cd in change_dic.items():
for name, change in cd.items():
newvw = self.setup_table(table,
[(name, change['new']),
(DUMMIES[name], change['old'])])
vw = newvw.filter(lambda x: getattr(x, DUMMIES[name]))
to_move_vw = newvw.remapwith(vw)
debug('converting attributes', 4)
for n, r in enumerate(to_move_vw):
# convert our attributes
setattr(r, name,
change['converter'](getattr(r, DUMMIES[name])))
# erase our temporary/holder attribute
setattr(r, DUMMIES[name], None)
debug('moved attribute %s times' % n, 3)
default_descs[table] = re.sub(
"(,|^)%s:%s" %
(name, self.type_to_metakit_type(change['old'])),
r"\1%s:%s" %
(name, self.type_to_metakit_type(change['new'])),
default_descs[table])
for table, finished_desc in default_descs.items():
self.db.getas(
"%s[%s]" %
(table,
finished_desc)) #setup our table with the right attrs
self.db.commit() # and drop our dummy column
def _move_row(self, table, old, new, converter=None):
"""Move data from old (propertyname, type) to new (propertyname, type).
This is designed for backwards compatability (to allow
for other database backends)."""
debug(
'_move_row(table=%(table)s old=%(old)s new=%(new)s converter=%(converter)s'
% locals(), 1)
if not hasattr(self.contentview[0], table) or not hasattr(
getattr(self.contentview[0], table), old[0]):
debug('Old property %s doesn\'t exist' % old[0], 9)
return
tmpantry_table = self.setup_table(table, [new, old])
vw = tmpantry_table.filter(lambda x: getattr(x, old[0]))
to_move_vw = tmpantry_table.remapwith(vw)
to_move = len(to_move_vw)
if to_move > 0:
self._backup_database()
for r in to_move_vw:
if converter:
setattr(r, new[0], converter(getattr(r, old[0])))
else:
setattr(r, new[0], getattr(r, old[0]))
setattr(r, old[0], None)
def _backup_database_file(self):
"""Create a backup copy of our database tagged with ".OLD" in case we muck things up."""
if not self.backed_up:
backupfile = self.file + '.OLD'
n = 1
while os.path.exists(backupfile):
backupfile = re.split('[0-9]*[.]OLD', backupfile)[0]
backupfile = backupfile + str(n) + '.OLD'
n += 1
shutil.copy(self.file, backupfile)
self.backupfile = backupfile
self.backed_up = True
def _backup_database(self):
if not self.backed_up:
self._backup_database_file()
print """
Your database will not work with older
versions of GOURMET.
A backup has been saved in %s""" % self.backupfile
import gourmet.dialog_extras, gourmet.version
gourmet.dialog_extras.show_message(
label=self.database_change_title,
sublabel=self.database_change_message %
{'backupfile': self.backupfile},
)
def _backup_database_and_make_progress_dialog(self, xmlbackup):
self._backup_database_file()
from gourmet.dialog_extras import ProgressDialog
subl = self.database_change_message % {'backupfile': self.backupfile}
subl += "\n"
subl += _(
"In case anything goes wrong, a backup copy of your recipe database is being exported to %s"
) % xmlbackup
self.pd = ProgressDialog(
title=_("Transferring data to new database format"),
label=self.database_change_title,
sublabel=subl,
pause=False,
stop=False)
self.pd.show()
def _row_type(self, table, name):
"""Return the row type for the table named name"""
if not hasattr(self.contentview[0], table): return
prop = getattr(getattr(self.contentview[0], table),
name) # get ourselves the property object
return prop.type.lower()
