def lookup_website(url):
'''Is this website already stored?'''
domain = urlparse.urlsplit(validify_url(url))[1]
# if not session:
# session = Session()
lookup = session.query(PieWebsite).filter(PieWebsite.Domain == domain)
numrtd = session.query(PieWebsite).filter(
PieWebsite.Domain == domain).count()
if numrtd > 0:
# print '_#_#_#_ Website found:', url, lookup[0]
r = lookup[0]
r = session.merge(r)
return r
else:
return None
def contribute_folder(path, components):
'''Establish - if necessary a new folder on disk and a
corresponding PieFolder entry in the database'''
root, subfolders, fn = components
if os.path.exists(path) and not os.path.isdir(path):
raise ValueError, u'Conflict: a file with this path exists: %s' % path
if os.path.isdir(path):
return True # return if the folder exists
else:
os.makedirs(path)
# don't do db stuff for things not in proper storage directories
if root not in ('projectdir', 'librarydir', 'meetingpaperdir',
'recentdocsdir'):
return
# ensure we don't duplicate any dormant PieFolders
existpf = session.query(PieFolder).filter(
and_(PieFolder.Root == root,
PieFolder.SubFolders == subfolders)).first()
if existpf: return existpf
else:
newpf = PieFolder()
newpf.set_path_precut(root, subfolders)
session.add(newpf)
session.commit()
return newpf
def add_new_folder(rootname, foldername):
'''add a new folder to a root directory'''
assert type(foldername) in (str, unicode)
if not rootname in ROOT_MAP.keys():
raise Exception, 'Root directory not found'
foldername = translate_non_alphanumerics(foldername)
existingfolders = session.query(PieFolder).filter(and_(
PieFolder.EndName == foldername,
PieFolder.Root == rootname)).all()
if len(existingfolders) > 0:
raise Exception, 'This folder already exists in database'
folderpath = os.path.join(ROOT_MAP[rootname], foldername)
if os.path.exists(folderpath) and not os.path.isdir(folderpath):
raise Exception, 'Conflict: File with this name already exists on disk'
newfolder = PieFolder(os.path.join(ROOT_MAP[rootname], foldername))
if os.path.exists(folderpath) and os.path.isdir(folderpath):
hfile = os.path.join(folderpath, _(INFO_FNAME))
if os.path.isfile(hfile):
hh = HeaderHandler(headerfile=hfile)
newfolder.SecurityLevel = hh.securitylevel
newfolder.RecordFile = hh.recordfile
else:
os.mkdir(newfolder.path())
session.add(newfolder)
session.commit()
FOLDER_LOOKUP[rootname].append(newfolder)
return newfolder
def add_new_folder(rootname, foldername):
'''add a new folder to a root directory'''
assert type(foldername) in (str, unicode)
if not rootname in ROOT_MAP.keys():
raise Exception, 'Root directory not found'
foldername = translate_non_alphanumerics(foldername)
existingfolders = session.query(PieFolder).filter(
and_(PieFolder.EndName == foldername,
PieFolder.Root == rootname)).all()
if len(existingfolders) > 0:
raise Exception, 'This folder already exists in database'
folderpath = os.path.join(ROOT_MAP[rootname], foldername)
if os.path.exists(folderpath) and not os.path.isdir(folderpath):
raise Exception, 'Conflict: File with this name already exists on disk'
newfolder = PieFolder(os.path.join(ROOT_MAP[rootname], foldername))
if os.path.exists(folderpath) and os.path.isdir(folderpath):
hfile = os.path.join(folderpath, _(INFO_FNAME))
if os.path.isfile(hfile):
hh = HeaderHandler(headerfile=hfile)
newfolder.SecurityLevel = hh.securitylevel
newfolder.RecordFile = hh.recordfile
else:
os.mkdir(newfolder.path())
session.add(newfolder)
session.commit()
FOLDER_LOOKUP[rootname].append(newfolder)
return newfolder
def has_setting(self, section, key):
existingsetting = session.query(PieInternal).filter(and_(
PieInternal.section == section,
PieInternal.key == key
)).first()
if existingsetting: return True
else: return False
def get(self, section, key):
existingsetting = session.query(PieInternal).filter(and_(
PieInternal.section == section,
PieInternal.key == key
)).first()
if not existingsetting: raise AttributeError, 'No such setting'
return existingsetting.value
def contribute_folder(path, components):
'''Establish - if necessary a new folder on disk and a
corresponding PieFolder entry in the database'''
root, subfolders, fn = components
if os.path.exists(path) and not os.path.isdir(path):
raise ValueError, u'Conflict: a file with this path exists: %s' % path
if os.path.isdir(path):
return True # return if the folder exists
else:
os.makedirs(path)
# don't do db stuff for things not in proper storage directories
if root not in ('projectdir',
'librarydir',
'meetingpaperdir',
'recentdocsdir'):
return
# ensure we don't duplicate any dormant PieFolders
existpf = session.query(PieFolder).filter(and_(
PieFolder.Root == root,
PieFolder.SubFolders == subfolders)).first()
if existpf: return existpf
else:
newpf = PieFolder()
newpf.set_path_precut(root, subfolders)
session.add(newpf)
session.commit()
return newpf
def lookup_website(url):
'''Is this website already stored?'''
domain = urlparse.urlsplit(validify_url(url))[1]
# if not session:
# session = Session()
lookup = session.query(
PieWebsite).filter(PieWebsite.Domain==domain)
numrtd = session.query(
PieWebsite).filter(PieWebsite.Domain==domain).count()
if numrtd > 0:
# print '_#_#_#_ Website found:', url, lookup[0]
r = lookup[0]
r = session.merge(r)
return r
else:
return None
def generate_folder_list():
'''Walk through the pieberry filesystem and ensure that all
folders are indexed'''
def gen_subfolders(root_key, curr_dir, sub_dir):
'''cut up the path'''
ds = curr_dir[len(ROOT_MAP[root_key]):].split(os.sep)
ds.append(sub_dir)
return [i for i in ds if i]
def contribute_projectfolder(piefolder):
'''init new primary project folder'''
hh = HeaderHandler(piefolder=piefolder)
hh.write_header()
FOLDER_LOOKUP['projectdir'].append(piefolder)
def verify_existing():
for qf in session.query(PieFolder):
if not os.path.isdir(qf.path()):
# print 'nonexistant folder -', qf
session.delete(qf)
session.commit()
verify_existing()
FOLDER_LOOKUP['projectdir'] = []
for root_key in ROOT_MAP.keys():
if root_key in ('cachedir', 'backupdir', 'desktopdir'): continue
for curr_dir, subdirs, files in os.walk(ROOT_MAP[root_key]):
for subdir in subdirs:
cut_subdirs = gen_subfolders(root_key, curr_dir, subdir)
# is there an existing piefolder in the db for this?
exisf = session.query(PieFolder).filter(
and_(
PieFolder.Root == root_key,
PieFolder.SubFolders == cut_subdirs,
)).first()
if not exisf: # if the folder isn't already in the db
print 'creating:', os.path.join(curr_dir, subdir)
print 'vars:', cut_subdirs, root_key
# create a new piefolder object
n_piefolder = PieFolder()
n_piefolder.set_path_precut(root_key, cut_subdirs)
session.add(n_piefolder)
if curr_dir == ROOT_MAP['projectdir']:
contribute_projectfolder(n_piefolder)
print n_piefolder
else:
if curr_dir == ROOT_MAP['projectdir']:
FOLDER_LOOKUP['projectdir'].append(exisf)
print 'found folder:', exisf
# Generate 'special' folders (i.e. the various root folders)
FOLDER_LOOKUP['special'] = [] # special (non-persistent) folders
# for the principle storage
# locations
FOLDER_LOOKUP['special'].append(PieFolder(ROOT_MAP['projectdir']))
FOLDER_LOOKUP['special'].append(PieFolder(ROOT_MAP['librarydir']))
FOLDER_LOOKUP['special'].append(PieFolder(ROOT_MAP['meetingpaperdir']))
def generate_folder_list():
'''Walk through the pieberry filesystem and ensure that all
folders are indexed'''
def gen_subfolders(root_key, curr_dir, sub_dir):
'''cut up the path'''
ds = curr_dir[len(ROOT_MAP[root_key]):].split(os.sep)
ds.append(sub_dir)
return [i for i in ds if i]
def contribute_projectfolder(piefolder):
'''init new primary project folder'''
hh = HeaderHandler(piefolder=piefolder)
hh.write_header()
FOLDER_LOOKUP['projectdir'].append(piefolder)
def verify_existing():
for qf in session.query(PieFolder):
if not os.path.isdir(qf.path()):
# print 'nonexistant folder -', qf
session.delete(qf)
session.commit()
verify_existing()
FOLDER_LOOKUP['projectdir'] = []
for root_key in ROOT_MAP.keys():
if root_key in ('cachedir', 'backupdir', 'desktopdir'): continue
for curr_dir, subdirs, files in os.walk(ROOT_MAP[root_key]):
for subdir in subdirs:
cut_subdirs = gen_subfolders(root_key, curr_dir, subdir)
# is there an existing piefolder in the db for this?
exisf = session.query(PieFolder).filter(and_(
PieFolder.Root == root_key,
PieFolder.SubFolders == cut_subdirs,
)).first()
if not exisf: # if the folder isn't already in the db
print 'creating:', os.path.join(curr_dir, subdir)
print 'vars:', cut_subdirs, root_key
# create a new piefolder object
n_piefolder = PieFolder()
n_piefolder.set_path_precut(root_key, cut_subdirs)
session.add(n_piefolder)
if curr_dir == ROOT_MAP['projectdir']:
contribute_projectfolder(n_piefolder)
print n_piefolder
else:
if curr_dir == ROOT_MAP['projectdir']:
FOLDER_LOOKUP['projectdir'].append(exisf)
print 'found folder:', exisf
# Generate 'special' folders (i.e. the various root folders)
FOLDER_LOOKUP['special'] = [] # special (non-persistent) folders
# for the principle storage
# locations
FOLDER_LOOKUP['special'].append(PieFolder(ROOT_MAP['projectdir']))
FOLDER_LOOKUP['special'].append(PieFolder(ROOT_MAP['librarydir']))
FOLDER_LOOKUP['special'].append(PieFolder(ROOT_MAP['meetingpaperdir']))
def set(self, section, key, value):
assert type(section) in (unicode, str)
assert type(key) in (unicode, str)
existingsetting = session.query(PieInternal).filter(
and_(PieInternal.section == section,
PieInternal.key == key)).first()
if existingsetting:
existingsetting.value = unicode(value)
else:
newsetting = PieInternal(section, key, value)
session.add(newsetting)
session.commit()
def set(self, section, key, value):
assert type(section) in (unicode, str)
assert type(key) in (unicode, str)
existingsetting = session.query(PieInternal).filter(and_(
PieInternal.section == section,
PieInternal.key == key
)).first()
if existingsetting:
existingsetting.value = unicode(value)
else:
newsetting = PieInternal(section, key, value)
session.add(newsetting)
session.commit()
def generate_initial_project_folder_list():
'''Do a filesystem sweep and return a list of the existing folders'''
FOLDER_LOOKUP['projectdir'] = []
existing = session.query(
PieFolder).filter(PieFolder.Root == 'projectdir').all()
for folder in existing:
if not os.path.isdir(folder.path()):
print u'Warning - %s was loaded but does not exist' % existing.path()
FOLDER_LOOKUP['projectdir'].extend(existing)
for diry in os.walk(ROOT_MAP['projectdir']).next()[1]:
if not get_project_folder_by_endname(diry):
newf = add_new_folder('projectdir', diry)
print u'Note: new unmapped project folder %s added' % os.path.join(
ROOT_MAP['projectdir'], diry)
def generate_initial_project_folder_list():
'''Do a filesystem sweep and return a list of the existing folders'''
FOLDER_LOOKUP['projectdir'] = []
existing = session.query(PieFolder).filter(
PieFolder.Root == 'projectdir').all()
for folder in existing:
if not os.path.isdir(folder.path()):
print u'Warning - %s was loaded but does not exist' % existing.path(
)
FOLDER_LOOKUP['projectdir'].extend(existing)
for diry in os.walk(ROOT_MAP['projectdir']).next()[1]:
if not get_project_folder_by_endname(diry):
newf = add_new_folder('projectdir', diry)
print u'Note: new unmapped project folder %s added' % os.path.join(
ROOT_MAP['projectdir'], diry)
def contribute_and_get_folder(path, components):
'''Like contribute_folder, but be sure to return the folder that
was contributed (more DB intensive)'''
root, subfolders, fn = components
if os.path.exists(path) and not os.path.isdir(path):
raise Exception, u'Conflict: a file with this path exists: %s' % path
if not os.path.isdir(path):
os.makedirs(path)
existpf = session.query(PieFolder).filter(and_(
PieFolder.Root == root,
PieFolder.SubFolders == subfolders)).first()
if existpf: return existpf
else:
newpf = PieFolder()
newpf.set_path_precut(root, subfolders)
session.add(newpf)
session.commit()
return newpf
def recommend_folder(obj):
fs = session.query(PieFolder).filter(PieFolder.Root == 'projectdir')
f_cands = []
for f in fs:
score = 0
for aterm in f.MatchTerms_Author:
if aterm.lower() in obj.Author().lower():
score += 1
for tterm in f.MatchTerms_Title:
if tterm.lower() in obj.Title().lower():
score += 1
if tterm.lower() in obj.FileData_FileName.lower():
score += 1
if score > 0: f_cands.append((score, f))
if len(f_cands) == 0: return
f_cands.sort(lambda x, y: cmp(y[0], x[0]))
pprint(f_cands)
return f_cands[0][1] # return a PieFolder
def recommend_folder(obj):
fs = session.query(PieFolder).filter(PieFolder.Root == 'projectdir')
f_cands = []
for f in fs:
score = 0
for aterm in f.MatchTerms_Author:
if aterm.lower() in obj.Author().lower():
score += 1
for tterm in f.MatchTerms_Title:
if tterm.lower() in obj.Title().lower():
score += 1
if tterm.lower() in obj.FileData_FileName.lower():
score += 1
if score > 0: f_cands.append((score, f))
if len(f_cands) == 0: return
f_cands.sort(lambda x, y: cmp(y[0], x[0]))
pprint(f_cands)
return f_cands[0][1] # return a PieFolder
def contribute_and_get_folder(path, components):
'''Like contribute_folder, but be sure to return the folder that
was contributed (more DB intensive)'''
root, subfolders, fn = components
if os.path.exists(path) and not os.path.isdir(path):
raise Exception, u'Conflict: a file with this path exists: %s' % path
if not os.path.isdir(path):
os.makedirs(path)
existpf = session.query(PieFolder).filter(
and_(PieFolder.Root == root,
PieFolder.SubFolders == subfolders)).first()
if existpf: return existpf
else:
newpf = PieFolder()
newpf.set_path_precut(root, subfolders)
session.add(newpf)
session.commit()
return newpf
def DebugRectifyObjectRels(self, evt):
'''Fix up objects and relationships - it's a version upgrade
thing'''
from pieberry.pieobject import reconcile_object_folder_gen
from pieberry.piedb import session
from pieberry.pieobject import PieObject
progress_dialog = wx.ProgressDialog(
_('Rectifying object relationships'),
'___________________________________________', maximum = session.query(PieObject).filter(PieObject.FileData_FileName != None).count() )
counter = 0
for obj in reconcile_object_folder_gen():
# print 'assert st 1'
# assert obj.aspects.has_key('encrypted')
# session.add(obj)
# session.flush()
# print 'assert st 2'
# assert obj.aspects.has_key('encrypted')
counter += 1
progress_dialog.Update(counter, unicode(obj))
progress_dialog.Destroy()
session.commit()
def DebugRectifyObjectRels(self, evt):
'''Fix up objects and relationships - it's a version upgrade
thing'''
from pieberry.pieobject import reconcile_object_folder_gen
from pieberry.piedb import session
from pieberry.pieobject import PieObject
progress_dialog = wx.ProgressDialog(
_('Rectifying object relationships'),
'___________________________________________',
maximum=session.query(PieObject).filter(
PieObject.FileData_FileName != None).count())
counter = 0
for obj in reconcile_object_folder_gen():
# print 'assert st 1'
# assert obj.aspects.has_key('encrypted')
# session.add(obj)
# session.flush()
# print 'assert st 2'
# assert obj.aspects.has_key('encrypted')
counter += 1
progress_dialog.Update(counter, unicode(obj))
progress_dialog.Destroy()
session.commit()
def tag_exists(t):
q = session.query(PieTag).filter(PieTag.TagName == t).all()
if q: return True
return False
def tag_exists(t):
q = session.query(PieTag).filter(PieTag.TagName == t).all()
if q: return True
return False
def gen_tag_lookup():
global ALL_TAGS
ALL_TAGS = {}
q = session.query(PieTag)
for tag in q:
ALL_TAGS[tag.TagName] = tag
def get_authorlist():
'''Get list of default authors known to the database'''
# session = Session()
query = session.query(PieWebsite).order_by(PieWebsite.DefaultAuthor)
return [pw.DefaultAuthor for pw in query]
def get_project_folder_by_endname(endname):
for fldr in session.query(PieFolder).filter(
and_(PieFolder.Root == 'projectdir',
PieFolder.EndName == endname)):
return fldr
return None
def gen_tag_lookup():
global ALL_TAGS
ALL_TAGS = {}
q = session.query(PieTag)
for tag in q:
ALL_TAGS[tag.TagName] = tag
def verify_existing():
for qf in session.query(PieFolder):
if not os.path.isdir(qf.path()):
# print 'nonexistant folder -', qf
session.delete(qf)
session.commit()
def has_setting(self, section, key):
existingsetting = session.query(PieInternal).filter(
and_(PieInternal.section == section,
PieInternal.key == key)).first()
if existingsetting: return True
else: return False
def get_project_folder_by_endname(endname):
for fldr in session.query(PieFolder).filter(and_(
PieFolder.Root == 'projectdir',
PieFolder.EndName == endname)):
return fldr
return None
def get_authorlist():
'''Get list of default authors known to the database'''
# session = Session()
query = session.query(PieWebsite).order_by(PieWebsite.DefaultAuthor)
return [pw.DefaultAuthor for pw in query]
def get(self, section, key):
existingsetting = session.query(PieInternal).filter(
and_(PieInternal.section == section,
PieInternal.key == key)).first()
if not existingsetting: raise AttributeError, 'No such setting'
return existingsetting.value
def verify_existing():
for qf in session.query(PieFolder):
if not os.path.isdir(qf.path()):
# print 'nonexistant folder -', qf
session.delete(qf)
session.commit()