def lookup_website(url): '''Is this website already stored?''' domain = urlparse.urlsplit(validify_url(url))[1] # if not session: # session = Session() lookup = session.query(PieWebsite).filter(PieWebsite.Domain == domain) numrtd = session.query(PieWebsite).filter( PieWebsite.Domain == domain).count() if numrtd > 0: # print '_#_#_#_ Website found:', url, lookup[0] r = lookup[0] r = session.merge(r) return r else: return None
def contribute_folder(path, components): '''Establish - if necessary a new folder on disk and a corresponding PieFolder entry in the database''' root, subfolders, fn = components if os.path.exists(path) and not os.path.isdir(path): raise ValueError, u'Conflict: a file with this path exists: %s' % path if os.path.isdir(path): return True # return if the folder exists else: os.makedirs(path) # don't do db stuff for things not in proper storage directories if root not in ('projectdir', 'librarydir', 'meetingpaperdir', 'recentdocsdir'): return # ensure we don't duplicate any dormant PieFolders existpf = session.query(PieFolder).filter( and_(PieFolder.Root == root, PieFolder.SubFolders == subfolders)).first() if existpf: return existpf else: newpf = PieFolder() newpf.set_path_precut(root, subfolders) session.add(newpf) session.commit() return newpf
def add_new_folder(rootname, foldername): '''add a new folder to a root directory''' assert type(foldername) in (str, unicode) if not rootname in ROOT_MAP.keys(): raise Exception, 'Root directory not found' foldername = translate_non_alphanumerics(foldername) existingfolders = session.query(PieFolder).filter(and_( PieFolder.EndName == foldername, PieFolder.Root == rootname)).all() if len(existingfolders) > 0: raise Exception, 'This folder already exists in database' folderpath = os.path.join(ROOT_MAP[rootname], foldername) if os.path.exists(folderpath) and not os.path.isdir(folderpath): raise Exception, 'Conflict: File with this name already exists on disk' newfolder = PieFolder(os.path.join(ROOT_MAP[rootname], foldername)) if os.path.exists(folderpath) and os.path.isdir(folderpath): hfile = os.path.join(folderpath, _(INFO_FNAME)) if os.path.isfile(hfile): hh = HeaderHandler(headerfile=hfile) newfolder.SecurityLevel = hh.securitylevel newfolder.RecordFile = hh.recordfile else: os.mkdir(newfolder.path()) session.add(newfolder) session.commit() FOLDER_LOOKUP[rootname].append(newfolder) return newfolder
def add_new_folder(rootname, foldername): '''add a new folder to a root directory''' assert type(foldername) in (str, unicode) if not rootname in ROOT_MAP.keys(): raise Exception, 'Root directory not found' foldername = translate_non_alphanumerics(foldername) existingfolders = session.query(PieFolder).filter( and_(PieFolder.EndName == foldername, PieFolder.Root == rootname)).all() if len(existingfolders) > 0: raise Exception, 'This folder already exists in database' folderpath = os.path.join(ROOT_MAP[rootname], foldername) if os.path.exists(folderpath) and not os.path.isdir(folderpath): raise Exception, 'Conflict: File with this name already exists on disk' newfolder = PieFolder(os.path.join(ROOT_MAP[rootname], foldername)) if os.path.exists(folderpath) and os.path.isdir(folderpath): hfile = os.path.join(folderpath, _(INFO_FNAME)) if os.path.isfile(hfile): hh = HeaderHandler(headerfile=hfile) newfolder.SecurityLevel = hh.securitylevel newfolder.RecordFile = hh.recordfile else: os.mkdir(newfolder.path()) session.add(newfolder) session.commit() FOLDER_LOOKUP[rootname].append(newfolder) return newfolder
def has_setting(self, section, key): existingsetting = session.query(PieInternal).filter(and_( PieInternal.section == section, PieInternal.key == key )).first() if existingsetting: return True else: return False
def get(self, section, key): existingsetting = session.query(PieInternal).filter(and_( PieInternal.section == section, PieInternal.key == key )).first() if not existingsetting: raise AttributeError, 'No such setting' return existingsetting.value
def contribute_folder(path, components): '''Establish - if necessary a new folder on disk and a corresponding PieFolder entry in the database''' root, subfolders, fn = components if os.path.exists(path) and not os.path.isdir(path): raise ValueError, u'Conflict: a file with this path exists: %s' % path if os.path.isdir(path): return True # return if the folder exists else: os.makedirs(path) # don't do db stuff for things not in proper storage directories if root not in ('projectdir', 'librarydir', 'meetingpaperdir', 'recentdocsdir'): return # ensure we don't duplicate any dormant PieFolders existpf = session.query(PieFolder).filter(and_( PieFolder.Root == root, PieFolder.SubFolders == subfolders)).first() if existpf: return existpf else: newpf = PieFolder() newpf.set_path_precut(root, subfolders) session.add(newpf) session.commit() return newpf
def lookup_website(url): '''Is this website already stored?''' domain = urlparse.urlsplit(validify_url(url))[1] # if not session: # session = Session() lookup = session.query( PieWebsite).filter(PieWebsite.Domain==domain) numrtd = session.query( PieWebsite).filter(PieWebsite.Domain==domain).count() if numrtd > 0: # print '_#_#_#_ Website found:', url, lookup[0] r = lookup[0] r = session.merge(r) return r else: return None
def generate_folder_list(): '''Walk through the pieberry filesystem and ensure that all folders are indexed''' def gen_subfolders(root_key, curr_dir, sub_dir): '''cut up the path''' ds = curr_dir[len(ROOT_MAP[root_key]):].split(os.sep) ds.append(sub_dir) return [i for i in ds if i] def contribute_projectfolder(piefolder): '''init new primary project folder''' hh = HeaderHandler(piefolder=piefolder) hh.write_header() FOLDER_LOOKUP['projectdir'].append(piefolder) def verify_existing(): for qf in session.query(PieFolder): if not os.path.isdir(qf.path()): # print 'nonexistant folder -', qf session.delete(qf) session.commit() verify_existing() FOLDER_LOOKUP['projectdir'] = [] for root_key in ROOT_MAP.keys(): if root_key in ('cachedir', 'backupdir', 'desktopdir'): continue for curr_dir, subdirs, files in os.walk(ROOT_MAP[root_key]): for subdir in subdirs: cut_subdirs = gen_subfolders(root_key, curr_dir, subdir) # is there an existing piefolder in the db for this? exisf = session.query(PieFolder).filter( and_( PieFolder.Root == root_key, PieFolder.SubFolders == cut_subdirs, )).first() if not exisf: # if the folder isn't already in the db print 'creating:', os.path.join(curr_dir, subdir) print 'vars:', cut_subdirs, root_key # create a new piefolder object n_piefolder = PieFolder() n_piefolder.set_path_precut(root_key, cut_subdirs) session.add(n_piefolder) if curr_dir == ROOT_MAP['projectdir']: contribute_projectfolder(n_piefolder) print n_piefolder else: if curr_dir == ROOT_MAP['projectdir']: FOLDER_LOOKUP['projectdir'].append(exisf) print 'found folder:', exisf # Generate 'special' folders (i.e. the various root folders) FOLDER_LOOKUP['special'] = [] # special (non-persistent) folders # for the principle storage # locations FOLDER_LOOKUP['special'].append(PieFolder(ROOT_MAP['projectdir'])) FOLDER_LOOKUP['special'].append(PieFolder(ROOT_MAP['librarydir'])) FOLDER_LOOKUP['special'].append(PieFolder(ROOT_MAP['meetingpaperdir']))
def generate_folder_list(): '''Walk through the pieberry filesystem and ensure that all folders are indexed''' def gen_subfolders(root_key, curr_dir, sub_dir): '''cut up the path''' ds = curr_dir[len(ROOT_MAP[root_key]):].split(os.sep) ds.append(sub_dir) return [i for i in ds if i] def contribute_projectfolder(piefolder): '''init new primary project folder''' hh = HeaderHandler(piefolder=piefolder) hh.write_header() FOLDER_LOOKUP['projectdir'].append(piefolder) def verify_existing(): for qf in session.query(PieFolder): if not os.path.isdir(qf.path()): # print 'nonexistant folder -', qf session.delete(qf) session.commit() verify_existing() FOLDER_LOOKUP['projectdir'] = [] for root_key in ROOT_MAP.keys(): if root_key in ('cachedir', 'backupdir', 'desktopdir'): continue for curr_dir, subdirs, files in os.walk(ROOT_MAP[root_key]): for subdir in subdirs: cut_subdirs = gen_subfolders(root_key, curr_dir, subdir) # is there an existing piefolder in the db for this? exisf = session.query(PieFolder).filter(and_( PieFolder.Root == root_key, PieFolder.SubFolders == cut_subdirs, )).first() if not exisf: # if the folder isn't already in the db print 'creating:', os.path.join(curr_dir, subdir) print 'vars:', cut_subdirs, root_key # create a new piefolder object n_piefolder = PieFolder() n_piefolder.set_path_precut(root_key, cut_subdirs) session.add(n_piefolder) if curr_dir == ROOT_MAP['projectdir']: contribute_projectfolder(n_piefolder) print n_piefolder else: if curr_dir == ROOT_MAP['projectdir']: FOLDER_LOOKUP['projectdir'].append(exisf) print 'found folder:', exisf # Generate 'special' folders (i.e. the various root folders) FOLDER_LOOKUP['special'] = [] # special (non-persistent) folders # for the principle storage # locations FOLDER_LOOKUP['special'].append(PieFolder(ROOT_MAP['projectdir'])) FOLDER_LOOKUP['special'].append(PieFolder(ROOT_MAP['librarydir'])) FOLDER_LOOKUP['special'].append(PieFolder(ROOT_MAP['meetingpaperdir']))
def set(self, section, key, value): assert type(section) in (unicode, str) assert type(key) in (unicode, str) existingsetting = session.query(PieInternal).filter( and_(PieInternal.section == section, PieInternal.key == key)).first() if existingsetting: existingsetting.value = unicode(value) else: newsetting = PieInternal(section, key, value) session.add(newsetting) session.commit()
def set(self, section, key, value): assert type(section) in (unicode, str) assert type(key) in (unicode, str) existingsetting = session.query(PieInternal).filter(and_( PieInternal.section == section, PieInternal.key == key )).first() if existingsetting: existingsetting.value = unicode(value) else: newsetting = PieInternal(section, key, value) session.add(newsetting) session.commit()
def generate_initial_project_folder_list(): '''Do a filesystem sweep and return a list of the existing folders''' FOLDER_LOOKUP['projectdir'] = [] existing = session.query( PieFolder).filter(PieFolder.Root == 'projectdir').all() for folder in existing: if not os.path.isdir(folder.path()): print u'Warning - %s was loaded but does not exist' % existing.path() FOLDER_LOOKUP['projectdir'].extend(existing) for diry in os.walk(ROOT_MAP['projectdir']).next()[1]: if not get_project_folder_by_endname(diry): newf = add_new_folder('projectdir', diry) print u'Note: new unmapped project folder %s added' % os.path.join( ROOT_MAP['projectdir'], diry)
def generate_initial_project_folder_list(): '''Do a filesystem sweep and return a list of the existing folders''' FOLDER_LOOKUP['projectdir'] = [] existing = session.query(PieFolder).filter( PieFolder.Root == 'projectdir').all() for folder in existing: if not os.path.isdir(folder.path()): print u'Warning - %s was loaded but does not exist' % existing.path( ) FOLDER_LOOKUP['projectdir'].extend(existing) for diry in os.walk(ROOT_MAP['projectdir']).next()[1]: if not get_project_folder_by_endname(diry): newf = add_new_folder('projectdir', diry) print u'Note: new unmapped project folder %s added' % os.path.join( ROOT_MAP['projectdir'], diry)
def contribute_and_get_folder(path, components): '''Like contribute_folder, but be sure to return the folder that was contributed (more DB intensive)''' root, subfolders, fn = components if os.path.exists(path) and not os.path.isdir(path): raise Exception, u'Conflict: a file with this path exists: %s' % path if not os.path.isdir(path): os.makedirs(path) existpf = session.query(PieFolder).filter(and_( PieFolder.Root == root, PieFolder.SubFolders == subfolders)).first() if existpf: return existpf else: newpf = PieFolder() newpf.set_path_precut(root, subfolders) session.add(newpf) session.commit() return newpf
def recommend_folder(obj): fs = session.query(PieFolder).filter(PieFolder.Root == 'projectdir') f_cands = [] for f in fs: score = 0 for aterm in f.MatchTerms_Author: if aterm.lower() in obj.Author().lower(): score += 1 for tterm in f.MatchTerms_Title: if tterm.lower() in obj.Title().lower(): score += 1 if tterm.lower() in obj.FileData_FileName.lower(): score += 1 if score > 0: f_cands.append((score, f)) if len(f_cands) == 0: return f_cands.sort(lambda x, y: cmp(y[0], x[0])) pprint(f_cands) return f_cands[0][1] # return a PieFolder
def contribute_and_get_folder(path, components): '''Like contribute_folder, but be sure to return the folder that was contributed (more DB intensive)''' root, subfolders, fn = components if os.path.exists(path) and not os.path.isdir(path): raise Exception, u'Conflict: a file with this path exists: %s' % path if not os.path.isdir(path): os.makedirs(path) existpf = session.query(PieFolder).filter( and_(PieFolder.Root == root, PieFolder.SubFolders == subfolders)).first() if existpf: return existpf else: newpf = PieFolder() newpf.set_path_precut(root, subfolders) session.add(newpf) session.commit() return newpf
def DebugRectifyObjectRels(self, evt): '''Fix up objects and relationships - it's a version upgrade thing''' from pieberry.pieobject import reconcile_object_folder_gen from pieberry.piedb import session from pieberry.pieobject import PieObject progress_dialog = wx.ProgressDialog( _('Rectifying object relationships'), '___________________________________________', maximum = session.query(PieObject).filter(PieObject.FileData_FileName != None).count() ) counter = 0 for obj in reconcile_object_folder_gen(): # print 'assert st 1' # assert obj.aspects.has_key('encrypted') # session.add(obj) # session.flush() # print 'assert st 2' # assert obj.aspects.has_key('encrypted') counter += 1 progress_dialog.Update(counter, unicode(obj)) progress_dialog.Destroy() session.commit()
def DebugRectifyObjectRels(self, evt): '''Fix up objects and relationships - it's a version upgrade thing''' from pieberry.pieobject import reconcile_object_folder_gen from pieberry.piedb import session from pieberry.pieobject import PieObject progress_dialog = wx.ProgressDialog( _('Rectifying object relationships'), '___________________________________________', maximum=session.query(PieObject).filter( PieObject.FileData_FileName != None).count()) counter = 0 for obj in reconcile_object_folder_gen(): # print 'assert st 1' # assert obj.aspects.has_key('encrypted') # session.add(obj) # session.flush() # print 'assert st 2' # assert obj.aspects.has_key('encrypted') counter += 1 progress_dialog.Update(counter, unicode(obj)) progress_dialog.Destroy() session.commit()
def tag_exists(t): q = session.query(PieTag).filter(PieTag.TagName == t).all() if q: return True return False
def gen_tag_lookup(): global ALL_TAGS ALL_TAGS = {} q = session.query(PieTag) for tag in q: ALL_TAGS[tag.TagName] = tag
def get_authorlist(): '''Get list of default authors known to the database''' # session = Session() query = session.query(PieWebsite).order_by(PieWebsite.DefaultAuthor) return [pw.DefaultAuthor for pw in query]
def get_project_folder_by_endname(endname): for fldr in session.query(PieFolder).filter( and_(PieFolder.Root == 'projectdir', PieFolder.EndName == endname)): return fldr return None
def verify_existing(): for qf in session.query(PieFolder): if not os.path.isdir(qf.path()): # print 'nonexistant folder -', qf session.delete(qf) session.commit()
def has_setting(self, section, key): existingsetting = session.query(PieInternal).filter( and_(PieInternal.section == section, PieInternal.key == key)).first() if existingsetting: return True else: return False
def get_project_folder_by_endname(endname): for fldr in session.query(PieFolder).filter(and_( PieFolder.Root == 'projectdir', PieFolder.EndName == endname)): return fldr return None
def get(self, section, key): existingsetting = session.query(PieInternal).filter( and_(PieInternal.section == section, PieInternal.key == key)).first() if not existingsetting: raise AttributeError, 'No such setting' return existingsetting.value