Beispiel #1
0
def add_new_folder(rootname, foldername):
    '''add a new folder to a root directory'''
    assert type(foldername) in (str, unicode)
    if not rootname in ROOT_MAP.keys():
        raise Exception, 'Root directory not found'
    foldername = translate_non_alphanumerics(foldername)
    existingfolders = session.query(PieFolder).filter(
        and_(PieFolder.EndName == foldername,
             PieFolder.Root == rootname)).all()
    if len(existingfolders) > 0:
        raise Exception, 'This folder already exists in database'
    folderpath = os.path.join(ROOT_MAP[rootname], foldername)
    if os.path.exists(folderpath) and not os.path.isdir(folderpath):
        raise Exception, 'Conflict: File with this name already exists on disk'
    newfolder = PieFolder(os.path.join(ROOT_MAP[rootname], foldername))
    if os.path.exists(folderpath) and os.path.isdir(folderpath):
        hfile = os.path.join(folderpath, _(INFO_FNAME))
        if os.path.isfile(hfile):
            hh = HeaderHandler(headerfile=hfile)
            newfolder.SecurityLevel = hh.securitylevel
            newfolder.RecordFile = hh.recordfile
    else:
        os.mkdir(newfolder.path())
    session.add(newfolder)
    session.commit()
    FOLDER_LOOKUP[rootname].append(newfolder)
    return newfolder
Beispiel #2
0
def add_new_folder(rootname, foldername):
    '''add a new folder to a root directory'''
    assert type(foldername) in (str, unicode)
    if not rootname in ROOT_MAP.keys():
        raise Exception, 'Root directory not found'
    foldername = translate_non_alphanumerics(foldername)
    existingfolders = session.query(PieFolder).filter(and_(
            PieFolder.EndName == foldername,
            PieFolder.Root == rootname)).all()
    if len(existingfolders) > 0:
        raise Exception, 'This folder already exists in database'
    folderpath = os.path.join(ROOT_MAP[rootname], foldername)
    if os.path.exists(folderpath) and not os.path.isdir(folderpath):
        raise Exception, 'Conflict: File with this name already exists on disk'
    newfolder = PieFolder(os.path.join(ROOT_MAP[rootname], foldername))
    if os.path.exists(folderpath) and os.path.isdir(folderpath):
        hfile = os.path.join(folderpath, _(INFO_FNAME))
        if os.path.isfile(hfile):
            hh = HeaderHandler(headerfile=hfile)
            newfolder.SecurityLevel = hh.securitylevel
            newfolder.RecordFile = hh.recordfile
    else:
        os.mkdir(newfolder.path())
    session.add(newfolder)
    session.commit()
    FOLDER_LOOKUP[rootname].append(newfolder)
    return newfolder
Beispiel #3
0
def generate_folder_list():
    '''Walk through the pieberry filesystem and ensure that all
    folders are indexed'''
    def gen_subfolders(root_key, curr_dir, sub_dir):
        '''cut up the path'''
        ds = curr_dir[len(ROOT_MAP[root_key]):].split(os.sep)
        ds.append(sub_dir)
        return [i for i in ds if i]

    def contribute_projectfolder(piefolder):
        '''init new primary project folder'''
        hh = HeaderHandler(piefolder=piefolder)
        hh.write_header()
        FOLDER_LOOKUP['projectdir'].append(piefolder)

    def verify_existing():
        for qf in session.query(PieFolder):
            if not os.path.isdir(qf.path()):
                # print 'nonexistant folder -', qf
                session.delete(qf)
        session.commit()

    verify_existing()

    FOLDER_LOOKUP['projectdir'] = []
    for root_key in ROOT_MAP.keys():
        if root_key in ('cachedir', 'backupdir', 'desktopdir'): continue
        for curr_dir, subdirs, files in os.walk(ROOT_MAP[root_key]):
            for subdir in subdirs:
                cut_subdirs = gen_subfolders(root_key, curr_dir, subdir)
                # is there an existing piefolder in the db for this?
                exisf = session.query(PieFolder).filter(
                    and_(
                        PieFolder.Root == root_key,
                        PieFolder.SubFolders == cut_subdirs,
                    )).first()
                if not exisf:  # if the folder isn't already in the db
                    print 'creating:', os.path.join(curr_dir, subdir)
                    print 'vars:', cut_subdirs, root_key
                    # create a new piefolder object
                    n_piefolder = PieFolder()
                    n_piefolder.set_path_precut(root_key, cut_subdirs)
                    session.add(n_piefolder)
                    if curr_dir == ROOT_MAP['projectdir']:
                        contribute_projectfolder(n_piefolder)
                    print n_piefolder
                else:
                    if curr_dir == ROOT_MAP['projectdir']:
                        FOLDER_LOOKUP['projectdir'].append(exisf)
                    print 'found folder:', exisf

    # Generate 'special' folders (i.e. the various root folders)
    FOLDER_LOOKUP['special'] = []  # special (non-persistent) folders
    # for the principle storage
    # locations
    FOLDER_LOOKUP['special'].append(PieFolder(ROOT_MAP['projectdir']))
    FOLDER_LOOKUP['special'].append(PieFolder(ROOT_MAP['librarydir']))
    FOLDER_LOOKUP['special'].append(PieFolder(ROOT_MAP['meetingpaperdir']))
Beispiel #4
0
def generate_folder_list():
    '''Walk through the pieberry filesystem and ensure that all
    folders are indexed'''

    def gen_subfolders(root_key, curr_dir, sub_dir):
        '''cut up the path'''
        ds = curr_dir[len(ROOT_MAP[root_key]):].split(os.sep)
        ds.append(sub_dir)
        return [i for i in ds if i]

    def contribute_projectfolder(piefolder):
        '''init new primary project folder'''
        hh = HeaderHandler(piefolder=piefolder)
        hh.write_header()
        FOLDER_LOOKUP['projectdir'].append(piefolder)
    
    def verify_existing():
        for qf in session.query(PieFolder):
            if not os.path.isdir(qf.path()):
                # print 'nonexistant folder -', qf
                session.delete(qf)
        session.commit()

    verify_existing()

    FOLDER_LOOKUP['projectdir'] = []
    for root_key in ROOT_MAP.keys():
        if root_key in ('cachedir', 'backupdir', 'desktopdir'): continue
        for curr_dir, subdirs, files in os.walk(ROOT_MAP[root_key]):
            for subdir in subdirs:
                cut_subdirs = gen_subfolders(root_key, curr_dir, subdir)
                # is there an existing piefolder in the db for this?
                exisf = session.query(PieFolder).filter(and_(
                        PieFolder.Root == root_key,
                        PieFolder.SubFolders == cut_subdirs,
                        )).first()
                if not exisf: # if the folder isn't already in the db
                    print 'creating:', os.path.join(curr_dir, subdir)
                    print 'vars:', cut_subdirs, root_key
                    # create a new piefolder object
                    n_piefolder = PieFolder()
                    n_piefolder.set_path_precut(root_key, cut_subdirs)
                    session.add(n_piefolder)
                    if curr_dir == ROOT_MAP['projectdir']:
                        contribute_projectfolder(n_piefolder)
                    print n_piefolder
                else:  
                    if curr_dir == ROOT_MAP['projectdir']:
                        FOLDER_LOOKUP['projectdir'].append(exisf)
                    print 'found folder:', exisf

    # Generate 'special' folders (i.e. the various root folders)
    FOLDER_LOOKUP['special'] = [] # special (non-persistent) folders
                                  # for the principle storage
                                  # locations
    FOLDER_LOOKUP['special'].append(PieFolder(ROOT_MAP['projectdir']))
    FOLDER_LOOKUP['special'].append(PieFolder(ROOT_MAP['librarydir']))
    FOLDER_LOOKUP['special'].append(PieFolder(ROOT_MAP['meetingpaperdir']))
Beispiel #5
0
 def set_path_precut(self, root, subfolders):
     '''Set the path, when we already have chopped up the path -
     more efficient'''
     assert root in ROOT_MAP.keys()
     assert type(subfolders) == list
     self.Root = root
     self.SubFolders = [i for i in subfolders if i]
     if self.SubFolders: 
         self.EndName = self.SubFolders[-1]
     else:
         self.EndName == [ a for a in ROOT_MAP[self.Root].split(os.sep) if len(a) > 0 ][-1]
     self.initialised = 1
Beispiel #6
0
 def set_path_precut(self, root, subfolders):
     '''Set the path, when we already have chopped up the path -
     more efficient'''
     assert root in ROOT_MAP.keys()
     assert type(subfolders) == list
     self.Root = root
     self.SubFolders = [i for i in subfolders if i]
     if self.SubFolders:
         self.EndName = self.SubFolders[-1]
     else:
         self.EndName == [
             a for a in ROOT_MAP[self.Root].split(os.sep) if len(a) > 0
         ][-1]
     self.initialised = 1
Beispiel #7
0
 def set_path(self, path):
     # if not os.path.isdir(path):
     #     raise ValueError, "Not a valid folder"
     # ^^ actually, non-existent directories should be ok
     for key, pdir in ROOT_MAP.items():
         if path[:len(pdir)] == pdir:
             fdroot = key
             self.Root = key
             self.SubFolders=[i for i in path[len(pdir):].split(os.sep) if i]
     if self.SubFolders: 
         self.EndName = self.SubFolders[-1]
     else:
         self.EndName = [ a for a in ROOT_MAP[self.Root].split(os.sep) if len(a) > 0 ][-1]
     if not fdroot: raise Exception, 'Folder outside pieberry domain'
     self.initialised = 1
     print 'initialised:', self
Beispiel #8
0
 def set_path(self, path):
     # if not os.path.isdir(path):
     #     raise ValueError, "Not a valid folder"
     # ^^ actually, non-existent directories should be ok
     for key, pdir in ROOT_MAP.items():
         if path[:len(pdir)] == pdir:
             fdroot = key
             self.Root = key
             self.SubFolders = [
                 i for i in path[len(pdir):].split(os.sep) if i
             ]
     if self.SubFolders:
         self.EndName = self.SubFolders[-1]
     else:
         self.EndName = [
             a for a in ROOT_MAP[self.Root].split(os.sep) if len(a) > 0
         ][-1]
     if not fdroot: raise Exception, 'Folder outside pieberry domain'
     self.initialised = 1
     print 'initialised:', self
Beispiel #9
0
 def set_file(self, loc, sqsess=session):
     '''Set all kinds of data associated with this being a local file.
     Should have full absolute path given to it.'''
     if not os.path.exists(loc):
         raise IOError, 'Trying to set file data for non existant file'
     self.FileData_FileName = os.path.basename(loc)
     self.FileData_Size = os.stat(loc).st_size
     if not self.FileData_DateModified:
         # don't override mod_date that might come from metadata
         self.FileData_DateModified = datetime.datetime.fromtimestamp(
             os.stat(loc)[8])
     diry = os.path.dirname(loc)
     fdroot = None
     for key, pdir in ROOT_MAP.items():
         # print 'Testing if', diry[:len(pdir)], '==', pdir
         if diry[:len(pdir)] == pdir:
             fdroot = key
             self.FileData_Root = key
             self.FileData_Folder = diry[len(pdir):].split(os.sep)
             self.est_folder_rel(sqsess=sqsess)
             break
     if not fdroot: raise Exception, 'File stored outside pieberry domain'
     print "DONE", self.FileData_Root, self.FileData_FileName
Beispiel #10
0
 def set_file(self, loc, sqsess=session):
     '''Set all kinds of data associated with this being a local file.
     Should have full absolute path given to it.'''
     if not os.path.exists(loc):
         raise IOError, 'Trying to set file data for non existant file'
     self.FileData_FileName = os.path.basename(loc)
     self.FileData_Size = os.stat(loc).st_size
     if not self.FileData_DateModified:
         # don't override mod_date that might come from metadata
         self.FileData_DateModified = datetime.datetime.fromtimestamp(
             os.stat(loc)[8])
     diry = os.path.dirname(loc)
     fdroot = None
     for key, pdir in ROOT_MAP.items():
         # print 'Testing if', diry[:len(pdir)], '==', pdir
         if diry[:len(pdir)] == pdir:
             fdroot = key
             self.FileData_Root = key
             self.FileData_Folder = diry[len(pdir):].split(os.sep)
             self.est_folder_rel(sqsess=sqsess)
             break
     if not fdroot: raise Exception, 'File stored outside pieberry domain'
     print "DONE", self.FileData_Root, self.FileData_FileName
Beispiel #11
0
    "Hubert Courser")

urllist = (
    "http://www.asdf.com/=?ueeeue",
    "https://conan.org/phoebe.html",
    "www.jones.net",
    "http://pu.er/2n34t",
    "http://www3.jesus.loves.uu/worship.php"
    )

filenamelist_pdf = ('basin.pdf', 'sink.pdf', 'bath.pdf', 'shower.pdf', 'loo.pdf')
filenamelist = ('basin', 'sink', 'bath', 'shower', 'loo')
digits = ('0', '1', '2', '3', '4', '5', '6', '7', '8', '9')


rootlist = ROOT_MAP.keys()

def spoof_pieobject(objtype="normal"):
    '''Generate a spoof PieObject'''
    if objtype == 'normal':
        t = random.choice(ipsum)
        a = random.choice(namelist)
        d = datetime.datetime.today()
        ro = PieObject(t, a, d)
        ro.FileData_FileName = random.choice(filenamelist)
        ro.FileData_Root = random.choice(rootlist)
    elif objtype == 'web':
        ro = PieObject()
        ro.WebData_Url = random.choice(urllist)
        ro.WebData_PageUrl = ro.WebData_Url
        ro.WebData_LinkText = random.choice(ipsum)
Beispiel #12
0
from sqlalchemy import Column, Integer, String, DateTime, Unicode, PickleType
from sqlalchemy import and_
from ConfigParser import SafeConfigParser
from pieberry.piedb import SQLABase, session
from pieberry.pieconfig.paths import ROOT_MAP
from pieberry.pieconfig.globalvars import DEBUG
from pieberry.pieutility.decoding import *

global FOLDER_LOOKUP
FOLDER_LOOKUP = {}
SECURITY_CLASSES = (_("UNCLASSIFIED"), _("IN-CONFIDENCE"), _("RESTRICTED"),
                    _("PROTECTED"), _("CONFIDENTIAL"), _("HIGHLY-PROTECTED"),
                    _("SECRET"), _("TOP-SECRET"))
INFO_FNAME = '_Folder_Info.txt'

for key, val in ROOT_MAP.items():
    FOLDER_LOOKUP[key] = []


def recommend_folder(obj):
    fs = session.query(PieFolder).filter(PieFolder.Root == 'projectdir')
    f_cands = []
    for f in fs:
        score = 0
        for aterm in f.MatchTerms_Author:
            if aterm.lower() in obj.Author().lower():
                score += 1
        for tterm in f.MatchTerms_Title:
            if tterm.lower() in obj.Title().lower():
                score += 1
            if tterm.lower() in obj.FileData_FileName.lower():
Beispiel #13
0
            "Millie Metzler", "Maira Railsback", "Ozie Hilger",
            "Nathaniel Gault", "Ivonne Galgano", "Taryn Delosreyes",
            "Yan Jerkins", "Elmer Boydston", "Agripina Botts", "Reuben Maddux",
            "Reatha Stansell", "Deloise Hillyard", "Celia Crocket",
            "Maurita Swick", "Phung Maddix", "Yesenia Lieb", "Hubert Courser")

urllist = ("http://www.asdf.com/=?ueeeue", "https://conan.org/phoebe.html",
           "www.jones.net", "http://pu.er/2n34t",
           "http://www3.jesus.loves.uu/worship.php")

filenamelist_pdf = ('basin.pdf', 'sink.pdf', 'bath.pdf', 'shower.pdf',
                    'loo.pdf')
filenamelist = ('basin', 'sink', 'bath', 'shower', 'loo')
digits = ('0', '1', '2', '3', '4', '5', '6', '7', '8', '9')

rootlist = ROOT_MAP.keys()


def spoof_pieobject(objtype="normal"):
    '''Generate a spoof PieObject'''
    if objtype == 'normal':
        t = random.choice(ipsum)
        a = random.choice(namelist)
        d = datetime.datetime.today()
        ro = PieObject(t, a, d)
        ro.FileData_FileName = random.choice(filenamelist)
        ro.FileData_Root = random.choice(rootlist)
    elif objtype == 'web':
        ro = PieObject()
        ro.WebData_Url = random.choice(urllist)
        ro.WebData_PageUrl = ro.WebData_Url
Beispiel #14
0
from pieberry.pieutility.decoding import *

global FOLDER_LOOKUP
FOLDER_LOOKUP = {}
SECURITY_CLASSES = (
    _("UNCLASSIFIED"),
    _("IN-CONFIDENCE"),
    _("RESTRICTED"),
    _("PROTECTED"),
    _("CONFIDENTIAL"),
    _("HIGHLY-PROTECTED"),
    _("SECRET"),
    _("TOP-SECRET"))
INFO_FNAME = '_Folder_Info.txt'

for key, val in ROOT_MAP.items():
    FOLDER_LOOKUP[key] = []

def recommend_folder(obj):
    fs = session.query(PieFolder).filter(PieFolder.Root == 'projectdir')
    f_cands = []
    for f in fs:
        score = 0
        for aterm in f.MatchTerms_Author:
            if aterm.lower() in obj.Author().lower():
                score += 1
        for tterm in f.MatchTerms_Title:
            if tterm.lower() in obj.Title().lower():
                score += 1
            if tterm.lower() in obj.FileData_FileName.lower():
                score += 1