def add_new_folder(rootname, foldername):
'''add a new folder to a root directory'''
assert type(foldername) in (str, unicode)
if not rootname in ROOT_MAP.keys():
raise Exception, 'Root directory not found'
foldername = translate_non_alphanumerics(foldername)
existingfolders = session.query(PieFolder).filter(
and_(PieFolder.EndName == foldername,
PieFolder.Root == rootname)).all()
if len(existingfolders) > 0:
raise Exception, 'This folder already exists in database'
folderpath = os.path.join(ROOT_MAP[rootname], foldername)
if os.path.exists(folderpath) and not os.path.isdir(folderpath):
raise Exception, 'Conflict: File with this name already exists on disk'
newfolder = PieFolder(os.path.join(ROOT_MAP[rootname], foldername))
if os.path.exists(folderpath) and os.path.isdir(folderpath):
hfile = os.path.join(folderpath, _(INFO_FNAME))
if os.path.isfile(hfile):
hh = HeaderHandler(headerfile=hfile)
newfolder.SecurityLevel = hh.securitylevel
newfolder.RecordFile = hh.recordfile
else:
os.mkdir(newfolder.path())
session.add(newfolder)
session.commit()
FOLDER_LOOKUP[rootname].append(newfolder)
return newfolder
def add_new_folder(rootname, foldername):
'''add a new folder to a root directory'''
assert type(foldername) in (str, unicode)
if not rootname in ROOT_MAP.keys():
raise Exception, 'Root directory not found'
foldername = translate_non_alphanumerics(foldername)
existingfolders = session.query(PieFolder).filter(and_(
PieFolder.EndName == foldername,
PieFolder.Root == rootname)).all()
if len(existingfolders) > 0:
raise Exception, 'This folder already exists in database'
folderpath = os.path.join(ROOT_MAP[rootname], foldername)
if os.path.exists(folderpath) and not os.path.isdir(folderpath):
raise Exception, 'Conflict: File with this name already exists on disk'
newfolder = PieFolder(os.path.join(ROOT_MAP[rootname], foldername))
if os.path.exists(folderpath) and os.path.isdir(folderpath):
hfile = os.path.join(folderpath, _(INFO_FNAME))
if os.path.isfile(hfile):
hh = HeaderHandler(headerfile=hfile)
newfolder.SecurityLevel = hh.securitylevel
newfolder.RecordFile = hh.recordfile
else:
os.mkdir(newfolder.path())
session.add(newfolder)
session.commit()
FOLDER_LOOKUP[rootname].append(newfolder)
return newfolder
def generate_folder_list():
'''Walk through the pieberry filesystem and ensure that all
folders are indexed'''
def gen_subfolders(root_key, curr_dir, sub_dir):
'''cut up the path'''
ds = curr_dir[len(ROOT_MAP[root_key]):].split(os.sep)
ds.append(sub_dir)
return [i for i in ds if i]
def contribute_projectfolder(piefolder):
'''init new primary project folder'''
hh = HeaderHandler(piefolder=piefolder)
hh.write_header()
FOLDER_LOOKUP['projectdir'].append(piefolder)
def verify_existing():
for qf in session.query(PieFolder):
if not os.path.isdir(qf.path()):
# print 'nonexistant folder -', qf
session.delete(qf)
session.commit()
verify_existing()
FOLDER_LOOKUP['projectdir'] = []
for root_key in ROOT_MAP.keys():
if root_key in ('cachedir', 'backupdir', 'desktopdir'): continue
for curr_dir, subdirs, files in os.walk(ROOT_MAP[root_key]):
for subdir in subdirs:
cut_subdirs = gen_subfolders(root_key, curr_dir, subdir)
# is there an existing piefolder in the db for this?
exisf = session.query(PieFolder).filter(
and_(
PieFolder.Root == root_key,
PieFolder.SubFolders == cut_subdirs,
)).first()
if not exisf: # if the folder isn't already in the db
print 'creating:', os.path.join(curr_dir, subdir)
print 'vars:', cut_subdirs, root_key
# create a new piefolder object
n_piefolder = PieFolder()
n_piefolder.set_path_precut(root_key, cut_subdirs)
session.add(n_piefolder)
if curr_dir == ROOT_MAP['projectdir']:
contribute_projectfolder(n_piefolder)
print n_piefolder
else:
if curr_dir == ROOT_MAP['projectdir']:
FOLDER_LOOKUP['projectdir'].append(exisf)
print 'found folder:', exisf
# Generate 'special' folders (i.e. the various root folders)
FOLDER_LOOKUP['special'] = [] # special (non-persistent) folders
# for the principle storage
# locations
FOLDER_LOOKUP['special'].append(PieFolder(ROOT_MAP['projectdir']))
FOLDER_LOOKUP['special'].append(PieFolder(ROOT_MAP['librarydir']))
FOLDER_LOOKUP['special'].append(PieFolder(ROOT_MAP['meetingpaperdir']))
def generate_folder_list():
'''Walk through the pieberry filesystem and ensure that all
folders are indexed'''
def gen_subfolders(root_key, curr_dir, sub_dir):
'''cut up the path'''
ds = curr_dir[len(ROOT_MAP[root_key]):].split(os.sep)
ds.append(sub_dir)
return [i for i in ds if i]
def contribute_projectfolder(piefolder):
'''init new primary project folder'''
hh = HeaderHandler(piefolder=piefolder)
hh.write_header()
FOLDER_LOOKUP['projectdir'].append(piefolder)
def verify_existing():
for qf in session.query(PieFolder):
if not os.path.isdir(qf.path()):
# print 'nonexistant folder -', qf
session.delete(qf)
session.commit()
verify_existing()
FOLDER_LOOKUP['projectdir'] = []
for root_key in ROOT_MAP.keys():
if root_key in ('cachedir', 'backupdir', 'desktopdir'): continue
for curr_dir, subdirs, files in os.walk(ROOT_MAP[root_key]):
for subdir in subdirs:
cut_subdirs = gen_subfolders(root_key, curr_dir, subdir)
# is there an existing piefolder in the db for this?
exisf = session.query(PieFolder).filter(and_(
PieFolder.Root == root_key,
PieFolder.SubFolders == cut_subdirs,
)).first()
if not exisf: # if the folder isn't already in the db
print 'creating:', os.path.join(curr_dir, subdir)
print 'vars:', cut_subdirs, root_key
# create a new piefolder object
n_piefolder = PieFolder()
n_piefolder.set_path_precut(root_key, cut_subdirs)
session.add(n_piefolder)
if curr_dir == ROOT_MAP['projectdir']:
contribute_projectfolder(n_piefolder)
print n_piefolder
else:
if curr_dir == ROOT_MAP['projectdir']:
FOLDER_LOOKUP['projectdir'].append(exisf)
print 'found folder:', exisf
# Generate 'special' folders (i.e. the various root folders)
FOLDER_LOOKUP['special'] = [] # special (non-persistent) folders
# for the principle storage
# locations
FOLDER_LOOKUP['special'].append(PieFolder(ROOT_MAP['projectdir']))
FOLDER_LOOKUP['special'].append(PieFolder(ROOT_MAP['librarydir']))
FOLDER_LOOKUP['special'].append(PieFolder(ROOT_MAP['meetingpaperdir']))
def set_path_precut(self, root, subfolders):
'''Set the path, when we already have chopped up the path -
more efficient'''
assert root in ROOT_MAP.keys()
assert type(subfolders) == list
self.Root = root
self.SubFolders = [i for i in subfolders if i]
if self.SubFolders:
self.EndName = self.SubFolders[-1]
else:
self.EndName == [ a for a in ROOT_MAP[self.Root].split(os.sep) if len(a) > 0 ][-1]
self.initialised = 1
def set_path_precut(self, root, subfolders):
'''Set the path, when we already have chopped up the path -
more efficient'''
assert root in ROOT_MAP.keys()
assert type(subfolders) == list
self.Root = root
self.SubFolders = [i for i in subfolders if i]
if self.SubFolders:
self.EndName = self.SubFolders[-1]
else:
self.EndName == [
a for a in ROOT_MAP[self.Root].split(os.sep) if len(a) > 0
][-1]
self.initialised = 1
def set_path(self, path):
# if not os.path.isdir(path):
# raise ValueError, "Not a valid folder"
# ^^ actually, non-existent directories should be ok
for key, pdir in ROOT_MAP.items():
if path[:len(pdir)] == pdir:
fdroot = key
self.Root = key
self.SubFolders=[i for i in path[len(pdir):].split(os.sep) if i]
if self.SubFolders:
self.EndName = self.SubFolders[-1]
else:
self.EndName = [ a for a in ROOT_MAP[self.Root].split(os.sep) if len(a) > 0 ][-1]
if not fdroot: raise Exception, 'Folder outside pieberry domain'
self.initialised = 1
print 'initialised:', self
def set_path(self, path):
# if not os.path.isdir(path):
# raise ValueError, "Not a valid folder"
# ^^ actually, non-existent directories should be ok
for key, pdir in ROOT_MAP.items():
if path[:len(pdir)] == pdir:
fdroot = key
self.Root = key
self.SubFolders = [
i for i in path[len(pdir):].split(os.sep) if i
]
if self.SubFolders:
self.EndName = self.SubFolders[-1]
else:
self.EndName = [
a for a in ROOT_MAP[self.Root].split(os.sep) if len(a) > 0
][-1]
if not fdroot: raise Exception, 'Folder outside pieberry domain'
self.initialised = 1
print 'initialised:', self
def set_file(self, loc, sqsess=session):
'''Set all kinds of data associated with this being a local file.
Should have full absolute path given to it.'''
if not os.path.exists(loc):
raise IOError, 'Trying to set file data for non existant file'
self.FileData_FileName = os.path.basename(loc)
self.FileData_Size = os.stat(loc).st_size
if not self.FileData_DateModified:
# don't override mod_date that might come from metadata
self.FileData_DateModified = datetime.datetime.fromtimestamp(
os.stat(loc)[8])
diry = os.path.dirname(loc)
fdroot = None
for key, pdir in ROOT_MAP.items():
# print 'Testing if', diry[:len(pdir)], '==', pdir
if diry[:len(pdir)] == pdir:
fdroot = key
self.FileData_Root = key
self.FileData_Folder = diry[len(pdir):].split(os.sep)
self.est_folder_rel(sqsess=sqsess)
break
if not fdroot: raise Exception, 'File stored outside pieberry domain'
print "DONE", self.FileData_Root, self.FileData_FileName
def set_file(self, loc, sqsess=session):
'''Set all kinds of data associated with this being a local file.
Should have full absolute path given to it.'''
if not os.path.exists(loc):
raise IOError, 'Trying to set file data for non existant file'
self.FileData_FileName = os.path.basename(loc)
self.FileData_Size = os.stat(loc).st_size
if not self.FileData_DateModified:
# don't override mod_date that might come from metadata
self.FileData_DateModified = datetime.datetime.fromtimestamp(
os.stat(loc)[8])
diry = os.path.dirname(loc)
fdroot = None
for key, pdir in ROOT_MAP.items():
# print 'Testing if', diry[:len(pdir)], '==', pdir
if diry[:len(pdir)] == pdir:
fdroot = key
self.FileData_Root = key
self.FileData_Folder = diry[len(pdir):].split(os.sep)
self.est_folder_rel(sqsess=sqsess)
break
if not fdroot: raise Exception, 'File stored outside pieberry domain'
print "DONE", self.FileData_Root, self.FileData_FileName
"Hubert Courser")
urllist = (
"http://www.asdf.com/=?ueeeue",
"https://conan.org/phoebe.html",
"www.jones.net",
"http://pu.er/2n34t",
"http://www3.jesus.loves.uu/worship.php"
)
filenamelist_pdf = ('basin.pdf', 'sink.pdf', 'bath.pdf', 'shower.pdf', 'loo.pdf')
filenamelist = ('basin', 'sink', 'bath', 'shower', 'loo')
digits = ('0', '1', '2', '3', '4', '5', '6', '7', '8', '9')
rootlist = ROOT_MAP.keys()
def spoof_pieobject(objtype="normal"):
'''Generate a spoof PieObject'''
if objtype == 'normal':
t = random.choice(ipsum)
a = random.choice(namelist)
d = datetime.datetime.today()
ro = PieObject(t, a, d)
ro.FileData_FileName = random.choice(filenamelist)
ro.FileData_Root = random.choice(rootlist)
elif objtype == 'web':
ro = PieObject()
ro.WebData_Url = random.choice(urllist)
ro.WebData_PageUrl = ro.WebData_Url
ro.WebData_LinkText = random.choice(ipsum)
from sqlalchemy import Column, Integer, String, DateTime, Unicode, PickleType
from sqlalchemy import and_
from ConfigParser import SafeConfigParser
from pieberry.piedb import SQLABase, session
from pieberry.pieconfig.paths import ROOT_MAP
from pieberry.pieconfig.globalvars import DEBUG
from pieberry.pieutility.decoding import *
global FOLDER_LOOKUP
FOLDER_LOOKUP = {}
SECURITY_CLASSES = (_("UNCLASSIFIED"), _("IN-CONFIDENCE"), _("RESTRICTED"),
_("PROTECTED"), _("CONFIDENTIAL"), _("HIGHLY-PROTECTED"),
_("SECRET"), _("TOP-SECRET"))
INFO_FNAME = '_Folder_Info.txt'
for key, val in ROOT_MAP.items():
FOLDER_LOOKUP[key] = []
def recommend_folder(obj):
fs = session.query(PieFolder).filter(PieFolder.Root == 'projectdir')
f_cands = []
for f in fs:
score = 0
for aterm in f.MatchTerms_Author:
if aterm.lower() in obj.Author().lower():
score += 1
for tterm in f.MatchTerms_Title:
if tterm.lower() in obj.Title().lower():
score += 1
if tterm.lower() in obj.FileData_FileName.lower():
"Millie Metzler", "Maira Railsback", "Ozie Hilger",
"Nathaniel Gault", "Ivonne Galgano", "Taryn Delosreyes",
"Yan Jerkins", "Elmer Boydston", "Agripina Botts", "Reuben Maddux",
"Reatha Stansell", "Deloise Hillyard", "Celia Crocket",
"Maurita Swick", "Phung Maddix", "Yesenia Lieb", "Hubert Courser")
urllist = ("http://www.asdf.com/=?ueeeue", "https://conan.org/phoebe.html",
"www.jones.net", "http://pu.er/2n34t",
"http://www3.jesus.loves.uu/worship.php")
filenamelist_pdf = ('basin.pdf', 'sink.pdf', 'bath.pdf', 'shower.pdf',
'loo.pdf')
filenamelist = ('basin', 'sink', 'bath', 'shower', 'loo')
digits = ('0', '1', '2', '3', '4', '5', '6', '7', '8', '9')
rootlist = ROOT_MAP.keys()
def spoof_pieobject(objtype="normal"):
'''Generate a spoof PieObject'''
if objtype == 'normal':
t = random.choice(ipsum)
a = random.choice(namelist)
d = datetime.datetime.today()
ro = PieObject(t, a, d)
ro.FileData_FileName = random.choice(filenamelist)
ro.FileData_Root = random.choice(rootlist)
elif objtype == 'web':
ro = PieObject()
ro.WebData_Url = random.choice(urllist)
ro.WebData_PageUrl = ro.WebData_Url
from pieberry.pieutility.decoding import *
global FOLDER_LOOKUP
FOLDER_LOOKUP = {}
SECURITY_CLASSES = (
_("UNCLASSIFIED"),
_("IN-CONFIDENCE"),
_("RESTRICTED"),
_("PROTECTED"),
_("CONFIDENTIAL"),
_("HIGHLY-PROTECTED"),
_("SECRET"),
_("TOP-SECRET"))
INFO_FNAME = '_Folder_Info.txt'
for key, val in ROOT_MAP.items():
FOLDER_LOOKUP[key] = []
def recommend_folder(obj):
fs = session.query(PieFolder).filter(PieFolder.Root == 'projectdir')
f_cands = []
for f in fs:
score = 0
for aterm in f.MatchTerms_Author:
if aterm.lower() in obj.Author().lower():
score += 1
for tterm in f.MatchTerms_Title:
if tterm.lower() in obj.Title().lower():
score += 1
if tterm.lower() in obj.FileData_FileName.lower():
score += 1