def __init__(self):
# Find the slapd binary
for slapd in SLAPD_PATHS:
assert (slapd != None)
if (os.path.isfile(slapd)):
break
self._clean()
# Fix up paths in the slapd configuration file
output = open(SLAPD_CONFIG, 'w')
input = open(SLAPD_CONFIG_IN, 'r')
for line in input:
line = line.replace('@LDAP_DIR@', LDAP_DIR)
output.write(line)
output.close()
input.close()
# Load the directory with our test data
posix.mkdir(SLAPD_DATA)
shutil.copy2(SLAPD_DB_CONFIG, SLAPD_DATA)
pid = os.spawnv(
os.P_NOWAIT, slapd,
[slapd, '-T', 'a', '-f', SLAPD_CONFIG, '-l', SLAPD_LDIF])
# Wait for completion
while (1):
try:
os.waitpid(pid, 0)
except OSError, e:
import errno
if e.errno == errno.EINTR:
continue
raise
break
def __init__(self):
# Find the slapd binary
for slapd in SLAPD_PATHS:
assert(slapd != None)
if (os.path.isfile(slapd)):
break
self._clean()
# Fix up paths in the slapd configuration file
output = open(SLAPD_CONFIG, 'w')
input = open(SLAPD_CONFIG_IN, 'r')
for line in input:
line = line.replace('@LDAP_DIR@', LDAP_DIR)
output.write(line)
output.close()
input.close()
# Load the directory with our test data
posix.mkdir(SLAPD_DATA)
shutil.copy2(SLAPD_DB_CONFIG, SLAPD_DATA)
pid = os.spawnv(os.P_NOWAIT, slapd, [slapd, '-T', 'a', '-f', SLAPD_CONFIG, '-l', SLAPD_LDIF])
# Wait for completion
while(1):
try:
os.waitpid(pid, 0)
except OSError, e:
import errno
if e.errno == errno.EINTR:
continue
raise
break
def split_passages(directory,
train=TRAIN_DEFAULT,
dev=DEV_DEFAULT,
link=False):
filenames = sorted(os.listdir(directory))
assert filenames, "No files to split"
directory = os.path.abspath(directory)
if not directory.endswith(os.sep):
directory += os.sep
for subdirectory in "train", "dev", "test":
if not os.path.exists(directory + subdirectory):
mkdir(directory + subdirectory)
print("%d files to split" % len(filenames))
print_format = "Creating link in %s to: " if link else "Copying to %s: "
print(print_format % "train", end="", flush=True)
for f in filenames[:train]:
copy(directory + f, directory + "train" + os.sep + f, link)
print(f, end=" ", flush=True)
print()
print(print_format % "dev", end="", flush=True)
for f in filenames[train:train + dev]:
copy(directory + f, directory + "dev" + os.sep + f, link)
print(f, end=" ", flush=True)
print()
print(print_format % "test", end="", flush=True)
for f in filenames[train + dev:]:
copy(directory + f, directory + "test" + os.sep + f, link)
print(f, end=" ", flush=True)
print()
def write_to_file(file_path, strMessage):
dirPath = file_path[:file_path.rfind('/')]
if not exists(dirPath):
mkdir(dirPath)
f = open(file_path, 'w+')
f.write(strMessage)
f.close()
def split_passages(directory, train, dev, link, quiet=False):
filenames = sorted(filter(not_split_dir, os.listdir(directory)), key=numeric)
assert filenames, "No files to split"
assert train + dev <= len(filenames), "Not enough files to split: %d+%d>%d" % (train, dev, len(filenames))
directory = os.path.abspath(directory)
if not directory.endswith(os.sep):
directory += os.sep
for subdirectory in "train", "dev", "test":
if not os.path.exists(directory + subdirectory):
mkdir(directory + subdirectory)
print("%d files to split: %d/%d/%d" % (len(filenames), train, dev, len(filenames) - train - dev))
print_format = "Creating link in %s to: " if link else "Copying to %s: "
if not quiet:
print(print_format % "train", end="", flush=True)
for f in filenames[:train]:
copy(directory + f, directory + "train" + os.sep + f, link)
if not quiet:
print(f, end=" ", flush=True)
if not quiet:
print()
print(print_format % "dev", end="", flush=True)
for f in filenames[train:train + dev]:
copy(directory + f, directory + "dev" + os.sep + f, link)
if not quiet:
print(f, end=" ", flush=True)
if not quiet:
print()
print(print_format % "test", end="", flush=True)
for f in filenames[train + dev:]:
copy(directory + f, directory + "test" + os.sep + f, link)
if not quiet:
print(f, end=" ", flush=True)
if not quiet:
print()
def decrypt_directory(indir, outdir, re_encrypt=False):
'''Decrypt the OmniFocus data in indir, writing the result to outdir. Prompts for a passphrase.'''
files = posix.listdir(indir)
indir = os.path.abspath(indir)
if outdir:
outdir = os.path.abspath(outdir)
if metadata_filename not in files:
raise EnvironmentError('Expected to find %r in %r' %
(metadata_filename, indir))
encryptionMetadata = DocumentKey.parse_metadata(
open(os.path.join(indir, metadata_filename), 'rb'))
metadataKey = DocumentKey.use_passphrase(
encryptionMetadata, getpass.getpass(prompt="Passphrase: "))
docKey = DocumentKey(encryptionMetadata.get('key').data,
unwrapping_key=metadataKey)
for secret in docKey.secrets:
secret.print()
workdir = outdir
if outdir is not None:
posix.mkdir(outdir)
if re_encrypt:
workdir = tempfile.mkdtemp()
basename = os.path.basename(indir)
for dirpath, dirnames, filenames in os.walk(indir):
for datafile in filenames:
inpath = os.path.join(dirpath, datafile)
if workdir is not None:
outpath = inpath.replace(indir, workdir)
if datafile == metadata_filename:
continue
display = "%s/%s" % (os.path.basename(dirpath),
datafile) if os.path.basename(
dirpath) != basename else datafile
if outdir is not None:
print('Decrypting %r' % (display, ))
if not os.path.exists(os.path.split(outpath)[0]):
os.makedirs(os.path.split(outpath)[0])
with open(os.path.join(indir, inpath), "rb") as infp, \
open(os.path.join(workdir, outpath), "wb") as outfp:
docKey.decrypt_file(datafile, infp, outfp)
else:
print('Reading %r' % (display, ))
with open(os.path.join(indir, inpath), "rb") as infp:
docKey.decrypt_file(datafile, infp, None)
if re_encrypt and outdir is not None:
print()
print("Re-Encrypt the database\n")
newKey = docKey.get_key_of_type(ActiveAES_CTR_HMAC, True)
encryptionMetadata['key'] = plistlib.Data(
docKey.wrapped_secrets(metadataKey))
encrypt_directory(encryptionMetadata, docKey, workdir, outdir)
# We've created a tempdirectory lets clean it up
import shutil
shutil.rmtree(workdir)
def write_to_file(file_path, strMessage):
dirPath = file_path[:file_path.rfind('/')]
if not exists(dirPath):
mkdir(dirPath)
f = open(file_path, 'w+')
f.write(strMessage)
f.close()
def encrypt_directory(metadata, docKey, indir, outdir):
'''Encrypt all individual files in indir, writing them to outdir.'''
files = posix.listdir(indir)
assert metadata_filename not in files # A non-encrypted directory must not have an encryption metadata file
if not os.path.exists(outdir):
posix.mkdir(outdir)
print('Writing encryption metadata')
if isinstance(metadata, dict):
metadata = [ metadata ] # Current OmniFileStore expects a 1-element list of dictionaries.
with open(os.path.join(outdir, metadata_filename), "wb") as outfp:
if hasattr(plistlib, 'dump'):
plistlib.dump(metadata, outfp)
else:
plistlib.writePlist(metadata, outfp)
basename = os.path.basename(indir)
for dirpath, dirnames, filenames in os.walk(indir):
for datafile in filenames:
inpath = os.path.join(dirpath, datafile)
outpath = inpath.replace(indir, outdir)
display = "%s/%s" % (os.path.basename(dirpath), datafile) if os.path.basename(dirpath) != basename else datafile
print('Encrypting %r' % (display,))
if not os.path.exists(os.path.split(outpath)[0]):
os.makedirs(os.path.split(outpath)[0])
with open(os.path.join(indir, inpath), "rb") as infp, \
open(outpath, "wb") as outfp:
docKey.encrypt_file(datafile, infp, outfp)
def split_passages(directory, train=TRAIN_DEFAULT, dev=DEV_DEFAULT, link=False):
filenames = sorted(os.listdir(directory))
assert filenames, "No files to split"
directory = os.path.abspath(directory)
if not directory.endswith(os.sep):
directory += os.sep
for subdirectory in "train", "dev", "test":
if not os.path.exists(directory + subdirectory):
mkdir(directory + subdirectory)
print("%d files to split" % len(filenames))
print_format = "Creating link in %s to: " if link else "Copying to %s: "
print(print_format % "train", end="", flush=True)
for f in filenames[:train]:
copy(directory + f, directory + "train" + os.sep + f, link)
print(f, end=" ", flush=True)
print()
print(print_format % "dev", end="", flush=True)
for f in filenames[train:train + dev]:
copy(directory + f, directory + "dev" + os.sep + f, link)
print(f, end=" ", flush=True)
print()
print(print_format % "test", end="", flush=True)
for f in filenames[train + dev:]:
copy(directory + f, directory + "test" + os.sep + f, link)
print(f, end=" ", flush=True)
print()
def checkDirPath(dirPath):
if not os.path.exists(dirPath):
posix.mkdir(dirPath)
if not dirPath.endswith("/"):
dirPath = dirPath + "/"
return dirPath
def checkDirPath(dirPath):
if not os.path.exists(dirPath):
posix.mkdir(dirPath)
if not dirPath.endswith("/"):
dirPath = dirPath + "/"
return dirPath
def mountToMountpoint(devName, mntPt, readOnly, fstype="reiserfs"):
"""
Method mounts a device <devName> to a mount point <mntPt>. The optional
parameter fstype can be used to specify a filesystem type different than
reiserfs. To mount a device which is in the /etc/fstab use the
following command:
mountToMountpoint("", <mntPt>, fstype="")
Synopsis: mountToMountpoint(devName, mntPt[, fstype = <fstype>])
devName: Device name (string).
mntPt: Mount point (string).
readOnly: 0 if not read-only, 1 if read-only (integer).
fstype: File system type (string).
Returns: Void.
"""
#if (fstype): fstype = '-t ' + fstype
fstype = ""
if (readOnly):
rdOnly = "-r "
else:
rdOnly = ""
#command = 'mount ' + rdOnly + ' ' + fstype + ' ' + devName + ' ' + mntPt
command = "sudo mount %s %s %s" % (rdOnly, devName, mntPt)
logger.debug("Command to mount disk: %s", command)
# if (getMountedDev(mntPt)): # already mounted
# warnMsg = "Device " + getMountedDev(mntPt) + ' already mounted ' + \
# 'to specified mountpoint: ' + mntPt
# info(1,warnMsg)
# else:
# if not os.path.exists(mntPt):
# try:
# posix.mkdir(mntPt)
# except exceptions.OSError,e:
# errMsg = "Failed creating mountpoint " + mntPt + ":" + str(e)
# error(errMsg)
# raise Exception, errMsg
# stat, out = commands.getstatusoutput(command)
# if (stat != 0):
# errMsg = "Failed mounting device " + getMountedDev(mntPt) + \
# ':' + out
# raise Exception, errMsg
if not os.path.exists(mntPt):
try:
posix.mkdir(mntPt)
except exceptions.OSError, e:
errMsg = "Failed creating mountpoint " + mntPt + ":" + str(e)
logger.exception(errMsg)
raise
def split_passages():
chdir('passages')
passages = glob('*.xml')
for split in 'train', 'dev', 'test':
if not path.exists(split):
mkdir(split)
for f in passages[:TRAIN_SIZE]:
rename(f, 'train/'+f)
for f in passages[TRAIN_SIZE:TRAIN_SIZE+DEV_SIZE]:
rename(f, 'dev/'+f)
for f in passages[TRAIN_SIZE+DEV_SIZE:]:
rename(f, 'test/'+f)
def mkrealdir(name):
st = posix.stat(name) # Get the mode
mode = S_IMODE(st[ST_MODE])
linkto = posix.readlink(name)
files = posix.listdir(name)
posix.unlink(name)
posix.mkdir(name, mode)
posix.chmod(name, mode)
linkto = cat('..', linkto)
#
for file in files:
if file not in ('.', '..'):
posix.symlink(cat(linkto, file), cat(name, file))
def mkrealdir(name):
st = posix.stat(name) # Get the mode
mode = S_IMODE(st[ST_MODE])
linkto = posix.readlink(name)
files = posix.listdir(name)
posix.unlink(name)
posix.mkdir(name, mode)
posix.chmod(name, mode)
linkto = cat('..', linkto)
#
for file in files:
if file not in ('.', '..'):
posix.symlink(cat(linkto, file), cat(name, file))
def reverseFeat(featFile, outFile):
FrameList = []
Header = ''
"""
Only 13 are stored, For IO efficiency we store only static features and dynamic features are computed on the fly.
The computation of dynamic features is configured with the -feat option. For example -feat 1s_c_d_dd means to read the vector and compute deltas and delta-deltas and combine them with 1-stream feature vector.
There are different types like s2_4x which means to compute deltas, delta-deltas, delta-deltas of the second order and combine them in a special 4-stream feature vector.
If you need a specific feature arrangement you can implement your own feature type in sphinxbase.
If you want to use features as is use 1s_c feature type which means to read the vector unmodified.
SEE: https://cmusphinx.github.io/wiki/mfcformat/
header (int32 length)
features of the frame 1 (13 floats or 13 * 4 bytes)
features of the frame 2 (13 floats or 13 * 4 bytes)
features of the frame ... (13 floats or 13 * 4 bytes)
features of the frame N (13 floats or 13 * 4 bytes)
"""
FeatDimension = 13 * 4
with open(featFile, 'rb') as fp:
header = fp.read(
4
) # 2 bytes for the header (int32), which is 13xN where N is no of frames for audio
Header = header
frame = fp.read(FeatDimension)
while frame != "":
# Do stuff with byte.
#print frame
FrameList.append(frame)
frame = fp.read(FeatDimension)
FrameList.reverse()
FrameList.insert(0, Header)
'''
Write reversed frames into ouFile
'''
testDir = outFile.rfind("/")
newDir = outFile[0:testDir]
#print newDir
if (isdir(newDir) is not True):
mkdir(newDir)
with open(outFile, 'wb') as fo:
for i in range(0, len(FrameList)):
#print i
fo.write(FrameList[i])
return ()
def __init__(self, pid, T=5, verbose=False):
self.pid = pid
self.T = T
self.verbose = verbose
self.info = tp.topParser()
try:
self.info.parseProcs(pid=self.pid)
except:
print "Process", pid, "not found."
exit(-1)
self.name = self.info.getInfo(self.pid, "COMMAND")
try: posix.mkdir("procs")
except: pass
self._logName = "procs/"+self.name+str(self.pid)+".txt"
self.log=open(self._logName,'w')
self.log.close()
self.start()
def decrypt_directory(indir, outdir):
'''Decrypt the OmniFocus data in indir, writing the result to outdir. Prompts for a passphrase.'''
files = posix.listdir(indir)
if metadata_filename not in files:
raise EnvironmentError('Expected to find %r in %r' %
( metadata_filename, indir))
docKey = DocumentKey( open(os.path.join(indir, metadata_filename), 'rb').read() )
docKey.use_password(bytes(getpass.getpass(prompt="Passphrase: "), encoding="latin1"))
if outdir is not None:
posix.mkdir(outdir)
for datafile in files:
if datafile == metadata_filename:
continue
if outdir is not None:
print('Decrypting %r' % (datafile,))
with open(os.path.join(indir, datafile), "rb") as infp, \
open(os.path.join(outdir, datafile), "wb") as outfp:
docKey.decrypt_file(datafile, infp, outfp)
else:
print('Reading %r' % (datafile,))
with open(os.path.join(indir, datafile), "rb") as infp:
docKey.decrypt_file(datafile, infp, None)
def copytree(src, dst):
names = posix.listdir(src)
posix.mkdir(dst, 0777)
dot_dotdot = '.', '..'
for name in names:
if name not in dot_dotdot:
srcname = path.cat(src, name)
dstname = path.cat(dst, name)
#print 'Copying', srcname, 'to', dstname
try:
#if path.islink(srcname):
# linkto = posix.readlink(srcname)
# posix.symlink(linkto, dstname)
#elif path.isdir(srcname):
if path.isdir(srcname):
copytree(srcname, dstname)
else:
copy2(srcname, dstname)
# XXX What about devices, sockets etc.?
except posix.error, why:
print 'Could not copy', srcname, 'to', dstname,
print '(', why[1], ')'
def split_passages(directory, train, dev, link, quiet=False):
filenames = sorted(filter(not_split_dir, os.listdir(directory)),
key=numeric)
assert filenames, "No files to split"
assert train + dev <= len(
filenames), "Not enough files to split: %d+%d>%d" % (train, dev,
len(filenames))
directory = os.path.abspath(directory)
if not directory.endswith(os.sep):
directory += os.sep
for subdirectory in "train", "dev", "test":
if not os.path.exists(directory + subdirectory):
mkdir(directory + subdirectory)
print("%d files to split: %d/%d/%d" %
(len(filenames), train, dev, len(filenames) - train - dev))
print_format = "Creating link in %s to: " if link else "Copying to %s: "
if not quiet:
print(print_format % "train", end="", flush=True)
for f in filenames[:train]:
copy(directory + f, directory + "train" + os.sep + f, link)
if not quiet:
print(f, end=" ", flush=True)
if not quiet:
print()
print(print_format % "dev", end="", flush=True)
for f in filenames[train:train + dev]:
copy(directory + f, directory + "dev" + os.sep + f, link)
if not quiet:
print(f, end=" ", flush=True)
if not quiet:
print()
print(print_format % "test", end="", flush=True)
for f in filenames[train + dev:]:
copy(directory + f, directory + "test" + os.sep + f, link)
if not quiet:
print(f, end=" ", flush=True)
if not quiet:
print()
def __init__(self, pid, T=5, verbose=False):
self.pid = pid
self.T = T
self.verbose = verbose
self.info = tp.topParser()
try:
self.info.parseProcs(pid=self.pid)
except:
print "Process", pid, "not found."
exit(-1)
self.name = self.info.getInfo(self.pid, "COMMAND")
try:
posix.mkdir("procs")
except:
pass
self._logName = "procs/" + self.name + str(self.pid) + ".txt"
self.log = open(self._logName, 'w')
self.log.close()
self.start()
def decrypt_directory(indir, outdir):
'''Decrypt the OmniFocus data in indir, writing the result to outdir. Prompts for a passphrase.'''
files = posix.listdir(indir)
if metadata_filename not in files:
raise EnvironmentError('Expected to find %r in %r' %
(metadata_filename, indir))
docKey = DocumentKey(
open(os.path.join(indir, metadata_filename), 'rb').read())
docKey.use_password(
bytes(getpass.getpass(prompt="Passphrase: "), encoding="latin1"))
if outdir is not None:
posix.mkdir(outdir)
for datafile in files:
if datafile == metadata_filename:
continue
if outdir is not None:
print('Decrypting %r' % (datafile, ))
with open(os.path.join(indir, datafile), "rb") as infp, \
open(os.path.join(outdir, datafile), "wb") as outfp:
docKey.decrypt_file(datafile, infp, outfp)
else:
print('Reading %r' % (datafile, ))
with open(os.path.join(indir, datafile), "rb") as infp:
docKey.decrypt_file(datafile, infp, None)
def run():
image_data = dict()
#for root, dirs, files in os.walk('maap/static/content'):
# for place in dirs:
# print place
dirs = os.listdir(CONTENT_REL_PATH)
for dir in dirs:
if not dir in RESIZE_ME:
continue
if not os.path.isdir(join(CONTENT_REL_PATH, dir)):
continue
if dir == '.svn':
continue
image_dir = join(CONTENT_REL_PATH, dir, 'images')
images = os.listdir(image_dir)
for image in images:
# ignore everything except for files
file_with_path = join(image_dir, image)
if not os.path.isfile(file_with_path):
continue
# ignore thumbnails
if image.find('thumb') >= 0:
continue
#just jpegs
if not image.find('.jp') >= 0:
continue
image_base, ext = os.path.splitext(image)
#call(["ls","-lh",file_with_path])
thumb_dir = join(image_dir, 'thumbs')
thennow_dir = join(image_dir, '274')
big_dir = join(image_dir, '820')
if not os.path.isdir(thumb_dir):
mkdir(thumb_dir)
if not os.path.isdir(thennow_dir):
mkdir(thennow_dir)
if not os.path.isdir(big_dir):
mkdir(big_dir)
print image_base
call(["convert","-resize","820>",file_with_path, join(big_dir,image_base) + '_820.jpg'])
call(["convert","-resize","274",file_with_path, join(thennow_dir,image_base) + '_274.jpg'])
call(["convert","-resize","100x100",file_with_path, join(thumb_dir,image_base) + '_100x100.jpg'])
while frame != "":
# Do stuff with byte.
#print frame
FrameList.append(frame)
frame = fp.read(FeatDimension)
FrameList.reverse()
FrameList.insert(0, Header)
L = len(FrameList)
print("No of frames: " + str(L))
'''
Write reversed frames into ouFile
'''
testDir = outFile.rfind("/")
newDir = outFile[0:testDir]
print newDir
if (isdir(newDir) is not True):
mkdir(newDir)
with open(outFile, 'wb') as fo:
for i in range(0, len(FrameList)):
print i
fo.write(FrameList[i])
"""
Test list.insert.
@param list.inser(index,value): Check if list.inser(index,value) inserts value before index
"""
TestList = ["feat1", "feat2", "feat3"]
TestList.reverse()
TestList.insert(0, "Header")
print TestList
