def checkPath(path):
#-------------------------------------------------------------------------------
oss.cd(path)
libs = []
covered = set()
for i in oss.paths(oss.ls('*.pth')):
libs.append(i.name)
for line in oss.readf(i):
if line.strip().startswith('#'):
continue
covered.add(line)
for i in oss.paths(oss.ls()):
if i in covered:
continue
if i.ext == '.py':
libs.append(i.name)
if oss.IsDir(i):
if oss.exists(i + '/__init__.py'):
libs.append(i.name)
return libs
def checkPath(path):
#-------------------------------------------------------------------------------
oss.cd(path)
libs = []
covered = set()
for i in oss.paths(oss.ls('*.pth')):
libs.append(i.name)
for line in oss.readf(i):
if line.strip().startswith('#'):
continue
covered.add(line)
for i in oss.paths(oss.ls()):
if i in covered:
continue
if i.ext == '.py':
libs.append(i.name)
if oss.IsDir(i):
if oss.exists(i + '/__init__.py'):
libs.append(i.name)
return libs
def checkDir(dir1, dir2, exts=None, quick=False, recurse=False, supress=False, indent=0):
#-------------------------------------------------------------------------------
d1 = set(oss.basename(oss.ls(dir1)))
d2 = set(oss.basename(oss.ls(dir2)))
same = True
p = util.PrintOnce(supress, indent)
for f in sorted(d1.difference(d2)):
if oss.IsDir(dir1 + '/' + f):
same = False
p.printOnce("Objects in '%s' not in '%s':" % (dir1, dir2))
p.print(' /{0}'.format(f))
elif checkFile(f, exts):
same = False
p.printOnce("Objects in '%s' not in '%s':" % (dir1, dir2))
p.print(' {0}'.format(f))
p.didPrint()
p.reset()
for f in sorted(d2.difference(d1)):
if oss.IsDir(dir2 + '/' + f):
same = False
p.printOnce("Objects in '%s' not in '%s':" % (dir2, dir1))
p.print(' /{0}'.format(f))
elif checkFile(f, exts):
same = False
p.printOnce("Objects in '%s' not in '%s':" % (dir2, dir1))
p.print(' {0}'.format(f))
p.didPrint()
dirs = []
p.reset()
for f in sorted(d1.intersection(d2)):
if oss.IsDir(dir1 + '/' + f):
if recurse:
dirs.append(f)
elif checkFile(f, exts):
if not compare(dir1 + '/' + f, dir2 + '/' + f, quick):
same = False
p.printOnce("Files with Differences{0}:".format(' (quick)' if quick else ''))
#p.print(' ', dir1 + '/' + f + " != " + dir2 + '/' + f)
p.print(' ', f)
p.didPrint()
if recurse:
p.reset()
for f in dirs:
p.printOnce("Directories in {0} and {1}:".format(dir1, dir2))
p.print(' /{0}'.format(f))
same = checkDir(dir1 + '/' + f, dir2 + '/' + f, exts, quick, recurse, supress, indent+1) and same
p.didPrint()
return same
def findModules(path):
#-------------------------------------------------------------------------------
oss.cd(path)
dirs = set()
pyFiles = set()
specific = set()
nonSpecific = set()
for f in oss.ls():
if f.endswith('.py'):
pyFiles.add(oss.path(f))
elif f.endswith('.pyd'):
specific.add(oss.path(f).name)
elif oss.isDir(f) and oss.exists(f + '/__init__.py'):
dirs.add(f)
elif f.endswith('.pth'):
dirs |= readPthFile(f)
else:
#print('what:', f)
pass
for f in pyFiles:
f = f.name
for ff in oss.ls():
if ff.endswith(f + '.pyd'):
specific.add(f)
break
else:
nonSpecific.add(f)
for d in dirs:
code = False
for f in oss.find(d):
if f.endswith('.pyd') or f.endswith('.dll'):
specific.add(clean(d))
break
if f.endswith('.py'):
code = True
else:
if code:
nonSpecific.add(clean(d))
return specific, nonSpecific
def get(pth):
#-------------------------------------------------------------------------------
t = []
for f in sorted(oss.ls(pth), key=lastMod):
f = f.replace('\\', '/')
t.append(' <li><a href="%s">%s</a></li>\n' % (f, oss.basename(f)))
return t
def sizeDir(pth, dirOnly, progress=False):
#-------------------------------------------------------------------------------
res = []
tsize = 0
if pth == '/' or pth == r'\\':
pth += '*'
for df in oss.ls(pth):
df = oss.canonicalPath(df)
if df in gExcludes:
print('excluding:', df, file=oss.stderr)
continue
if progress:
print('...', df.encode('utf8', 'ignore'))
gVisited.add(df)
if os.path.isdir(df):
size = calcSize(df, progress)
res.append(util.sstruct(size=size, pth=df + '\\'))
else:
try:
size = os.stat(df).st_size
except WindowsError as ex:
print('Error: "%s"' % str(ex), file=oss.stderr)
if not dirOnly:
res.append(util.sstruct(size=size, pth=df))
tsize += size
res.append(util.sstruct(size=tsize, pth=pth))
res.sort(key=lambda ss: int(ss.size), reverse=True)
return res
def sizeDir(pth, dirOnly, progress=False):
#-------------------------------------------------------------------------------
res = []
tsize = 0
if pth == '/' or pth == r'\\':
pth += '*'
for df in oss.ls(pth):
df = oss.canonicalPath(df)
if df in gExcludes:
print('excluding:', df, file=oss.stderr)
continue
if progress:
print('...', df.encode('utf8', 'ignore'))
gVisited.add(df)
if os.path.isdir(df):
size = calcSize(df, progress)
res.append(util.sstruct(size=size, pth=df+'\\'))
else:
try:
size = os.stat(df).st_size
except WindowsError as ex:
print('Error: "%s"' % str(ex), file=oss.stderr)
if not dirOnly:
res.append(util.sstruct(size=size, pth=df))
tsize += size
res.append(util.sstruct(size=tsize, pth=pth))
res.sort(key=lambda ss: int(ss.size), reverse=True)
return res
def LoadLibraries(path, func, cxt=None):
#-------------------------------------------------------------------------------
if isinstance(path, (str, unicode)):
path = [path]
plugins = []
for pth in path:
for f in oss.ls(pth + '/*.py', pth + '/*.pyd'):
plugins.extend(func(oss.splitnm(oss.basename(f)), path, cxt))
return plugins
def main(argv):
#-------------------------------------------------------------------------------
""" usage:
"""
args, opts = oss.gopt(argv[1:], [], [], main.__doc__ + __doc__)
for i in oss.ls(r'C:\windows\fonts\*.ttf'):
print(i.split('\\')[-1])
oss.exit(0)
def main(argv):
#-------------------------------------------------------------------------------
""" usage:
"""
args, opts = oss.gopt(argv[1:], [], [], main.__doc__ + __doc__)
for i in oss.ls(r'C:\windows\fonts\*.ttf'):
print(i.split('\\')[-1])
oss.exit(0)
def LoadLibraries(path, func, cxt=None):
#-------------------------------------------------------------------------------
if isinstance(path, (str, unicode)):
path = [path]
plugins = []
for pth in path:
for f in oss.ls(pth + '/*.py', pth + '/*.pyd'):
plugins.extend(func(oss.splitnm(oss.basename(f)), path, cxt))
return plugins
def LoadPlugins(path, suffix=None, prefix=None):
#-------------------------------------------------------------------------------
plugins = []
if not isinstance(path, list):
path = [path]
for pth in path:
for f in oss.ls(pth + '/*.py', pth + '/*.pyd'):
plugins.extend(LoadPlugin(oss.splitnm(oss.basename(f)), path, suffix, prefix))
return plugins
def LoadPlugins(path, suffix=None, prefix=None):
#-------------------------------------------------------------------------------
plugins = []
if not isinstance(path, list):
path = [path]
for pth in path:
for f in oss.ls(pth + '/*.py', pth + '/*.pyd'):
plugins.extend(
LoadPlugin(oss.splitnm(oss.basename(f)), path, suffix, prefix))
return plugins
def getNodes():
#-------------------------------------------------------------------------------
dd = {}
for d in oss.ls(DIR_PATH + '/node/node*/'):
c = oss.pathsplit(d)
dd[c[5]] = {
'node' : int(c[5][4:]),
'cpulist' : cvtToList(rl(d + '/cpulist')),
'memTotal' : rdMemInfo(d + '/meminfo', 'MemTotal'),
}
return dd
def getNodes():
#-------------------------------------------------------------------------------
dd = {}
for d in oss.ls(DIR_PATH + '/node/node*/'):
c = oss.pathsplit(d)
dd[c[5]] = {
'node': int(c[5][4:]),
'cpulist': cvtToList(rl(d + '/cpulist')),
'memTotal': rdMemInfo(d + '/meminfo', 'MemTotal'),
}
return dd
def getCPUs():
#-------------------------------------------------------------------------------
dd = {}
for d in oss.ls(DIR_PATH + '/cpu/cpu*/topology/'):
c = oss.pathsplit(d)
dd[c[5]] = {
'cpu' : int(c[5][3:]),
'phys_pkg_id' : rl(d + '/physical_package_id'),
'core_id' : rl(d + '/core_id'),
'core_sibs' : cvtToList(rl(d + '/core_siblings_list')),
'thread_sibs' : cvtToList(rl(d + '/thread_siblings_list')),
}
return dd
def getCPUs():
#-------------------------------------------------------------------------------
dd = {}
for d in oss.ls(DIR_PATH + '/cpu/cpu*/topology/'):
c = oss.pathsplit(d)
dd[c[5]] = {
'cpu': int(c[5][3:]),
'phys_pkg_id': rl(d + '/physical_package_id'),
'core_id': rl(d + '/core_id'),
'core_sibs': cvtToList(rl(d + '/core_siblings_list')),
'thread_sibs': cvtToList(rl(d + '/thread_siblings_list')),
}
return dd
def getCaches():
#-------------------------------------------------------------------------------
dd = {}
for d in oss.ls(DIR_PATH + '/cpu/cpu*/cache/*'):
c = oss.pathsplit(d)
dd[d] = {
'cpu': int(c[5][3:]),
'type': rl(d + '/type')[:7],
'level': rl(d + '/level'),
'shared_cpu_list': cvtToList(rl(d + '/shared_cpu_list')),
'size': rl(d + '/size'),
'num_sets': rl(d + '/number_of_sets'),
'assoc': rl(d + '/ways_of_associativity'),
}
return dd
def getCaches():
#-------------------------------------------------------------------------------
dd = {}
for d in oss.ls(DIR_PATH + '/cpu/cpu*/cache/*'):
c = oss.pathsplit(d)
dd[d] = {
'cpu' : int(c[5][3:]),
'type' : rl(d + '/type')[:7],
'level' : rl(d + '/level'),
'shared_cpu_list' : cvtToList(rl(d + '/shared_cpu_list')),
'size' : rl(d + '/size'),
'num_sets' : rl(d + '/number_of_sets'),
'assoc' : rl(d + '/ways_of_associativity'),
}
return dd
def main(argv):
#-------------------------------------------------------------------------------
""" usage: mkcvs [OPTIONS] <dir>
Makes a cvs archive from the specified directory
Options:
-i | --ignore : specify extensions or files or directories to ignore
example: -i .bmp -i .jpg or -i badfile
"""
args, opts = oss.gopt(argv[1:], [], [], [], [('i', 'ignore')], main.__doc__)
if not oss.exists(oss.env['CVSROOT']):
print "CVSROOT is down"
oss.exit(1)
if args:
oss.cd(args[0])
name = oss.basename(oss.pwd())
if oss.exists(oss.env['CVSROOT'] + '/' + name):
print >> oss.stderr, "Already exists:", oss.env['CVSROOT'] + '/' + name
oss.exit(1)
if opts.ignore is not None:
if isinstance(opts.ignore, list):
IGNORE_PATTERNS.extend(opts.ignore)
else:
IGNORE_PATTERNS.append(opts.ignore)
il = bldIgnoreList()
print "Ignoring Files: '%s'" % il
oss.r(r'C:\bin\cvs.exe import %s %s %s %s1' % (il, name, name, name))
oss.cd('..')
oss.r("mv %s /tmp/%s.bak" % (name, name))
oss.r(r'C:\bin\cvs.exe co %s' % name)
oss.cd(name)
for f in oss.ls():
print f
oss.r("C:\bin\cf.py")
oss.exit(0)
def decvs(self, args, opts):
#- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
for f in oss.ls():
bf = oss.basename(f)
if bf.lower() == 'cvs':
print('removing:', f)
oss.r('rm -rf cvs')
continue
if self.ignoreFile(bf):
continue
if oss.IsDir(bf):
oss.pushcd(bf)
self.decvs(args, opts)
oss.popcd()
def main(argv):
#-------------------------------------------------------------------------------
""" usage: mk_cmd_help.py [options] [<file_name> ...]
options:
-o | --output : html file output <default: 'cmd_help.html'
generate an html output file of the help strings from the specified
commands
"""
args, opts = oss.gopt(argv[1:], [], [('o', 'output')],
main.__doc__ + __doc__)
outfile = 'cmd_help.html' if opts.output is None else opts.output
if len(args) == 0:
args = oss.ls('*.py')
title = oss.pwd()
print("Writing to file: '%s'" % outfile)
with open(outfile, 'w') as otf:
otf.write(HTML_BEGIN % (title, title))
## make a table of links to prgms details
links = ['<a href="#{0}">{0}</a>'.format(nm) for nm in args]
otf.write('<a name="top"><h2>Contents</h2></a>\n')
otf.write("<center>\n" +
utils.makeTable(links, tattr='border="1" width="90%"') +
"\n</center>\n\n")
otf.flush()
## make an entry for each prgms details
for prgm in args:
print(prgm)
otf.write('\n\n<hr/>\n')
otf.write('<a name="{0}">'.format(prgm))
otf.write('<h2>' + prgm + '</h2></a>\n')
s = oss.r(prgm + ' --help', '$')
otf.write('<font size="5"><pre>' + cgi.escape(s) +
'</pre></font>\n')
otf.write('<a href="#top">Top</a>\n')
otf.flush()
otf.write(HTML_END)
oss.exit(0)
def checkfor(self, dir = '.'):
#- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
lst = []
if not oss.exists('CVS/Entries'):
return
## get list of files
inf = file('CVS/Entries')
have = set()
for line in inf:
try:
have.add(line.split('/')[1])
except:
pass
inf.close()
try:
igf = file(".cffignore")
for line in igf:
line = line[:-1]
have.add(line)
except IOError:
pass
inf.close()
for f in oss.ls():
f = oss.basename(f)
if self.ignoreFile(f):
continue
if oss.IsDir(f):
if not oss.exists(f + '/CVS'):
if f not in have:
lst.append("%s/%s/" % (dir, f))
else:
oss.pushcd(f)
lst.extend(self.checkfor(dir + '/' + f))
oss.popcd()
elif f not in have:
lst.append("%s/%s" % (dir, f))
return lst
def doDir(dir = '.'):
#-------------------------------------------------------------------------------
try:
inf = open("CVS/Entries")
except IOError:
oss.usage(3, "Not a CVS archive")
have = set()
for line in inf:
try:
have.add(line.split('/')[1].upper())
except:
pass
inf.close()
ignr = set()
try:
with open(".cffignore") as igf:
for line in igf:
ignr.add(line.strip())
except IOError:
pass
lst = []
for f in oss.ls():
f = oss.basename(f)
if ignoreFile(f, ignr):
continue
if oss.IsDir(f):
if not oss.exists(f + '/CVS'):
if f.upper() not in have:
lst.append("%s/%s/" % (dir, f))
else:
opwd = oss.pwd()
oss.cd(f)
lst.extend(doDir(dir + '/' + f))
oss.cd(opwd)
elif f.upper() not in have:
lst.append("%s/%s" % (dir, f))
return lst
def doDir(dir='.'):
#-------------------------------------------------------------------------------
try:
inf = open("CVS/Entries")
except IOError:
oss.usage(3, "Not a CVS archive")
have = set()
for line in inf:
try:
have.add(line.split('/')[1].upper())
except:
pass
inf.close()
ignr = set()
try:
with open(".cffignore") as igf:
for line in igf:
ignr.add(line.strip())
except IOError:
pass
lst = []
for f in oss.ls():
f = oss.basename(f)
if ignoreFile(f, ignr):
continue
if oss.IsDir(f):
if not oss.exists(f + '/CVS'):
if f.upper() not in have:
lst.append("%s/%s/" % (dir, f))
else:
opwd = oss.pwd()
oss.cd(f)
lst.extend(doDir(dir + '/' + f))
oss.cd(opwd)
elif f.upper() not in have:
lst.append("%s/%s" % (dir, f))
return lst
def main(argv):
#-------------------------------------------------------------------------------
""" usage: copyit <options>
options
-o | --once : run through loop once and then halt
-d | --dest : specify detination directory
-s | --src : specifly source directory
"""
args, opts = oss.gopt(argv[1:], [('o', 'once')], [('d', 'dest'), ('s', 'src')], main.__doc__)
if opts.dest is None:
opts.dest = getDir(DEST_DIRS)
if opts.src is None:
opts.src = getDir(SRC_DIRS)
if not oss.exists(opts.dest):
opts.usage(1, "Destination dir '%s' doesn't exist" % opts.dest)
if not oss.exists(opts.src):
opts.usage(1, "Source dir '%s' doesn't exist" % opts.src)
print("Destination:", opts.dest)
print("Source:", opts.src)
oss.cd(opts.src)
destDirName = opts.dest + ''.join(map(lambda s: "%02d" % s, time.localtime()[:6]))
print("Making", destDirName)
oss.mkdir(destDirName)
xfrc = XferFileClass()
id = fileCount = idx = 0
doneMsg = False
while 1:
done = True
files = oss.ls()
if not files:
if not doneMsg:
print('\n--- All files Done ---')
doneMsg = True
else:
doneMsg = False
for ef in files:
if oss.isDir(ef):
continue
ext = oss.splitext(ef)
## mozzilla
if ext in IGNORE_EXTENSIONS:
continue
fn = destDirName + '/%08d' % (id) + ext
fs = xfrc.xferFile(ef, fn)
if fs:
print(' xfer: %s <-- %s, %d' % (fn, ef, fs))
fileCount += 1
else:
done = False
id += 1
if done and opts.once:
break
ch = util.delayKey(DELAY_TIME)
if ch in ['q', 'Q']:
break
elif ch == 'x':
pprint.pprint(xfrc.xfers)
elif ch == 'p':
print('pausing')
while util.delayKey(1000) == None:
pass
print('\npausing stopped')
idx += 1
if idx > 30:
idx = 0
xfrc.bad = set()
if fileCount == 0:
print("Removing unused directory '%s'" % destDirName)
oss.rmdir(destDirName)
else:
print("Copied %d files" % fileCount)
print('Bad Files:', xfrc.bad)
import pylib.osscripts as oss
oss.cd('/home/chrish/tmp')
for f in oss.ls('*.st'):
d = f.split('.')[:-3]
f = '.'.join(d) + '.html'
d = f.split('_')
nm = d[0]
f = '_'.join(d[1:])
print 'http://xhamster.com/movies/' + nm + '/' + f
