def appendToCsv(d,path,filename,df,ndf,bdf):
try:
f = bz2.BZ2File( join(path,filename),'r')
ms = [Record().fromString( s.rstrip('\n')) for s in f.readlines()]
fp = FileNameParts(filename)
appendToBulkdataCsv(ms,fp,bdf)
appendToBothDataCsv(ms,fp,df,ndf)
except IOError, err:
if logger: logger.error(err)
def worklist():
global srcdir
w = []
subdirs = [
each for each in os.listdir(dirs['src']) if each in filters['onlydirs']
]
subdirs.sort()
for d in subdirs:
makeSubDirectoriesFor(d)
# take all likely names in src directory
# create simpleNames for each name and each possible alias
srcdir = join(dirs['src'], d)
# include undeleted filenames that might have a file extension
files = fnmatch.filter(os.listdir(srcdir), '*.*[!~]')
# exclude bogus files created by touch
files = [FileNameParts(f) for f in files if not f.startswith('*')]
original = [fp for fp in files]
src = set(original)
aliased = []
for o in original:
for imp in alias.get(o.imp, []):
aliased.append(LinkNameParts(o.filename, imp))
src.update(aliased)
clean(d, src)
allowed, sources, links = targets(original, aliased)
programs = set(sources)
programs.update(links)
# symlink these programs from codedir so they are available
# for conversion to highlight-ed xml files
codedir = join(dirs['tmp'], d, 'code')
for f in programs:
linkToSource(directory=srcdir,
srcFilename=f.filename,
dstDir=codedir,
filename=f.programName)
linkToSource(directory=srcdir,
srcFilename=f.filename,
dstDir=codedir,
filename=f.codeName)
w.append((d, programs, allowed))
return w
def appendToCsv(d, path, filename, df, ndf, bdf):
try:
f = bz2.BZ2File(join(path, filename), 'r')
try:
ms = [Record().fromString(s.rstrip('\n')) for s in f.readlines()]
except Exception as ex:
print join(path, filename), 'corrupted, deleting'
os.remove(join(path, filename))
return
fp = FileNameParts(filename)
appendToBulkdataCsv(ms, fp, bdf)
appendToBothDataCsv(ms, fp, df, ndf)
except IOError, err:
if logger: logger.error(err)