def SymLinkAction(target, source, env):
for trg in target:
# symlink the target from the source
fullsrc = str(source[0])
src = os.path.basename(str(source[0]))
dst = str(trg)
log(11, ['SymLinkAction %s -> %s' % (src, dst)])
try:
os.remove(dst)
except:
pass
try:
log(12, ['symlink(%s, %s)' % (src, dst)])
os.symlink(src, dst)
except:
try:
log(12, ['hardlink(%s, %s)' % (fullsrc, dst)])
os.link(fullsrc, dst)
except:
try:
log(12, ['copy(%s, %s)' % (fullsrc, dst)])
os.copy(fullsrc, dst)
except Exception, ex:
log(0, ['SymLinkAction failed: %s' % (str(ex))])
raise ex
def runInitScripts(self, scriptPath, *args, **kwargs):
if not os.path.exists(scriptPath):
return False, 'Path not found: %s' % scriptPath
if os.path.isfile(scriptPath):
os.copy(scriptPath, self.workDir)
if os.path.isdir(scriptPath):
shutil.rmtree(self.workDir)
shutil.copytree(scriptPath, self.workDir)
os.chdir(self.workDir)
initList = os.listdir(self.workDir)
for initFile in sorted(initList):
if os.name == 'nt' and self.execType != 'bat':
return False, ''
if os.name == 'posix' and self.execType != 'sh':
return False, ''
extLen = len(self.execType)
if initFile[-extLen:] == self.execType:
initDir = initFile[: - (extLen + 1)]
os.environ[initDir] = self.workDir
self.log('INFO', 'Env: %s' % os.environ[initDir])
self.prepareEnvironment(os.environ[initDir])
self.log('INFO', 'chdir: %s' % os.environ[initDir])
try:
self.log('INFO', 'Running script %s from %s.' % (initFile, self.workDir))
p1 = subprocess.Popen([self.workDir + os.sep + initFile, ], stdout=subprocess.PIPE)
self.log('INFO', p1.stdout.read())
except Exception, e:
msg = 'Error in executing init script: ' + initFile + ' path: ' + self.workDir
self.application.exceptionTrace(e, message=msg)
self.setStatus('1')
def makecolorimage(self):
"""Make color image (in sections)"""
if (self.stampsize == self.samplesize == 0) and self.testfirst:
# Already did the full image!
print 'Full size image already made.'
imfile = self.testimages[-1]
outfile = join(self.outdir, self.outfile)
if self.deletetests:
print 'Renaming to', outfile
os.rename(imfile, outfile)
else:
print 'Copying to', outfile
os.copy(imfile, outfile)
imfull = Image.open(outfile)
return imfull
# Clean up: Delete test images
if self.deletetests:
for testimage in self.testimages:
if exists(testimage):
os.remove(testimage)
dx = dy = self.stampsize
if dx * dy == 0:
dx = dy = self.maxstampsize
imfull = Image.new(self.mode, (self.nx, self.ny))
if self.mode == 'RGB':
if self.verbose:
print 'Making full color image, one stamp (section) at a time...'
elif self.mode == 'L':
if self.verbose:
print 'Making full grayscale image, one stamp (section) at a time...'
for yo in range(self.ylo,self.yhi,dy):
dy1 = min([dy, self.yhi-yo])
for xo in range(self.xlo,self.xhi,dx):
dx1 = min([dx, self.xhi-xo])
if self.verbose:
print '%5d, %5d / (%d x %d)' % (xo, yo, self.nx, self.ny)
limits = yo, yo+dy, xo, xo+dx
stamps = self.loadstamps(limits)
im = RGBscale2im(stamps, self.levdict, self.noiselums, self.colorsatfac, self.mode)
if self.show and self.showstamps:
im.show()
imfull.paste(im, (xo,self.ny-yo-dy1,xo+dx1,self.ny-yo))
outfile = join(self.outdir, self.outfile)
if self.legend:
self.addlegend(im=imfull)
else:
if self.verbose:
print 'Saving', outfile, '...'
imfull.save(outfile)
if self.show:
self.showimage(outfile, Image)
return imfull
def execute(self, pipedata, prefix=""):
task = self.bakery.logging_task('Convert sources to TTF')
if self.bakery.forcerun:
return
if pipedata.get('compiler') == 'make':
import subprocess
directory = UpstreamDirectory(op.join(self.builddir, 'sources'))
snapshot_bin = directory.BIN
process = subprocess.Popen(['make'],
cwd=op.join(self.builddir, 'sources'))
process.communicate()
directory = UpstreamDirectory(op.join(self.builddir, 'sources'))
snapshot_after_bin = directory.BIN
for fontpath in set(snapshot_bin) ^ set(snapshot_after_bin):
if fontpath.lower().endswith('.ttf'):
os.copy(op.join(self.builddir, 'sources', fontpath),
self.builddir)
self.run_processes(op.basename(fontpath), pipedata)
return pipedata
self.convert(pipedata)
return pipedata
def create_ca(my_dir='etc/ca', ca_name='demoCA'):
complete_dir = my_dir + os.sep + ca_name
if os.path.exists(complete_dir):
return
os.mkdir(complete_dir)
os.mkdir(complete_dir + '/certs')
os.mkdir(complete_dir + '/crl')
os.mkdir(complete_dir + '/newcerts')
os.mkdir(complete_dir + '/private')
w = open(complete_dir + '/index.txt', 'w')
w.close()
ret = os.system(
'openssl req -new -keyout {complete_dir}/private/cakey.pem -config init.ssl -nodes -out {complete_dir}/careq.pem'
.format(complete_dir=complete_dir))
# 2000 is number of days (arbitrary, really)
if ret != 0:
return False
cwd = os.getcwd()
os.chdir(my_dir)
ret = os.system(
'openssl ca -create_serial -out {ca_name}/cacert.pem -days 2000 -batch -keyfile {ca_name}/private/cakey.pem -selfsign -extensions v3_ca -infiles {ca_name}/careq.pem'
.format(ca_name=ca_name))
os.copy('%s/cacert.pem' % ca_name, 'cacert.pem')
os.chdir(cwd)
return ret == 0
def execute(self, pipedata, prefix=""):
task = self.bakery.logging_task('Convert sources to TTF')
if self.bakery.forcerun:
return
if pipedata.get('compiler') == 'make':
import subprocess
directory = UpstreamDirectory(op.join(self.builddir, 'sources'))
snapshot_bin = directory.BIN
process = subprocess.Popen(['make'], cwd=op.join(self.builddir, 'sources'))
process.communicate()
directory = UpstreamDirectory(op.join(self.builddir, 'sources'))
snapshot_after_bin = directory.BIN
for fontpath in set(snapshot_bin) ^ set(snapshot_after_bin):
if fontpath.lower().endswith('.ttf'):
os.copy(op.join(self.builddir, 'sources', fontpath), self.builddir)
self.run_processes(op.basename(fontpath), pipedata)
return pipedata
self.convert(pipedata)
return pipedata
def create_ca(my_dir="etc/ca", ca_name="demoCA"):
complete_dir = my_dir + os.sep + ca_name
if os.path.exists(complete_dir):
return
os.mkdir(complete_dir)
os.mkdir(complete_dir + "/certs")
os.mkdir(complete_dir + "/crl")
os.mkdir(complete_dir + "/newcerts")
os.mkdir(complete_dir + "/private")
w = open(complete_dir + "/index.txt", "w")
w.close()
ret = os.system(
"openssl req -new -keyout {complete_dir}/private/cakey.pem -config init.ssl -nodes -out {complete_dir}/careq.pem".format(
complete_dir=complete_dir
)
)
# 2000 is number of days (arbitrary, really)
if ret != 0:
return False
cwd = os.getcwd()
os.chdir(my_dir)
ret = os.system(
"openssl ca -create_serial -out {ca_name}/cacert.pem -days 2000 -batch -keyfile {ca_name}/private/cakey.pem -selfsign -extensions v3_ca -infiles {ca_name}/careq.pem".format(
ca_name=ca_name
)
)
os.copy("%s/cacert.pem" % ca_name, "cacert.pem")
os.chdir(cwd)
return ret == 0
def store(self, photo_file, new_path):
"""Stores photo file into path into storage.
Args:
photo_file: str, path to photo file
new_path: str, path in storage to copy photo_file
"""
os.copy(photo_file, os.path.join(self.path, new_path))
def main():
args = parser_args()
# Moves over raw pdb to be editted
os.copy('/home/aseamann/public_html/rawPDBs/' + args.rawPDB + ".pdb",
'/home/aseamann/public_html/modPDBs/' + args.modPDB + ".pdb")
# Submits pdb to PdbTools3
pdb = PdbTools3('/home/aseamann/public_html/modPDBs/' + args.modPDB)
all_chains = "FALSE"
trimmed = "FALSE"
tcr_only = "FALSE"
p_only = "FALSE"
mhc_only = "FALSE"
renum = "FALSE"
# Based on arguments passed, updated T/F for if the chain still remains in file
if args.mhc_split:
pdb.split_mhc()
mhc_only = "TRUE"
elif args.trim:
pdb.clean_tcr_count_trim()
trimmed = "TRUE"
tcr_alpha = "TRUE"
tcr_beta = "TRUE"
renum = "TRUE"
elif args.tcr_split:
pdb.clean_tcr()
tcr_alpha = "TRUE"
tcr_beta = "TRUE"
elif args.peptide_split:
pdb.split_p()
p_only = "TRUE"
else:
all_chains = "TRUE"
if args.renum:
pdb.clean_docking_count()
renum = "TRUE"
# Dictionary that hold values to be submitted
pdb_data = {
'pdbID_mod': args.rawPDB,
'tempID': args.modPDB,
'all_chains': all_chains,
'trimmed': trimmed,
'tcr_only': tcr_only,
'peptide': p_only,
'mhc': mhc_only,
'renum': renum
}
# Writes to sql file that will be submitted to db
with open('modPDBinsert.sql', 'w') as f:
f.write('USE aseamann;\n')
output = "INSERT INTO modPDB(pdbID_mod, tempID, all_chains, trimmed, tcr_only, peptide, mhc, renum) " \
"VALUES('%(pdbID_mod)s', '%(tempID)s', '%(all_chains)s', '%(trimmed)s', '%(tcr_only)s', '%(peptide)s', '%(mhc)s', '%(renum)s');" % pdb_data
f.write(output + '\n')
os.system('mysql< modPDBinsert.sql') # Submit to MySql
def save_data( data, db_file ):
"""Save the data into the given file name."""
if os.path.exists( db_file ):
if os.path.exists( db_file+".bak" ):
os.unlink( db_file+".bak" )
os.copy( db_file, db_file+".bak" )
with open( db_file+".tmp", "w" ) as target:
wtr= csv.writer( target )
wtr.writerows( data )
os.unlink( db_file )
os.rename( db_file+".tmp", db_file )
def copy() : if len(sys.argv) == 4 : try : shutil.copytree( sys.argv[2], sys.argv[3]) except OSError as exc: # python >2.5 if exc.errno == errno.ENOTDIR: os.copy(sys.argv[2]) else: sys.exit(exc) print "copied world from '" + sys.argv[2] + "' too '" + sys.argv[3] + "'"; else : print "Wrong Argument Count!"
def _backupConfigs(self):
if not os.path.exists(self.configDir):
return False
backupFolder = os.path.join(self.configDir, "backup",
datetime.datetime.now().isoformat())
for filename in [
x for x in os.listdir(self.configDir)
if os.path.isdir(os.path.join(self.configDir, x))
]:
os.copy(os.path.join(self.configDir, filename),
os.path.join(backupFolder, filename))
def main():
parser = argparse.ArgumentParser(description='Plot 3d matrix data')
parser.add_argument('input', help='File containing 3d data as matrix')
parser.add_argument('-o', '--output', default=None, help='Output file')
parser.add_argument('-c', '--column', default=None, help='Column for 2d plot', type=int)
parser.add_argument('-r', '--row', default=None, help='Row for 2d plot', type=int)
parser.add_argument('-l', '--logscale', action="store_true" , help='plot z axis with logscale')
parser.add_argument('-q', '--quiet', action="store_true" , help='no screen output')
parser.add_argument('-p', '--config', default=None, help='gnuplot config file')
args = parser.parse_args()
if args.column==None and args.row==None:
f=open("temp.plt", 'w')
f.write("set pm3d\n")
f.write("unset surface\n")
if args.logscale:
f.write("set logscale z\n")
if not args.quiet:
f.write("splot '%s' matrix\n" % args.input)
f.write("pause -1\n")
if args.output!=None:
f.write("set terminal postscript enhanced color\n")
f.write("set output '%s'\n" % args.output)
f.write("splot '%s' matrix\n" % args.input)
f.close();
cmd = 'gnuplot temp.plt'
subprocess.call(cmd, shell=True)
if args.config==None:
os.remove("temp.plt")
else:
cmd = 'mv temp.plt %s' % args.config
subprocess.call(cmd, shell=True)
else:
data = numpy.loadtxt(args.input)
f=open("temp.dat", 'w')
if args.column!=None:
subdata=data[:,args.column]
for x,y in zip(numpy.arange(0,len(subdata)), subdata):
print x,y
f.write("%i\t%e\n" % (x,y))
else:
subdata=data[args.row,:]
for x,y in zip(numpy.arange(0,len(subdata)), subdata):
print x,y
f.write("%i\t%e\n" % (x,y))
f.close()
if args.output!=None:
os.copy("temp.dat", args.output)
cmd = 'xmgrace temp.dat'
subprocess.call(cmd, shell=True)
os.remove("temp.dat")
def disable_chroot_daemons(chroot):
"""
Disable daemons from starting in a chroot
Run this before installing daemon packages like ssh or tor
"""
# Normally, policy-rc.d doesn't exist. If it does, then daemons have already been disabled
if os.path.exists(chroot+'/usr/sbin/policy-rc.d'):
logging.debug("Daemons already disabled in chroot {}".format(chroot))
return
logging.debug("Disabling daemons in chroot {}".format(chroot))
write_file(['#!/bin/sh', 'exit 101'], chroot+'/usr/sbin/policy-rc.d', mode=0o755)
sh(
'dpkg-divert --add --local --divert /usr/sbin/invoke-rc.d.chroot --rename /usr/sbin/invoke-rc.d',
env={ 'LANG':'C' }, chroot=chroot)
os.copy(chroot+'/bin/true', chroot+'/usr/sbin/invoke-rc.d')
def setup_copy(newname, pattern):
os.chdir("..")
os.mkdir(newname)
os.chdir(newname)
os.copy("../wakawakawaka/LICENSE", ".")
os.copy("../wakawakawaka/README.md", ".")
os.copy("../wakawakawaka/base.py", ".")
os.copy("../wakawakawaka/vote.txt", ".")
os.copy("../wakawakawaka/waka.txt", ".")
os.system("git init")
os.system("")
os.system("git add .")
os.system('git commit -am "Base"')
load_pattern(pattern)
push_pattern(pattern)
def cli_sanitize(input_filepath, save=False, delete=False):
""" Copies out a filename without spaces, or deletes that file.
Filename invertibility is not
Will not work for directories with spaces in their names.
"""
input_filepath = os.path.abspath(input_filepath)
new_filepath = os.path.join(os.path.dirname(input_filepath), os.path.basename(input_filepath).replace(' ', '__'))
if delete:
os.remove(new_filepath)
if save:
os.copy(input_filepath, new_filepath)
return new_filepath
def __make_local_copy(self):
"""
Retrieves the file from the location specified at <source> and makes
a local copy.
"""
# Verifiy the file does not already exist
if os.path.exists(self.filepath):
last_updated_time = os.path.getctime(self.filepath)
return
# Retrieve and save the file
print("Retrieving file from " + self.source + "...")
try:
# The source is a URL
request.urlretrieve(self.source, self.filepath)
print("Downloaded and saved " + self.filename)
except ValueError:
# The source is a file path
os.copy(self.source, self.filepath)
def resize_file(filename):
f = filename.rsplit(".", 1)[0]
try:
im = Image.open(filename)
w, h = im.size
max_width = 400
#print w,h
if w > max_width:
nw = max_width
nh = nw * h / w
#print nw,nh
else:
os.copy(filename, f)
return
im.thumbnail((nw, nh), Image.ANTIALIAS)
im.save(f, quality=80)
#os.remove(filename)
except IOError, e:
print "cannot create thumbnail for '%s'" % filename
print e
def process_lidc_annotations(only_patient=None, agreement_threshold=0):
# lines.append(",".join())
file_no = 0
pos_count = 0
neg_count = 0
all_lines = []
for anno_dir in [
d for d in glob.glob("resources/luna16_annotations/*")
if os.path.isdir(d)
]:
xml_paths = glob.glob(anno_dir + "/*.xml")
for xml_path in xml_paths:
print(file_no, ": ", xml_path)
pos, neg, extended = load_lidc_xml(
xml_path=xml_path,
only_patient=only_patient,
agreement_threshold=agreement_threshold)
if pos is not None:
pos_count += len(pos)
neg_count += len(neg)
print("Pos: ", pos_count, " Neg: ", neg_count)
file_no += 1
all_lines += extended
# if file_no > 10:
# break
# extended_line = [nodule_id, x_center_perc, y_center_perc, z_center_perc, diameter_perc, malignacy, sphericiy, margin, spiculation, texture, calcification, internal_structure, lobulation, subtlety ]
df_annos = pandas.DataFrame(all_lines,
columns=[
"patient_id", "anno_index", "coord_x",
"coord_y", "coord_z", "diameter",
"malscore", "sphericiy", "margin",
"spiculation", "texture", "calcification",
"internal_structure", "lobulation",
"subtlety"
])
df_annos.to_csv(settings.BASE_DIR + "lidc_annotations.csv", index=False)
os.copy()
def makecolorimage(self):
"""Make color image (in sections)"""
if (self.stampsize == self.samplesize == 0) and self.testfirst:
# Already did the full image!
print 'Full size image already made.'
imfile = self.testimages[-1]
outfile = join(self.outdir, self.outfile)
if self.deletetests:
print 'Renaming to', outfile
os.rename(imfile, outfile)
else:
print 'Copying to', outfile
os.copy(imfile, outfile)
imfull = Image.open(outfile)
return imfull
# Clean up: Delete test images
if self.deletetests:
for testimage in self.testimages:
if exists(testimage):
os.remove(testimage)
dx = dy = self.stampsize
if dx * dy == 0:
dx = dy = self.maxstampsize
imfull = Image.new(self.mode, (self.nx, self.ny))
print
if self.mode == 'RGB':
print 'Making full color image, one stamp (section) at a time...'
elif self.mode == 'L':
print 'Making full grayscale image, one stamp (section) at a time...'
for yo in range(0,self.ny,dy):
dy1 = min([dy, self.ny-yo])
for xo in range(0,self.nx,dx):
dx1 = min([dx, self.nx-xo])
print '%5d, %5d / (%d x %d)' % (xo, yo, self.nx, self.ny)
#print dx, dy, self.nx, self.ny, dx1, dy1
#stamps = self.dataRGB[:,yo:yo+dy,xo:xo+dx]
limits = yo, yo+dy, xo, xo+dx
stamps = self.loadstamps(limits)
im = RGBscale2im(stamps, self.levdict, self.noiselums, self.colorsatfac, self.mode)
if self.show and self.showstamps:
im.show()
#print array(stamps).shape, im.size, xo,self.ny-yo-dy1,xo+dx1,self.ny-yo
imfull.paste(im, (xo,self.ny-yo-dy1,xo+dx1,self.ny-yo))
#outfile = outname+'.png'
outfile = join(self.outdir, self.outfile)
print 'Saving', outfile, '...'
imfull.save(outfile)
#imfull.save(root+'.jpg')
if self.show:
try:
os.system('open ' + outfile)
except: # If you're not on a Mac
imfull.show() # Converts to jpg first, so a bit degraded
return imfull
healstr['file'] = healstr['file'].replace('/net/dl1/','/dl1/')
# Write the full list plus an index
print('Writing list to '+listfile)
Table(healstr).write(listfile)
# append other fits binary tables
hdulist = fits.open(listfile)
hdu = fits.table_to_hdu(Table(indexj)) # second, catalog
hdulist.append(hdu)
hdulist.writeto(listfile,overwrite=True)
hdulist.close()
if os.path.exists(listfile+'.gz'): os.remove(listfile+'.gz')
ret = subprocess.call(['gzip',listfile]) # compress final catalog
# Copy to local directory for faster reading speed
if os.path.exists(localdir+'dnidever/nsc/instcal/'+version+'/'): os.delete(localdir+'dnidever/nsc/instcal/'+version+'/')
os.copy(listfile+'.gz',localdir+'dnidever/nsc/instcal/'+version+'/')
# PUT NSIDE IN HEADER!!
# Using existing list
else:
print('Reading list from '+listfile)
healstr = fits.getdata(listfile,1)
index = fits.getdata(listfile,2)
upix = index['pix']
npix = len(index)
# Copy to local directory for faster reading speed
file_copy,listfile,localdir+'dnidever/nsc/instcal/'+version+'/',/over
# Load the list of healpix pixels for this server to be run LOCALLY
print(err);
return;
ckId = 'RIFF';
ckSize = size + 44 - 8;
ctype = 'WAVE';
sId = 'fmt ';
sSize = 0x10;
stag = 1;
sChnl = channel_count;
sSampleR = sample_rate;
sBytePS = sample_rate * (bit_deep / 8) * channel_count;
sBlockA = bit_deep * channel_count / 8;
sBitD = bit_deep;
dId = 'data';
dSize = size;
os.copy(pcm_file,pcm_file+".wav");
dst_file = open(pcm_file+".wav","w");
wavHeader= struct.pack(">4sI4s4sIHHIIHH4sI",ckId,ckSize,
ctype,sId,sSize,stag,sChnl,sSampleR,sBytePS,
sBlockA,sBitD,dId,dSize);
dst_file.write(dst_file)
def resample_wav(wav_source,wav_dst,dst_rate)
def makecolorimage(self):
"""Make color image (in sections)"""
if (self.stampsize == self.samplesize == 0) and self.testfirst:
# Already did the full image!
print "Full size image already made."
imfile = self.testimages[-1]
outfile = join(self.outdir, self.outfile)
if self.deletetests:
print "Renaming to", outfile
os.rename(imfile, outfile)
else:
print "Copying to", outfile
os.copy(imfile, outfile)
imfull = Image.open(outfile)
return imfull
# Clean up: Delete test images
if self.deletetests:
for testimage in self.testimages:
if exists(testimage):
os.remove(testimage)
dx = dy = self.stampsize
if dx * dy == 0:
dx = dy = self.maxstampsize
imfull = Image.new(self.mode, (self.nx, self.ny))
print
if self.mode == "RGB":
print "Making full color image, one stamp (section) at a time..."
elif self.mode == "L":
print "Making full grayscale image, one stamp (section) at a time..."
# for yo in range(0,self.ny,dy):
# dy1 = min([dy, self.ny-yo])
# for xo in range(0,self.nx,dx):
# dx1 = min([dx, self.nx-xo])
for yo in range(self.ylo, self.yhi, dy):
dy1 = min([dy, self.yhi - yo])
for xo in range(self.xlo, self.xhi, dx):
dx1 = min([dx, self.xhi - xo])
print "%5d, %5d / (%d x %d)" % (xo, yo, self.nx, self.ny)
# print dx, dy, self.nx, self.ny, dx1, dy1
# stamps = self.dataRGB[:,yo:yo+dy,xo:xo+dx]
limits = yo, yo + dy, xo, xo + dx
stamps = self.loadstamps(limits)
im = RGBscale2im(stamps, self.levdict, self.noiselums, self.colorsatfac, self.mode)
if self.show and self.showstamps:
im.show()
# print array(stamps).shape, im.size, xo,self.ny-yo-dy1,xo+dx1,self.ny-yo
imfull.paste(im, (xo, self.ny - yo - dy1, xo + dx1, self.ny - yo))
# outfile = outname+'.png'
outfile = join(self.outdir, self.outfile)
if self.legend:
self.addlegend(im=imfull)
else:
print "Saving", outfile, "..."
imfull.save(outfile)
# imfull.save(root+'.jpg')
if self.show:
self.showimage(outfile, Image)
return imfull
def _hackFile(self, phase, options, step, stepoptions):
self.log.devdebug("aspect id: %s", self.log.params['AspectId'])
self.log.devdebug("options are: %s", str(options))
self.log.devdebug("aspect address: %s", hex(id(self)))
self.targetFile = options['file']
self.log.info("Modifying file '%s'", self.targetFile)
try:
self.isLink = os.path.exists(self.targetFile) \
and os.path.islink(self.targetFile)
except:
try:
subprocess.check_call(
['sudo', 'test', '-L', self.targetFile] )
self.isLink = True
except Exception as e:
self.log.info("File '%s' is not a link or could not be queried", self.targetFile)
self.log.debug(" Exception: %s: %s", e.__class__.__name__, str(e))
if self.isLink:
self.log.debug("'%s' is a link", self.targetFile)
try:
self.linkPath = os.readlink(self.targetFile)
except:
try:
self.linkPath = subprocess.check_output(
['sudo', 'readlink', self.targetFile] ).strip()
except Exception as e:
self.log.info("Link for '%s' could not be resolved", self.targetFile)
self.log.debug(" Exception: %s: %s", e.__class__.__name__, str(e))
self.isLink = False
dirtolink, _ = os.path.split(self.targetFile)
self.originalLink = self.linkPath
self.linkPath = os.path.join(dirtolink, self.linkPath)
if self.isLink:
try:
os.unlink(self.targetFile)
os.copy(self.linkPath, self.targetFile)
except:
try:
subprocess.call(
['sudo', 'unlink', self.targetFile],
stdout = self.log.out(),
stderr = self.log.err())
subprocess.check_call(
['sudo', 'cp', self.linkPath, self.targetFile],
stdout = self.log.out(),
stderr = self.log.err())
except Exception as e:
self.log.info("Link for '%s' could not be duplicated to an original file", self.targetFile)
self.log.debug(" Exception: %s: %s", e.__class__.__name__, str(e))
subprocess.call(
['sudo', 'ln', '-s', self.linkPath, self.targetFile],
stdout = self.log.out(),
stderr = self.log.err())
self.isLink = False
self.tempFile = self.targetFile + ".csmake-TemporaryFileHack-save"
if 'temp-file' in options:
self.tempFile = options['temp-file']
self.log.info("Saving original file in '%s'", self.tempFile)
if os.path.exists(self.tempFile):
self.log.warning("The temporary original file already exists")
self.log.warning("Not applying this change as it is assumed it is already applied")
self.log.passed()
return None
self.dirpath = os.path.dirname(self.targetFile)
if not self._createPathTo(self.dirpath):
self.log.error("Create path to directory '%s' failed", self.dirpath)
self.dirpath = None
self.log.failed()
return None
#Save the mode string for the dir to restore it on exit
if os.path.exists(self.targetFile):
self.realFileMode = oct(os.stat(self.targetFile).st_mode & 0777)[1:]
self._ensureDirectoryExists(self.tempFile)
result = subprocess.call(
['sudo', 'chmod', '777', self.targetFile],
stdout = self.log.out(),
stderr = self.log.err() )
if result != 0:
self.log.warning("Attempting to change the mode of the target file failed")
shutil.copy2(self.targetFile, self.tempFile)
else:
#Go ahead and touch the file
with file(self.targetFile, 'w') as openTarget:
openTarget.write('')
if 'replace' in options:
return self._replace(options)
if 'replace-file' in options:
return self._replaceFile(options)
if 'patch' in options:
return self._applyPatch(options)
if 'patch-file' in options:
return self._applyPatchFile(options)
if 'append' in options:
return self._append(options)
if 'append-file' in options:
return self._appendFile(options)
self.log.error("A file hack aspect was specified, but no action option was given")
self.log.failed()
return False
print 'could not read ' + str(justfilename) + ' for TxtToLines in rputiles'
pass
return lines
##########################
uzipped = TS + 'actant logs'
zips = uzipped + '/zipped logs'
listz = os.listdir(zips)
listu = os.listdir(uzipped)
for f in listu:
if 'AQTOR' in f:
hf = uzipped + '/' + f + '/Data/HyperFeedInstruments.txt'
subdir = uzipped + '/' + f
sublist = os.listdir(subdir)
for filename in sublist:
if 'screenshot' in filename:
os.copy(subdir + filename, tgt + shortname + shortdate + filename
## print hf
shortname = f.split('_')[1]
shortdate = (f.split('_')[2])[0:5]
## print shortname, shortdate
hstocks = TxtToLines(hf)
for lineb in hstocks:
if '//' not in lineb:
print shortname, shortdate, lineb
print('\n======generate cover page and readme #####################')
#disable page number in cover page
Popen([
'pandoc', input_folder + "coverpage.docx", "-o",
build_folder + "coverpage.pdf"
]).communicate()
Popen(['pandoc', root_folder + "Readme.md", "-o",
build_folder + "Readme.pdf"]).communicate()
from PyPDF2 import PdfFileReader, PdfFileMerger
pdf_files = [
build_folder + "coverpage.pdf", build_folder + "Readme.pdf",
merged_filename_base + ".pdf"
]
#for f in pdf_files: print(f)
merger = PdfFileMerger()
for filename in pdf_files:
merger.append(PdfFileReader(filename, "rb"))
final_output_filename = "../pdf/" + "FreeCAD_Mod_Dev_Guide" + ts + ".pdf"
daily_build_filename = "../pdf/" + "FreeCAD_Mod_Dev_Guide" + ".pdf"
if os.path.exists(final_output_filename):
os.remove(final_output_filename)
if os.path.exists(daily_build_filename):
os.remove(daily_build_filename)
merger.write(final_output_filename)
os.copy(final_output_filename, daily_build_filename)
print("final merged file: ", final_output_filename)
print('====== merge done!================')
from setuptools import setup # Always prefer setuptools over distutils
from codecs import open # To use a consistent encoding
from os import path
here = path.abspath(path.dirname(__file__))
if not path.exists(path.join(here, 'README.rst')):
try:
import subprocess
subprocess.call(['pandoc',
'--from=markdown', '--to=rst',
path.join(here, 'README.md'),
'--output', path.join(here, 'README.rst')])
except:
import os
os.copy(path.join(here, 'README.md'), path.join(here, 'README.rst'))
# Get the long description from the relevant file
with open(path.join(here, 'README.rst'), encoding='utf-8') as f:
long_description = f.read()
import version
setup(
name='openvpn2dns',
version=version.STRING,
description='DNS server in python serving openvpn status files as dns zones',
long_description=long_description,
# any later version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
# more details.
#
# You should have received a copy of the GNU General Public License along
# with this program. If not, see <http://www.gnu.org/licenses/>.
#
import os
import logging
ROOT = './'
path = ROOT + '/zimu_text'
logging.basicConfig(filename='clean_subtitle.log', level=logging.DEBUG)
for root, dirs, files in os.walk(path):
for file_name in files:
name, extension = os.path.splitext(file_name)
if extension.lower() in ['.txt', '.srt', '.ass', '.ssa', '.lrc']:
old_file = root + '/' + file_name
new_file = './zimu_cleaned/' + file_name
try:
os.copy(old_file, new_file)
except Exception as e:
logging.error(str(e))
logging.info()
os.path.basename(spzip)[:-4])
#Need to specify a compiler, or do the distutils.cfg?
#The f2py thing is weird. The script being on the path is
#insufficient for the call in wine, but seemed to work on the
#XP VM. This should be fixed in the installer.
runner.run(['python', 'setup.py', 'bdist_wininst',
'--f2py=python C:\\\\Python27\\\\Scripts\\\\f2py.py'],
bits=32, cwd=spacepybuild)
#Now install it, grab the omni database, run unit tests
spinstaller = os.path.basename(
sorted(glob.glob(os.path.join(spacepybuild, 'dist', 'spacepy-*.exe')))[-1])
runner.run([spinstaller, '/s'], bits=32,
cwd=os.path.join(spacepybuild, 'dist'))
#Force the directory for the ".spacepy" file, so can populate it w/OMNI, etc.
runner.env[32]['SPACEPY'] = "C:\\spacepy_user"
dotspacepy = os.path.join(c, 'spacepy_user', '.spacepy')
os.makedirs(dotspacepy)
import spacepy #this is in the BUILD environment, not the target!
shutil.copytree(os.path.join(spacepy.DOT_FLN, 'data'),
os.path.join(dotspacepy, 'data'))
retval = runner.run(['python', 'test_spacepy.py'], bits=32,
cwd=os.path.join(spacepybuild, 'tests'))[0]
if retval:
print('Unit tests failed.')
runner.finish(False)
else:
print('{0} built and tested.'.format(spinstaller))
os.copy(os.path.join(spacepybuild, 'dist', spinstaller),
installers)
runner.finish(True)
def main(self):
runner = Winerunner()
self.runner = runner
self.download_anaconda()
self.install_anaconda()
runner.run([os.path.join(self.installers,
'ffnet-0.7.1.win32-py2.7.exe'), '/s'],
32)
#THIS at least can go unattended
runner.run([os.path.join(self.installers,
'cdf35_0_2-setup-32.exe'), '/q'],
32)
#oh, dear, mingw
#http://www.mingw.org/wiki/InstallationHOWTOforMinGW
#but don't use links in that guide!
#Despite what it says, maybe need dev for gmp, mpc, mpfr, zlib?
#binutils-2.24-1-mingw32-bin.tar.xz
#gcc-core-4.8.1-4-mingw32-bin.tar.lzma
#gcc-core-4.8.1-4-mingw32-dev.tar.lzma
#gcc-core-4.8.1-4-mingw32-dll.tar.lzma
#gcc-fortran-4.8.1-4-mingw32-bin.tar.lzma
#gcc-fortran-4.8.1-4-mingw32-dev.tar.lzma
#gcc-fortran-4.8.1-4-mingw32-dll.tar.lzma
#gettext-0.18.3.2-1-1-mingw32-bin.tar.xz
#gmp-5.1.2-1-mingw32-dev.tar.lzma
#gmp-5.1.2-1-mingw32-dll.tar.lzma
#libasprintf-0.18.3.2-1-mingw32-dll-0.tar.xz
#libgettextpo-0.18.3.2-1-mingw32-dll-0.tar.xz
#libiconv-1.14-3-mingw32-dll.tar.lzma
#libintl-0.18.3.2-1-mingw32-dll-8.tar.xz
#mingwrt-3.21-mingw32-dev.tar.xz
#mingwrt-3.21-mingw32-dll.tar.xz
#mpc-1.0.1-2-mingw32-dev.tar.lzma
#mpc-1.0.1-2-mingw32-dll.tar.lzma
#mpfr-3.1.2-2-mingw32-dev.tar.lzma
#mpfr-3.1.2-2-mingw32-dll.tar.lzma
#pthreads-w32-2.9.1-1-mingw32-dev.tar.lzma
#pthreads-w32-2.9.1-1-mingw32-dll.tar.lzma
#w32api-3.17-2-mingw32-dev.tar.lzma
#zlib-1.2.8-1-mingw32-dev.tar.lzma
#zlib-1.2.8-1-mingw32-dll.tar.lzma
c = runner.c_drive(32)
mingout = os.path.join(c, 'MinGW')
os.mkdir(mingout)
mingin = os.path.join(self.installers, 'mingw')
for f in os.listdir(mingin):
cmd = ['tar', '-C', mingout, '-xf', os.path.join(mingin, f)]
subprocess.check_call(cmd, stdout=runner.logfd, stderr=runner.logfd)
#Putting in python scripts dir even though direct call to f2py fails
#also apparently the case matters sometimes.
runner.reginsert('[HKEY_CURRENT_USER\Environment]\n'
'"PATH"="C:\\\\MinGW\\\\bin;C:\\\\Python27;'
'C:\\\\Python27\\\\Scripts"\n')
runner.reginsert('[HKEY_CURRENT_USER\Environment]\n'
'"Path"="C:\\\\MinGW\\\\bin;C:\\\\Python27;'
'C:\\\\Python27\\\\Scripts"\n')
#f2py dies horribly in link if dll's aren't in same place as cc1.exe, as.exe
#even though same gcc command line works fine if run by hand.
for d in ('libgcc_s_dw2-1.dll', 'libgmp-10.dll', 'libmpc-3.dll',
'libmpfr-4.dll', 'libintl-8.dll', 'libiconv-2.dll',
'zlib1.dll'):
shutil.copy(os.path.join(mingout, 'bin', d),
os.path.join(mingout, 'libexec', 'gcc', 'mingw32',
'4.8.1'))
shutil.copy(os.path.join(mingout, 'bin', d),
os.path.join(mingout, 'mingw32', 'bin'))
spacepybuild = os.path.join(c, 'spacepy')
spzip = sorted(glob.glob(os.path.join(self.installers, 'spacepy-*.zip')))[-1]
subprocess.check_call(['unzip', spzip, '-d', spacepybuild],
stdout=runner.logfd, stderr=runner.logfd)
spacepybuild = os.path.join(spacepybuild,
os.path.basename(spzip)[:-4])
#Need to specify a compiler, or do the distutils.cfg?
#The f2py thing is weird. The script being on the path is
#insufficient for the call in wine, but seemed to work on the
#XP VM. This should be fixed in the installer.
runner.run(['python', 'setup.py', 'bdist_wininst',
'--f2py=python C:\\\\Python27\\\\Scripts\\\\f2py.py'],
bits=32, cwd=spacepybuild)
#Now install it, grab the omni database, run unit tests
spinstaller = os.path.basename(
sorted(glob.glob(os.path.join(spacepybuild, 'dist', 'spacepy-*.exe')))[-1])
runner.run([spinstaller, '/s'], bits=32,
cwd=os.path.join(spacepybuild, 'dist'))
#Force the directory for the ".spacepy" file, so can populate it w/OMNI, etc.
runner.env[32]['SPACEPY'] = "C:\\spacepy_user"
dotspacepy = os.path.join(c, 'spacepy_user', '.spacepy')
os.makedirs(dotspacepy)
import spacepy #this is in the BUILD environment, not the target!
shutil.copytree(os.path.join(spacepy.DOT_FLN, 'data'),
os.path.join(dotspacepy, 'data'))
retval = runner.run(['python', 'test_spacepy.py'], bits=32,
cwd=os.path.join(spacepybuild, 'tests'))[0]
if retval:
print('Unit tests failed.')
runner.finish(False)
else:
print('{0} built and tested.'.format(spinstaller))
os.copy(os.path.join(spacepybuild, 'dist', spinstaller),
self.installers)
runner.finish(True)
def main(args):
"""
Simple function that looks at the arguments passed, checks to make sure
everything expected exists, and then defines a validation data set for
later processing by TrainTiramisu.py and TestTiramisu.py
"""
args.rootDir = os.path.normpath(args.rootDir)
args.outputDir = os.path.normpath(args.outputDir)
# ensuring all expected files and directories exist
if not os.path.exists(args.rootDir):
raise Exception("ERROR: The dir '" + args.rootDir + "' doesn't exist")
if not os.path.exists(args.rootDir + "/data"):
raise Exception("ERROR: The dir '" + args.rootDir +
"/data' doesn't exist")
if not os.path.exists(args.rootDir + "/masks"):
raise Exception("ERROR: The dir '"+args.rootDir+"/masks' doesn't " + \
"exist")
if not os.path.exists(args.rootDir + "/test.txt"):
raise Exception("ERROR: The file '"+args.rootDir+"/test.txt' " + \
"doesn't exist")
if not os.path.exists(args.rootDir + "/train.txt"):
raise Exception("ERROR: The dir '"+args.rootDir+"/train.txt' "+ \
"doesn't exist")
# Make all output directories if needed
if not os.path.exists(args.outputDir):
os.mkdir(args.outputDir)
if not os.path.exists(args.outputDir + "/test"):
os.mkdir(args.outputDir + "/test")
if not os.path.exists(args.outputDir + "/test/data"):
os.mkdir(args.outputDir + "/test/data")
if not os.path.exists(args.outputDir + "/validate"):
os.mkdir(args.outputDir + "/validate")
if not os.path.exists(args.outputDir + "/validate/data"):
os.mkdir(args.outputDir + "/validate/data")
if not os.path.exists(args.outputDir + "/validate/masks"):
os.mkdir(args.outputDir + "/validate/masks")
if not os.path.exists(args.outputDir + "/train"):
os.mkdir(args.outputDir + "/train")
if not os.path.exists(args.outputDir + "/train/data"):
os.mkdir(args.outputDir + "/train/data")
if not os.path.exists(args.outputDir + "/train/masks"):
os.mkdir(args.outputDir + "/train/masks")
# Read in test and train files
testList = [line.rstrip('\n') for line in open(args.rootDir + "/test.txt")]
trainList = [
line.rstrip('\n') for line in open(args.rootDir + "/train.txt")
]
# Randomly suffle the train list
random.seed(args.randSeed)
random.shuffle(trainList)
# Copy over all test data
for name in testList:
print("test: " + name)
copy_tree(args.rootDir + "/data/" + name,
args.outputDir + "/test/data/" + name)
# Copy over validate data
for name in trainList[:min(args.validNum, len(trainList))]:
print("validate: " + name)
copy_tree(args.rootDir+"/data/"+name,args.outputDir+ \
"/validate/data/"+name)
os.copy(args.rootDir+"/masks/"+name+".png",args.outputDir+ \
"/validate/masks/"+name+".png")
# Copy remaining data to train directory
for name in trainList[args.validNum:]:
print("train: " + name)
copy_tree(args.rootDir+"/data/"+name,args.outputDir+ \
"/train/data/"+name)
os.copy(args.rootDir+"/masks/"+name+".png",args.outputDir+ \
"/train/masks/"+name+".png")
def makecolorimage(self):
"""Make color image (in sections)"""
if (self.stampsize == self.samplesize == 0) and self.testfirst:
# Already did the full image!
print 'Full size image already made.'
imfile = self.testimages[-1]
outfile = join(self.outdir, self.outfile)
if self.deletetests:
print 'Renaming to', outfile
os.rename(imfile, outfile)
else:
print 'Copying to', outfile
os.copy(imfile, outfile)
imfull = Image.open(outfile)
return imfull
# Clean up: Delete test images
if self.deletetests:
for testimage in self.testimages:
if exists(testimage):
os.remove(testimage)
dx = dy = self.stampsize
if dx * dy == 0:
dx = dy = self.maxstampsize
imfull = Image.new(self.mode, (self.nx, self.ny))
if self.mode == 'RGB':
if self.verbose:
print 'Making full color image, one stamp (section) at a time...'
elif self.mode == 'L':
if self.verbose:
print 'Making full grayscale image, one stamp (section) at a time...'
for yo in range(self.ylo, self.yhi, dy):
dy1 = min([dy, self.yhi - yo])
for xo in range(self.xlo, self.xhi, dx):
dx1 = min([dx, self.xhi - xo])
if self.verbose:
print '%5d, %5d / (%d x %d)' % (xo, yo, self.nx, self.ny)
limits = yo, yo + dy, xo, xo + dx
stamps = self.loadstamps(limits)
im = RGBscale2im(stamps, self.levdict, self.noiselums,
self.colorsatfac, self.mode)
if self.show and self.showstamps:
im.show()
imfull.paste(im,
(xo, self.ny - yo - dy1, xo + dx1, self.ny - yo))
outfile = join(self.outdir, self.outfile)
if self.legend:
self.addlegend(im=imfull)
else:
if self.verbose:
print 'Saving', outfile, '...'
imfull.save(outfile)
if self.show:
self.showimage(outfile, Image)
return imfull
最后，看一下os模块的总结
***************************************************************
'''
# 当前执行这个python文件的工作目录相关的工作路径
os.getcwd() #(pwd) 获取当前工作目录，即当前python脚本工作的目录路径 **
os.chdir("dirname") #(cd ) 改变当前脚本工作目录；相当于shell下cd **
os.curdir 返回当前目录: ('.') **
os.pardir 获取当前目录的父目录字符串名：('..') **
# 和文件夹相关
os.makedirs('dirname1/dirname2') #(mkdir -p ) 可生成多层递归目录 ***
os.removedirs('dirname1') #(rm -r) 若目录为空，则删除，并递归到上一级目录，如若也为空，则删除，依此类推 ***
os.mkdir('dirname') #(mkdir) 生成单级目录；相当于shell中mkdir dirname ***
os.rmdir('dirname') #(rmdir) 删除单级空目录，若目录不为空则无法删除，报错；相当于shell中rmdir dirname ***
os.listdir('dirname') #(ls -al) 列出指定目录下的所有文件和子目录，包括隐藏文件，并以列表方式打印 **
os.copy('olddir','newdir/newfile') #(cp olddir newdir) #oldfile只能是文件夹，newfile可以是文件，也可以是目标目录
# 和文件相关
os.remove() 删除一个文件 ***
os.rename("oldname","newname") 重命名文件/目录 ***
os.stat('path/filename') 获取文件/目录信息 **
shutil.copytree("olddir","newdir") #复制文件夹： olddir和newdir都只能是目录，且newdir必须不存在
shutil.move("oldpos","newpos") #移动文件（目录）
# 和操作系统差异相关
os.sep #输出操作系统特定的路径分隔符，win下为"\\",Linux下为"/" *
os.linesep #输出当前平台使用的行终止符，win下为"\t\n",Linux下为"\n" *
os.pathsep #输出用于分割文件路径的字符串 win下为;,Linux下为: *
os.name #输出字符串指示当前使用平台。win->'nt'; Linux->'posix' *和执行系统命令相关
# os.system("bash command") #运行shell命令，直接显示 **
# os.popen("bash command").read() # 运行shell命令，获取执行结果 **
os.environ 获取系统环境变量 **
def QMRun(inp, program=setting.qmcode, **kwargs):
"""
interface to run qmcode with written inp files. It manages to start
qmcode in **cwd**, write to output, collect result,
clean up scratch, tmp files according to setup.
However, each code require different implementation.
input:
inp(str): path to input file
code(str): qmcode to run, default set to setting.qmcode
kwargs (optional):
threads=n(int): number of threads per job
bigmem=Boolean: big memory request, implemented for CPMD and others
CPMD:
save_restart=Boolean
scr=/path/to/scratch
"""
if 'threads' in kwargs:
_threads = kwargs['threads']
else:
_threads = 1
if 'bigmem' in kwargs:
_bigmem = kwargs['bigmem']
else:
if setting.memory > 16:
_bigmem = True
else:
_bigmem = False
if 'omp' in kwargs:
omp = kwargs['omp']
else:
omp = 1
if 'save_restart' in kwargs:
_save_restart = kwargs['save_restart']
else:
_save_restart = False
if 'save_density' in kwargs:
_save_density = kwargs['save_density']
else:
_save_density = False
if 'chdir' in kwargs and kwargs['chdir']:
cwd = os.getcwd()
os.chdir(inp)
###########################################
# SYSTEM CALL SUBPROCESS: Running mpi job #
###########################################
def compute(exestr, outpath, threads_per_job, **kwargs):
"""
initiate a single MPI job, wait for it, and write to output
"""
os.environ["OMP_NUM_THREADS"] = str(omp)
outfile = open(outpath, "w")
if threads_per_job > 1:
mpi_cmd = "%s %d"% (setting.mpistr, threads_per_job)
for mpi_flag in setting.mpi_flags:
if mpi_flag == '--cpus-per-proc':
flag = mpi_flag + ' ' + str(threads_per_job)
else:
flag = mpi_flag
mpi_cmd = mpi_cmd + ' ' + flag
cmd = mpi_cmd + ' ' + exestr
else:
cmd = exestr
ut.progress('QMInp.run', 'running job with command: %s\n' % cmd)
run = sp.Popen(cmd, shell=True, stdout=outfile)
# wait each mpijob to finish before lauching another
# otherwise all mpijobs will be launched simutaniously
run.wait()
outfile.close()
########## END OF SYSTEM CALL ##########
#######################
# CPMD IMPLEMENTATION #
#######################
if program.lower() == 'cpmd':
if 'exe' in kwargs:
exe = kwargs['exe']
else:
exe = setting.cpmd_exe
if 'scr' in kwargs and kwargs['scr']:
scrdir = kwargs['scr']
ut.delete(inpname, 'FILEPATH', 2)
ut.insert(inpname, 'CPMD', ' FILEPATH\n %s' % scrdir)
inp_list = sorted(glob.glob('*.inp'))
for job in inp_list:
out = os.path.splitext(job)[0] + '.out'
exestr = "%s %s" % (exe, job)
compute(exestr, out, _threads)
rst_list = sorted(glob.glob('RESTART.*'))
if rst_list:
rst_n = rst_list[-1]
if os.path.exists('RESTART'):
os.remove('RESTART')
os.link(rst_n, 'RESTART')
final_out = re.sub(r'_[0-9][0-9].out$', '.out', out)
if final_out != out:
os.copy(out, final_out)
# clean up files
files = glob.glob('*')
tmp = filter(\
lambda x: '.out' not in x \
and '.inp' not in x\
and '.psp' not in x\
and '.xyz' not in x\
and 'KPTS_GENERATION' not in x\
and 'RESTART' not in x\
and 'DENSITY' not in x\
and 'SPINDEN' not in x, files
)
for f in tmp: os.remove(f)
if not _save_restart:
rst_list = glob.glob("RESTART*")
for rfile in rst_list:
os.remove(rfile)
densities = glob.glob('*DEN*')
for i in range(len(densities)):
exe = setting.cpmd_cpmd2cube_exe
log_name = densities[i] + '_%02d.log' % i
log = open(log_name, 'w')
run = sp.Popen("%s -fullmesh %s" % (exe, densities[i]),
shell=True,
stdout=log)
run.wait()
log.close()
if os.path.exists(out):
qio_out = qio.QMOut(out, program='cpmd')
else:
qio_out = None
#######################
# VASP IMPLEMENTATION #
#######################
elif program.lower() == 'vasp':
if 'exe' in kwargs:
exestr = kwargs['exe']
else:
exestr = setting.vasp_exe
qmoutput = inp + '.out'
qmlog = inp + '.log'
compute(exestr, qmoutput, _threads)
qio_out = qio.QMOut('vasprun.xml', program='vasp')
if not _save_restart:
try:
os.remove('WAVECAR')
except:
pass
try:
os.remove('POTCAR')
except:
pass
os.rename(qmoutput, qmlog)
shutil.copyfile('vasprun.xml', qmoutput)
#########################
# NWChem IMPLEMENTATION #
#########################
elif program.lower() == 'nwchem':
if 'exe' in kwargs:
exe = kwargs['exe']
else:
exe = setting.nwchem_exe
exestr = "%s %s" % (exe, inp)
qmoutput = os.path.splitext(inp)[0] + '.out'
compute(exestr, qmoutput, _threads)
qio_out = qio.QMOut(qmoutput, program='nwchem')
files = glob.glob('*.*')
tmp = filter(\
lambda x: '.out' not in x \
and '.inp' not in x\
and '.cube' not in x\
and '.movecs' not in x, files
)
for f in tmp: os.remove(f)
movecs = glob.glob('*.movecs')
for f in movecs:
exe = setting.nwchem_mov2asc_exe
nb = qio_out.n_basis
out = re.sub('\.movecs','.modat',f)
exestr = "%s %d %s %s" % (exe, nb, f, out)
run = sp.Popen(exestr, shell=True)
run.wait()
qio_out.getMO(out)
if not _save_restart:
os.remove(f)
#########################
# BigDFT IMPLEMENTATION #
#########################
elif program.lower() == 'bigdft':
if 'exe' in kwargs:
exe = kwargs['exe']
else:
exe = setting.bigdft_exe
inp = os.path.splitext(inp)[0]
exestr = "%s %s" % (exe, inp)
qmoutput = inp + '.out'
compute(exestr, qmoutput, _threads)
qio_out = qio.QMOut(qmoutput, program='bigdft')
#########################
# Abinit IMPLEMENTATION #
#########################
elif program.lower() == 'abinit':
if 'exe' in kwargs:
exe = kwargs['exe']
else:
exe = setting.abinit_exe
inp = os.path.splitext(inp)[0]
exestr = "%s < %s" % (exe, inp + '.files')
qmlog = inp + '.log'
qmoutput = inp + '.out'
compute(exestr, qmlog, _threads)
qio_out = qio.QMOut(qmoutput, program='abinit')
# clean up files
files = glob.glob('*')
tmp = filter(\
lambda x: '.out' not in x \
and '.log' not in x\
and '.inp' not in x\
and '.files' not in x\
and 'psp' not in x\
and '_EIG' not in x\
and '_WFK' not in x\
and '_DEN' not in x, files
)
for f in tmp: os.remove(f)
if not _save_restart:
for rst in glob.glob('*_WFK'):
os.remove(rst)
if not _save_density:
for den in glob.glob('*_DEN'):
os.remove(den)
#############################
# Gaussian09 IMPLEMENTATION #
#############################
elif program.lower() == 'gaussian':
if 'exe' in kwargs:
exe = kwargs['exe']
else:
exe = setting.gaussian_exe
exestr = "%s %s" % (exe, inp)
qmoutput = os.path.splitext(inp)[0] + '.out'
qmlog = os.path.splitext(inp)[0] + '.log'
compute(exestr, qmoutput, _threads)
os.rename(qmlog, qmoutput)
chks = glob.glob('*.chk')
for chk in chks:
exe = setting.gaussian_formchk_exe
exestr = "%s %s" % (exe, chk)
run = sp.Popen(exestr, shell=True)
run.wait()
qio_out = qio.QMOut(qmoutput, program='gaussian')
##################################
# QuantumESPRESSO IMPLEMENTATION #
##################################
elif program.lower() == 'espresso':
if 'exe' in kwargs:
exe = kwargs['exe']
else:
exe = setting.espresso_exe
inp_list = sorted(glob.glob('*.inp'))
for job in inp_list:
out = os.path.splitext(job)[0] + '.out'
exestr = "%s < %s" % (exe, job)
compute(exestr, out, _threads)
qio_out = qio.QMOut(out, program='espresso')
if not _save_restart:
rst_list = glob.glob("*.wfc*")
rst_list.extend(glob.glob("*.restart_*"))
else:
rst_list = []
for r in rst_list:
os.remove(r)
#########################
# GAMESS IMPLEMENTATION #
#########################
elif program.lower() == 'gamess':
ut.exit("ERROR! program '%s' not implemented" % program)
# and others... #
else:
ut.exit("ERROR! program '%s' not recognized" % program)
if 'cwd' in locals():
os.chdir(cwd)
return qio_out
os.mkdir(work_dir)
JOB_NAME=name
RANDOM_SEED=random_seed
EVENT_NUMBER=event_number
ENERGY=energy
RUN_NUMBER=run_number
#Do substitution for required value
configure(template_dir+"/template_sim.cfg", work_dir+"/"+name+"_sim.cfg", JOB_NAME, ENERGY, RANDOM_SEED, EVENT_NUMBER, RUN_NUMBER)
configure(template_dir+"/template_rec.cfg", work_dir+"/"+name+"_rec.cfg", JOB_NAME, ENERGY, RANDOM_SEED, EVENT_NUMBER, RUN_NUMBER)
configure(template_dir+"/template_ana.cfg", work_dir+"/"+name+"_ana.cfg", JOB_NAME, ENERGY, RANDOM_SEED, EVENT_NUMBER, RUN_NUMBER)
configure(template_dir+"/common_sim.cfg", work_dir+"/common_sim.cfg", JOB_NAME, ENERGY, RANDOM_SEED, EVENT_NUMBER, RUN_NUMBER)
configure(template_dir+"/pbsjobs", work_dir+"/pbsjobs", JOB_NAME, ENERGY, RANDOM_SEED, EVENT_NUMBER, RUN_NUMBER)
print "Starting the "+PBS_QUEUE+" job " + name
#os.system("qsub "+work_dir+"/pbsjobs")
#end of do_mc function
def copy_template(sdir, tdir, prefix)
os.copy(sdir+"/template_sim.cfg", tdir+"/"+prefix+"_sim.cfg")
os.copy(sdir+"/template_rec.cfg", tdir+"/"+prefix+"_rec.cfg")
os.copy(sdir+"/template_ana.cfg", tdir+"/"+prefix+"_ana.cfg")
N=50000
jobs = 20
#for job in range(1,jobs+1):
# do_mc(1777, N, job, "-20334, -20335, -20339, -20336")
print rundir
print scriptdir
print templatedir
#XP VM. This should be fixed in the installer.
runner.run([
'python', 'setup.py', 'bdist_wininst',
'--f2py=python C:\\\\Python27\\\\Scripts\\\\f2py.py'
],
bits=32,
cwd=spacepybuild)
#Now install it, grab the omni database, run unit tests
spinstaller = os.path.basename(
sorted(glob.glob(os.path.join(spacepybuild, 'dist', 'spacepy-*.exe')))[-1])
runner.run([spinstaller, '/s'],
bits=32,
cwd=os.path.join(spacepybuild, 'dist'))
#Force the directory for the ".spacepy" file, so can populate it w/OMNI, etc.
runner.env[32]['SPACEPY'] = "C:\\spacepy_user"
dotspacepy = os.path.join(c, 'spacepy_user', '.spacepy')
os.makedirs(dotspacepy)
import spacepy #this is in the BUILD environment, not the target!
shutil.copytree(os.path.join(spacepy.DOT_FLN, 'data'),
os.path.join(dotspacepy, 'data'))
retval = runner.run(['python', 'test_spacepy.py'],
bits=32,
cwd=os.path.join(spacepybuild, 'tests'))[0]
if retval:
print('Unit tests failed.')
runner.finish(False)
else:
print('{0} built and tested.'.format(spinstaller))
os.copy(os.path.join(spacepybuild, 'dist', spinstaller), installers)
runner.finish(True)
def copy_file(self,src,dst):
try:
os.copy(src, dst)
except IOError, e:
logging.error(e)
def QMRun(inp, program=setting.qmcode, **kwargs):
"""
interface to run qmcode with written inp files. It manages to start
qmcode in **cwd**, write to output, collect result,
clean up scratch, tmp files according to setup.
However, each code require different implementation.
input:
inp(str): path to input file
code(str): qmcode to run, default set to setting.qmcode
kwargs (optional):
threads=n(int): number of threads per job
bigmem=Boolean: big memory request, implemented for CPMD and others
CPMD:
save_restart=Boolean
scr=/path/to/scratch
"""
if 'threads' in kwargs:
_threads = kwargs['threads']
else:
_threads = 1
if 'bigmem' in kwargs:
_bigmem = kwargs['bigmem']
else:
if setting.memory > 16:
_bigmem = True
else:
_bigmem = False
if 'omp' in kwargs:
omp = kwargs['omp']
else:
omp = 1
if 'save_restart' in kwargs:
_save_restart = kwargs['save_restart']
else:
_save_restart = False
if 'save_density' in kwargs:
_save_density = kwargs['save_density']
else:
_save_density = False
if 'chdir' in kwargs and kwargs['chdir']\
or 'run_dir' in kwargs and kwargs['run_dir']:
dest_dir = os.path.splitext(inp)[0]
cwd = os.getcwd()
os.chdir(dest_dir)
###########################################
# SYSTEM CALL SUBPROCESS: Running mpi job #
###########################################
def compute(exestr, outpath, threads_per_job, **kwargs):
"""
initiate a single MPI job, wait for it, and write to output
"""
os.environ["OMP_NUM_THREADS"] = str(omp)
outfile = open(outpath, "w")
if threads_per_job > 1:
mpi_cmd = "%s %d" % (setting.mpistr, threads_per_job)
for mpi_flag in setting.mpi_flags:
if mpi_flag == '--cpus-per-proc':
flag = mpi_flag + ' ' + str(threads_per_job)
else:
flag = mpi_flag
mpi_cmd = mpi_cmd + ' ' + flag
cmd = mpi_cmd + ' ' + exestr
else:
cmd = exestr
ut.progress('QMInp.run', 'running job with command: %s\n' % cmd)
if 'env' in kwargs:
run = sp.Popen(cmd, shell=True, stdout=outfile, env=kwargs['env'])
else:
run = sp.Popen(cmd, shell=True, stdout=outfile)
# wait each mpijob to finish before lauching another
# otherwise all mpijobs will be launched simutaniously
run.wait()
outfile.close()
########## END OF SYSTEM CALL ##########
#################################
# SYSTEM CALL for post analysis #
#################################
def sys_run(cmd_str, log_file=None):
if log_file:
log = open(log_file, 'w')
try:
qtk.progress("QMRun", cmd_str)
if log_file:
run = sp.Popen(cmd_str, shell=True, stdout=log)
else:
run = sp.Popen(cmd_str, shell=True)
run.wait()
except Exception as err:
qtk.warning('%s failed with error: %s' % (cmd_str, err))
if log_file:
log.close()
########## END OF SYSTEM CALL ##########
#######################
# CPMD IMPLEMENTATION #
#######################
if program.lower() == 'cpmd':
if 'exe' in kwargs:
exe = kwargs['exe']
else:
exe = setting.cpmd_exe
if 'scr' in kwargs and kwargs['scr']:
scrdir = kwargs['scr']
ut.delete(inpname, 'FILEPATH', 2)
ut.insert(inpname, 'CPMD', ' FILEPATH\n %s' % scrdir)
inp_list = sorted(glob.glob('*.inp'))
for job in inp_list:
out = os.path.splitext(job)[0] + '.out'
exestr = "%s %s" % (exe, job)
compute(exestr, out, _threads)
if (len(inp_list) > 1):
rst_list = sorted(glob.glob('RESTART.*'))
if rst_list:
rst_n = rst_list[-1]
if os.path.exists('RESTART'):
os.remove('RESTART')
os.link(rst_n, 'RESTART')
final_out = re.sub(r'_[0-9][0-9].out$', '.out', out)
if final_out != out:
os.copy(out, final_out)
# clean up files
files = sorted(glob.glob('*'))
tmp = filter(\
lambda x: '.out' not in x \
and '.inp' not in x\
and '.psp' not in x\
and '.xyz' not in x\
and 'KPTS_GENERATION' not in x\
and 'RESTART' not in x\
and 'DENSITY' not in x\
and 'SPINDEN' not in x, files
)
for f in tmp:
os.remove(f)
if not _save_restart:
rst_list = sorted(glob.glob("RESTART*"))
for rfile in rst_list:
os.remove(rfile)
densities = sorted(glob.glob('*DEN*'))
for i in range(len(densities)):
exe = setting.cpmd_cpmd2cube_exe
cmd = "%s -fullmesh %s" % (exe, densities[i])
log_name = densities[i] + '_%02d.log' % i
sys_run(cmd, log_name)
#log = open(log_name, 'w')
#try:
# run = sp.Popen("%s -fullmesh %s" % (exe, densities[i]),
# shell=True,
# stdout=log)
# run.wait()
#except Exception as err:
# qtk.warning('%s failed with error: %s' % (exe, err))
#log.close()
if os.path.exists(out):
qio_out = qio.QMOut(out, program='cpmd')
else:
qio_out = None
#######################
# VASP IMPLEMENTATION #
#######################
elif program.lower() == 'vasp':
if 'exe' in kwargs:
exestr = kwargs['exe']
else:
exestr = setting.vasp_exe
qmoutput = inp + '.out'
qmlog = inp + '.log'
compute(exestr, qmoutput, _threads)
qio_out = qio.QMOut('vasprun.xml', program='vasp')
if not _save_restart:
try:
os.remove('WAVECAR')
except:
pass
try:
os.remove('POTCAR')
except:
pass
os.rename(qmoutput, qmlog)
shutil.copyfile('vasprun.xml', qmoutput)
#########################
# NWChem IMPLEMENTATION #
#########################
elif program.lower() == 'nwchem':
if 'exe' in kwargs:
exe = kwargs['exe']
else:
exe = setting.nwchem_exe
exestr = "%s %s" % (exe, inp)
qmoutput = os.path.splitext(inp)[0] + '.out'
compute(exestr, qmoutput, _threads)
qio_out = qio.QMOut(qmoutput, program='nwchem')
files = sorted(glob.glob('*.*'))
tmp = filter(\
lambda x: '.out' not in x \
and '.inp' not in x\
and '.cube' not in x\
and '.movecs' not in x, files
)
for f in tmp:
os.remove(f)
movecs = sorted(glob.glob('*.movecs'))
for f in movecs:
exe = setting.nwchem_mov2asc_exe
nb = qio_out.n_basis
out = re.sub('\.movecs', '.modat', f)
exestr = "%s %d %s %s" % (exe, nb, f, out)
sys_run(exestr)
#try:
# run = sp.Popen(exestr, shell=True)
# run.wait()
# qio_out.getMO(out)
#except Exception as err:
# qtk.warning('%s failed with error: %s' % (exe, err))
if not _save_restart:
os.remove(f)
#########################
# BigDFT IMPLEMENTATION #
#########################
elif program.lower() == 'bigdft':
if 'exe' in kwargs:
exe = kwargs['exe']
else:
exe = setting.bigdft_exe
env = os.environ.copy()
inp = os.path.splitext(inp)[0]
exestr = "%s %s" % (exe, inp)
qmoutput = inp + '.out'
qmlog = 'log-%s.yaml' % inp
compute(exestr, qmoutput, _threads, env=env)
qio_out = qio.QMOut(qmlog, program='bigdft')
#########################
# Abinit IMPLEMENTATION #
#########################
elif program.lower() == 'abinit':
if 'exe' in kwargs:
exe = kwargs['exe']
else:
exe = setting.abinit_exe
inp = os.path.splitext(inp)[0]
exestr = "%s < %s" % (exe, inp + '.files')
qmlog = inp + '.log'
qmoutput = inp + '.out'
compute(exestr, qmlog, _threads)
# unfold bandstructure
if 'unfold' in kwargs and kwargs['unfold']:
folds = kwargs['unfold']
exe = setting.abinit_f2b_exe
wfk_list = sorted(glob.glob("*_WFK"))
if len(wfk_list) > 0:
wfk = wfk_list[-1]
qtk.progress("QMRun",
"linking wfk file %s to unfold_WFK" % wfk)
os.link(wfk, 'unfold_WFK')
wfk_root = wfk.replace('_WFK', '')
log_name = '%s_f2b.log' % wfk_root
fold_str = [str(f) for f in folds]
cmd_str = "%s unfold_WFK %s" % (exe, ':'.join(fold_str))
sys_run(cmd_str, log_name)
qtk.progress("QMRun", "Done, remove unfold_WFK")
os.remove('unfold_WFK')
#try:
# qtk.progress("QMRun", cmd_str)
# run = sp.Popen(cmd_str, shell=True, stdout=log)
# run.wait()
#except Exception as err:
# qtk.warning('%s failed with error: %s' % (exe, err))
#log.close()
else:
qtk.warning('unfolding but no wavefunction file found...')
if 'unfold_cleanup' in kwargs and kwargs['unfold_cleanup']:
qtk.progress("QMRun", "deleting all WFK files")
for wfk in sorted(glob.glob("*_WFK")):
os.remove(wfk)
# clean up files
files = sorted(glob.glob('*'))
tmp = filter(\
lambda x: '.out' not in x \
and '.f2b' not in x\
and '.log' not in x\
and '.inp' not in x\
and '.files' not in x\
and 'psp' not in x\
and '_EIG' not in x\
and '_WFK' not in x\
and '_DOS' not in x\
and '_DEN' not in x, files
)
for f in tmp:
os.remove(f)
if not _save_restart:
for rst in sorted(glob.glob('*_WFK')):
os.remove(rst)
if not _save_density:
for den in sorted(glob.glob('*_DEN')):
os.remove(den)
densities = sorted(glob.glob('*_DEN'))
if len(densities) > 0:
i = len(densities) - 1
exe = setting.abinit_cut3d_exe
#den_inp = densities[i] + '.cov'
den_inp = densities[-1] + '.cov'
cube_file = inp + '.cube'
den_inp_file = open(den_inp, 'wb')
den_inp_file.write("%s\n14\n%s\n0" % (densities[i], cube_file))
den_inp_file.close()
log_name = densities[i] + '.log'
cmd = "%s < %s" % (exe, den_inp)
sys_run(cmd, log_name)
#log = open(log_name, 'w')
#try:
# run = sp.Popen("%s < %s" % (exe, den_inp),
# shell=True,
# stdout=log)
# run.wait()
#except Exception as err:
# qtk.warning('%s failed with error: %s' % (exe, err))
#log.close()
qio_out = qtk.QMOut(qmoutput, program='abinit')
#############################
# Gaussian09 IMPLEMENTATION #
#############################
elif program.lower() == 'gaussian':
if 'exe' in kwargs:
exe = kwargs['exe']
else:
exe = setting.gaussian_exe
exestr = "%s %s" % (exe, inp)
qmoutput = os.path.splitext(inp)[0] + '.out'
qmlog = os.path.splitext(inp)[0] + '.log'
compute(exestr, qmoutput, _threads)
os.rename(qmlog, qmoutput)
chks = sorted(glob.glob('*.chk'))
for chk in chks:
exe = setting.gaussian_formchk_exe
exestr = "%s %s" % (exe, chk)
sys_run(exestr)
#run = sp.Popen(exestr, shell=True)
#run.wait()
if _save_density:
fchk = sorted(glob.glob("*.fchk"))[-1]
q = qtk.CUBE(fchk)
qio_out = qio.QMOut(qmoutput, program='gaussian')
##################################
# QuantumESPRESSO IMPLEMENTATION #
##################################
elif program.lower() == 'espresso':
if 'exe' in kwargs:
exe = kwargs['exe']
else:
exe = setting.espresso_exe
inp_list = sorted(glob.glob('*.inp'))
for job in inp_list:
out = os.path.splitext(job)[0] + '.out'
exestr = "%s < %s" % (exe, job)
compute(exestr, out, _threads)
qio_out = qio.QMOut(out, program='espresso')
if not _save_restart:
rst_list = sorted(glob.glob("*.wfc*"))
rst_list.extend(sorted(glob.glob("*.restart_*")))
else:
rst_list = []
for r in rst_list:
os.remove(r)
#########################
# GAMESS IMPLEMENTATION #
#########################
elif program.lower() == 'gamess':
ut.exit("ERROR! program '%s' not implemented" % program)
# and others... #
else:
ut.exit("ERROR! program '%s' not recognized" % program)
if 'cwd' in locals():
os.chdir(cwd)
qio_out.path = inp
return qio_out
tracks = test_db.get_tracks() tracks.sort() assert tracks == music_files test_db.set_uploaded(music_files[0]) tracks = test_db.get_tracks() tracks.sort() assert tracks == music_files[1:] del test_db test_db = db.DB(db_path) test_db.set_uploaded(music_files[1]) tracks = test_db.get_tracks() tracks.sort() assert tracks == music_files[2:] assert test_db.total_uploaded_tracks() == 2 os.mkdir(db_dir_path) for database in databases: os.copy(database, db_dir_path) try: test_new_db() finally: for dbname in os.listdir(db_dir_path): filename = os.path.join(db_dir_path, dbname) os.remove(filename) os.rmdir(db_dir_path)