def deletederived():
"""
delete derived .r files
"""
shutil.rmtree(
util.getdirectory("derived", branch, tracker, geneeff, stimno))
def logpathfrommonth(branch, tracker, month, year):
"""
input: branch, tracker, month, year
output: path to log file dir for given year and month
"""
return os.path.join(util.getdirectory("logs", branch, tracker), "twocof",
"%4d" % year, "%02d" % month)
def outputfilepaths(self):
"""
output file paths
"""
basedir = util.getdirectory("analysis", self.branch, self.tracker,
self.geneeff, self.stimno)
return [os.path.join(basedir, "%s.periodic.txt" % self.stem)]
def deleteprocessingstatus():
"""
delete the processing status file
"""
statusdir = util.getdirectory(const.processingstatuslocation, branch,
tracker, geneeff, stimno)
statuspath = os.path.join(statusdir, const.processingstatusfilename)
if os.path.exists(statuspath):
os.remove(statuspath)
def outputfilepaths(self):
"""
output file paths
"""
basedir = util.getdirectory(self.outputlocation, self.branch,
self.tracker, self.geneeff, self.stimno)
return [
os.path.join(
basedir,
"%s.combo1-ave-err%s" % (self.stem, const.plotextension))
]
def outputfilepaths(self):
"""
output file paths
"""
basedir = util.getdirectory("analysis", self.branch, self.tracker,
self.geneeff, self.stimno)
self.xsumspath = os.path.join(basedir,
"%s.xmotion-sums.txt" % self.stem)
self.ysumspath = os.path.join(basedir,
"%s.ymotion-sums.txt" % self.stem)
return [self.xsumspath, self.ysumspath]
def writeprocessingstatus(branch, tracker, geneeff, stimno):
"""
write a file indicating what's been successfully processed
"""
statusdir = util.getdirectory(const.processingstatuslocation, branch,
tracker, geneeff, stimno)
statuspath = os.path.join(statusdir, const.processingstatusfilename)
data = {}
data["date"] = util.getdatestamp()
data["status"] = "successful"
data["dates processed"] = util.getchoreographydatestamps(
branch, tracker, geneeff, stimno)
f = open(statuspath, 'wt')
json.dump(data, f)
f.close()
def performtask(self):
"""
do it!
input: none
output: results string; should begin with "error" if an
error occurred
"""
# only one file:
outputfilepath = self.outputfilepaths()[0]
if self.overwrite or not self.outputexists():
# this branch should always be taken in the process I
# have planned, but I don't want to rely on that
if not self.checkcreateoutputfolders():
return "error: could not create output folders for PeriodicAnalysis!"
# generate input filename:
sourcedir = util.getdirectory("source", self.branch, self.tracker,
self.geneeff, self.stimno)
filename = "%s.speed.r" % self.stem
sourcefilepath = os.path.join(sourcedir, filename)
td = self.trackdatacache.gettrackdata(sourcefilepath)
# need to pass the intervals to the program, so write out
# to a temporary file:
intervalpath = self.writeintervalfile(td)
# run the command and return its output; note that the
# "self.runcommand" method ought to trap all exceptions,
# so no need to try/except here
execpath = os.path.join(util.getresourcelocation("executable"),
self.executablename)
commandstring = "python %s %s %s %s" % (
execpath, inputfilepath, intervalpath, outputfilepath)
results = self.runcommand(commandstring)
# clean up
os.remove(intervalpath)
return results
def readalldata(branch, targettracker):
"""
read all data from a tracker
input: branch, tracker
output: dict with rawdata[scalar][expt][interval] = stats
"""
rawdata = {}
for scalar in const.aggregatedstatsscalarlist[targettracker]:
rawdata[scalar] = {}
for tracker, geneeff, stimno in util.walktracker([branch, targettracker], "statistics"):
filedir = util.getdirectory("statistics", branch, tracker, geneeff, stimno)
filename = "%s@%s@%s.%s-stats-perobj.txt" % (geneeff, tracker, stimno, scalar)
filepath = os.path.join(filedir, filename)
if os.path.exists(filepath):
# eventually put in try-except with reporting here?
rawdata[scalar][(geneeff, stimno)] = readstatsfile(filepath)
return rawdata
def createdirectories(branch, tracker, clean=True):
"""
create the upper level of output directories;
input: tracker; clean = True means delete existing first
"""
trackerstatspath = util.getdirectory("statistics", branch, tracker)
aggregatedstatspath = os.path.join(trackerstatspath, const.aggregatedstatsdir)
if os.path.exists(aggregatedstatspath):
if clean:
# delete all contents:
shutil.rmtree(aggregatedstatspath)
os.mkdir(aggregatedstatspath)
else:
os.mkdir(aggregatedstatspath)
for scalar in const.aggregatedstatsscalarlist[tracker]:
scalarpath = os.path.join(aggregatedstatspath, scalar)
if not os.path.exists(scalarpath):
os.mkdir(scalarpath)
print "usage: clean.py branch tracker"
sys.exit(2)
branch = sys.argv[1]
tracker = sys.argv[2]
if tracker == "dev":
trackerlist = testtrackers
elif tracker in productiontrackers:
if "--sure" in sys.argv:
trackerlist = [tracker]
else:
print "you must use the --sure flag for production trackers!"
sys.exit(1)
else:
trackerlist = [tracker]
# delete all files in the above directories
for tracker in trackerlist:
for location in locationlist:
folderpath = util.getdirectory(location, branch, tracker)
if os.path.exists(folderpath):
filelist = os.listdir(folderpath)
print "deleting %d items from %s" % (len(filelist), folderpath)
for filename in filelist:
filepath = os.path.join(folderpath, filename)
if os.path.isdir(filepath):
shutil.rmtree(filepath)
else:
os.remove(filepath)
# set the file permissions
os.umask(const.umask)
# read all the data in:
rawdata = readalldata(branch, tracker)
# re-organize:
sliceddata = transformdata(rawdata, tracker)
# set up directories
createdirectories(branch, tracker)
# iterate over scalars and time intervals:
for scalar in sliceddata:
for stimno in sliceddata[scalar]:
for t1, t2 in sliceddata[scalar][stimno]:
filename = filenamepattern % (tracker, scalar, t1, t2)
dirpath = util.getdirectory("statistics", branch,
tracker, const.aggregatedstatsdir)
dirpath = os.path.join(dirpath, scalar, stimno)
if not os.path.exists(dirpath):
os.mkdir(dirpath)
filepath = os.path.join(dirpath, filename)
writeaggregatedstatsfile(filepath, sliceddata[scalar][stimno][(t1, t2)])
# ------------------------- imports -------------------------
import os
import shutil
import sys
from lib import constants as const
from lib import util
# ------------------------- script starts here -------------------------
if __name__ == "__main__":
if len(sys.argv) < 3:
sys.exit("usage: retrack.py oldtracker newtracker (always branch rd)")
oldtracker = sys.argv[1]
newtracker = sys.argv[2]
# new location:
triples = util.walktracker(["rd", newtracker], location="source")
for tr, gene, stim in triples:
basedir = util.getdirectory("source", "rd", tr, gene, stim)
for filename in os.listdir(basedir):
if oldtracker in filename and filename.endswith('.r'):
newfilename = filename.replace(oldtracker, newtracker)
os.rename(os.path.join(basedir, filename), os.path.join(basedir, newfilename))
"t5",
"t6",
]
# ------------------------- script starts here -------------------------
if __name__ == "__main__":
if len(sys.argv) < 5:
print "usage: backup.py branch tracker geneeff stimno"
sys.exit(2)
branch = sys.argv[1]
tracker = sys.argv[2]
geneeff = sys.argv[3]
stimno = sys.argv[4]
if tracker in productiontrackers and "--sure" not in sys.argv:
print "you must use the --sure flag for production trackers!"
sys.exit(1)
for location in locationlist:
oldpath = util.getdirectory(location, branch, tracker, geneeff, stimno)
newpath = oldpath + suffix
if os.path.exists(oldpath):
print "copying %s" % location
shutil.copytree(oldpath, newpath)
else:
print "%s has no files; skipping" % location
default=False,
help="increase level of output")
(options, args) = parser.parse_args()
if len(args) < 3:
parser.print_usage()
sys.exit(2)
tracker = args[0]
geneeff = args[1]
stimno = args[2]
os.umask(const.umask)
# just need one .r file, doesn't matter which:
sourcefilename = "%s@%s@%s.x.r" % (geneeff, tracker, stimno)
sourcedir = util.getdirectory("source", branch, tracker, geneeff, stimno)
sourcepath = os.path.join(sourcedir, sourcefilename)
td = trackdata.readfile(sourcepath)
choredir = util.getdirectory("choreography", branch, tracker, geneeff,
stimno)
for datestamp in td.getdatestamps():
datefolder = os.path.join(choredir, datestamp)
if options.verbose:
print "creating %s" % datefolder
os.makedirs(datefolder)
def performtask(self):
"""
do it!
input: none
output: results string; should begin with "error" if an
error occurred
"""
# only one file:
outputfilepath = self.outputfilepaths()[0]
if self.overwrite or not self.outputexists():
# this branch should always be taken in the process I
# have planned, but I don't want to rely on that
if not self.checkcreateoutputfolders():
return "error: could not create output folders for GenerateAveErrMulti1PlotTask!"
# read data
td = {}
for scalar in self.scalardict:
filename = self.filenamepattern % (self.geneeff, self.tracker,
self.stimno, scalar)
sourcedir = util.getdirectory("source", self.branch,
self.tracker, self.geneeff,
self.stimno)
sourcefilepath = os.path.join(sourcedir, filename)
td[scalar] = self.trackdatacache.gettrackdata(sourcefilepath)
# not very modularized yet...
fig = plt.figure()
axis = fig.add_subplot(5, 1, 1)
axis.set_title(self.stem)
self.setupgridticks(axis, "area", 0.5, majorlabels=False)
self.plotscalar(td, "area", axis, elinewidth=0.5, markersize=2)
axis = fig.add_subplot(5, 1, 2)
self.setupgridticks(axis, "curve", 10, majorlabels=False)
self.plotscalar(td, "curve", axis, elinewidth=0.5, markersize=2)
axis = fig.add_subplot(5, 1, 3)
self.setupgridticks(axis, "dir", 0.5, majorlabels=False)
self.plotscalar(td, "dir", axis, elinewidth=0.5, markersize=2)
axis = fig.add_subplot(5, 1, 4)
self.setupgridticks(axis, "midline", 0.5, majorlabels=False)
self.plotscalar(td, "midline", axis, elinewidth=0.5, markersize=2)
axis = fig.add_subplot(5, 1, 5)
axis.set_xlabel('time (s)')
self.setupgridticks(axis, "speed085", 0.5, majorlabels=True)
self.plotscalar(td, "speed085", axis, elinewidth=0.5, markersize=2)
# default (saved) is 800 x 600 pixels at 100 dpi = 8 x 6 inches;
# double each length:
fig.set_size_inches(16., 12.)
fig.savefig(outputfilepath)
# fig.clf() not enough to release memory! need to use close()
plt.close(fig)
return "wrote %s" % os.path.basename(outputfilepath)
else:
return "skipping %s...output exists" % os.path.basename(
outputfilepath)
parser.add_option("--verbose",
action="store_true",
dest="verbose",
default=False,
help="increase level of output")
(options, args) = parser.parse_args()
# at a minimum, we need a tracker and branch:
if len(args) < 1:
parser.print_usage()
sys.exit(2)
branch = options.branch
tracker = args[0]
# does it exist:
trackerpath = util.getdirectory("choreography", branch, tracker)
if not os.path.exists(trackerpath):
print "file %s not found" % trackerpath
sys.exit(1)
if not os.path.isdir(trackerpath):
print "%s does not seem to be a directory" % trackerpath
sys.exit(1)
# as before, the method is going to be to first generate a list of
# gene/stim to be done and then run them either sequentially or
# on the cluster
tasklist = []
if len(args) == 1:
help="overwrite output files if they exist")
parser.add_option("--nooverwrite", action="store_false", dest="overwrite",
help="do not overwrite output files if they exist")
parser.add_option("--all", action="store_true", dest="processall", default=False,
help="process all experiments even if no new data")
(options, args) = parser.parse_args()
# at a minimum, we need a tracker and branch:
if len(args) < 1:
parser.print_usage()
sys.exit(2)
tracker = args[0]
branch = options.branch
# does it exist:
trackerpath = util.getdirectory("choreography", branch, tracker)
if not os.path.exists(trackerpath):
sys.exit("file %s not found" % trackerpath)
if not os.path.isdir(trackerpath):
sys.exit("%s does not seem to be a directory" % trackerpath)
# as before, the method is going to be to first generate a list of
# gene/stim to be done and then run them either sequentially or
# on the cluster
tasklist = []
if len(args) == 1:
def performtask(self):
"""
write the files
input: none
output: results string; should begin with "error" if an
error occurred
"""
if self.overwrite or not self.outputexists():
# this branch should always be taken in the process I
# have planned, but I don't want to rely on that
if not self.checkcreateoutputfolders():
return "error: could not create output folders for AnalyzeXYMotion!"
# we need two of the .r files:
sourcedir = util.getdirectory("source", self.branch, self.tracker,
self.geneeff, self.stimno)
sourcefilename = "%s.%s.r" % (self.stem, "x")
sourcepath = os.path.join(sourcedir, sourcefilename)
tdx = self.trackdatacache.gettrackdata(sourcepath)
sourcefilename = "%s.%s.r" % (self.stem, "y")
sourcepath = os.path.join(sourcedir, sourcefilename)
tdy = self.trackdatacache.gettrackdata(sourcepath)
# identify and loop over intervals, writing one line per interval
# do for x and y:
messages = []
f = open(self.ysumspath, 'wt')
f.write(
"t1\tt2\ttime on y\ttotal time\trel time\tpath on y\ttotal path\trel path\tntracks\tave t on y\tstd dev\tsum sqrs\tave path on y\tstd dev\tsum sqrs\n"
)
for t1, t2 in util.getintervallist(tdx, "xy-motion"):
(ytime, totaltime, ypath, totalpath, ntracks, timeave, timestd,
timesumsqrs, ypathave, ypathstd,
ypathsumsqrs) = self.windowedcalc(tdx, tdy, t1, t2, 'y')
# note that numpy.divide will emit NaN or Inf, appropriate for
# tabular output
f.write(
"%.1f\t%.1f\t%.1f\t%.1f\t%.3f\t%.1f\t%.1f\t%.3f\t%d\t%.3f\t%.3f\t%.3f\t%.3f\t%.3f\t%.3f\n"
% (t1, t2, ytime, totaltime, numpy.divide(
ytime, totaltime), ypath, totalpath,
numpy.divide(ypath, totalpath), ntracks, timeave,
timestd, timesumsqrs, ypathave, ypathstd, ypathsumsqrs))
f.close()
messages.append('wrote %s' % self.ysumspath)
f = open(self.xsumspath, 'wt')
f.write(
"t1\tt2\ttime on x\ttotal time\trel time\tpath on x\ttotal path\trel path\tntracks\tave t on x\tstd dev\tsum sqrs\tave path on x\tstd dev\tsum sqrs\n"
)
for t1, t2 in util.getintervallist(tdx, "xy-motion"):
(ytime, totaltime, ypath, totalpath, ntracks, timeave, timestd,
timesumsqrs, ypathave, ypathstd,
ypathsumsqrs) = self.windowedcalc(tdx, tdy, t1, t2, 'x')
# note that numpy.divide will emit NaN or Inf, appropriate for
# tabular output
f.write(
"%.1f\t%.1f\t%.1f\t%.1f\t%.3f\t%.1f\t%.1f\t%.3f\t%d\t%.3f\t%.3f\t%.3f\t%.3f\t%.3f\t%.3f\n"
% (t1, t2, ytime, totaltime, numpy.divide(
ytime, totaltime), ypath, totalpath,
numpy.divide(ypath, totalpath), ntracks, timeave,
timestd, timesumsqrs, ypathave, ypathstd, ypathsumsqrs))
f.close()
messages.append('wrote %s' % self.xsumspath)
return '\n'.join(messages)
else:
return "skipping xy-motion analysis...all output exists"