def getNewestFile(self, filepattern: str) -> bytes:
"""Searches for files matching filePattern and returns the data from the newest one."""
data = None
baseDir, filePattern = os.path.split(filepattern)
self._checkAllowedDirs(baseDir)
if not os.path.isabs(baseDir):
# TODO - handle relative paths
pass
filename = utils.findNewestFile(baseDir, filePattern)
self._checkAllowedFileTypes(filename)
if filename is None:
# No file matching pattern
raise FileNotFoundError(
'No file found matching pattern {}'.format(filePattern))
elif not os.path.exists(filename):
raise FileNotFoundError(
'File missing after match {}'.format(filePattern))
else:
with open(filename, 'rb') as fp:
data = fp.read()
return data
def getNewestFile(self, filePattern):
data = None
if self.local:
baseDir, filePattern = os.path.split(filePattern)
if not os.path.isabs(baseDir):
# TODO - handle relative paths
pass
filename = findNewestFile(baseDir, filePattern)
if filename is None:
# No file matching pattern
raise FileNotFoundError('No file found matching pattern {}'.format(filePattern))
elif not os.path.exists(filename):
raise FileNotFoundError('File missing after match {}'.format(filePattern))
else:
with open(filename, 'rb') as fp:
data = fp.read()
else:
getNewestFileCmd = projUtils.getNewestFileReqStruct(filePattern)
retVals = projUtils.clientSendCmd(self.commPipes, getNewestFileCmd)
data = retVals.data
return data
def test_noMatchingFiles(self):
print("Test findNewestFile no matching files:")
filename = utils.findNewestFile('/tmp/testdir/', 'no_such_file')
assert filename is None
def test_pathPartiallyInPattern(self):
print("Test findNewestFile path partially in pattern:")
filename = utils.findNewestFile('/tmp', 'testdir/file1_20170101*')
self.assert_result_matches_filename(filename)
def test_pathInPattern(self):
print("Test findNewestFile path embedded in pattern:")
filename = utils.findNewestFile('/tmp/testdir',
'/tmp/testdir/file1_20170101*')
self.assert_result_matches_filename(filename)
def test_emptyPath(self):
print("Test findNewestFile empty path:")
filename = utils.findNewestFile('', '/tmp/testdir/file1_20170101*')
self.assert_result_matches_filename(filename)
def test_normalCase(self):
print("Test findNewestFile normal case:")
filename = utils.findNewestFile('/tmp/testdir', 'file1_20170101*')
self.assert_result_matches_filename(filename)
def on_message(client, message):
fileWatcher = WsFileWatcher.fileWatcher
response = {'status': 400, 'error': 'unhandled request'}
try:
request = json.loads(message)
response = request.copy()
if 'data' in response: del response['data']
cmd = request.get('cmd')
dir = request.get('dir')
filename = request.get('filename')
timeout = request.get('timeout', 0)
compress = request.get('compress', False)
logging.log(logging.INFO, "{}: {} {}".format(cmd, dir, filename))
# Do Validation Checks
if cmd not in ['getAllowedFileTypes', 'ping', 'error']:
# All other commands must have a filename or directory parameter
if dir is None and filename is not None:
dir, filename = os.path.split(filename)
if filename is None:
errStr = "{}: Missing filename param".format(cmd)
return send_error_response(client, response, errStr)
if dir is None:
errStr = "{}: Missing dir param".format(cmd)
return send_error_response(client, response, errStr)
if cmd in ('watchFile', 'getFile', 'getNewestFile'):
if not os.path.isabs(dir):
# make path relative to the watch dir
dir = os.path.join(fileWatcher.watchDir, dir)
if WsFileWatcher.validateRequestedFile(dir, filename,
cmd) is False:
errStr = '{}: {}'.format(cmd,
WsFileWatcher.validationError)
return send_error_response(client, response, errStr)
if cmd in ('putTextFile', 'putBinaryFile', 'dataLog'):
if not os.path.exists(dir):
os.makedirs(dir)
if not os.path.exists(dir):
errStr = '{}: No such directory: {}'.format(cmd, dir)
return send_error_response(client, response, errStr)
# Now handle requests
if cmd == 'initWatch':
minFileSize = request.get('minFileSize')
demoStep = request.get('demoStep')
if minFileSize is None:
errStr = "InitWatch: Missing minFileSize param"
return send_error_response(client, response, errStr)
WsFileWatcher.fileWatchLock.acquire()
try:
fileWatcher.initFileNotifier(dir, filename, minFileSize,
demoStep)
finally:
WsFileWatcher.fileWatchLock.release()
response.update({'status': 200})
return send_response(client, response)
elif cmd == 'watchFile':
WsFileWatcher.fileWatchLock.acquire()
filename = os.path.join(dir, filename)
try:
retVal = fileWatcher.waitForFile(filename, timeout=timeout)
finally:
WsFileWatcher.fileWatchLock.release()
if retVal is None:
errStr = "WatchFile: 408 Timeout {}s: {}".format(
timeout, filename)
response.update({'status': 408, 'error': errStr})
logging.log(logging.WARNING, errStr)
return send_response(client, response)
else:
response.update({'status': 200, 'filename': filename})
return send_data_response(client, response, compress)
elif cmd == 'getFile':
filename = os.path.join(dir, filename)
if not os.path.exists(filename):
errStr = "GetFile: File not found {}".format(filename)
return send_error_response(client, response, errStr)
response.update({'status': 200, 'filename': filename})
return send_data_response(client, response, compress)
elif cmd == 'getNewestFile':
resultFilename = findNewestFile(dir, filename)
if resultFilename is None or not os.path.exists(
resultFilename):
errStr = 'GetNewestFile: file not found: {}'.format(
os.path.join(dir, filename))
return send_error_response(client, response, errStr)
response.update({'status': 200, 'filename': resultFilename})
return send_data_response(client, response, compress)
elif cmd == 'listFiles':
if not os.path.isabs(dir):
errStr = "listFiles must have an absolute path: {}".format(
dir)
return send_error_response(client, response, errStr)
filePattern = os.path.join(dir, filename)
fileList = [x for x in glob.iglob(filePattern, recursive=True)]
fileList = WsFileWatcher.filterFileList(fileList)
response.update({
'status': 200,
'filePattern': filePattern,
'fileList': fileList
})
return send_response(client, response)
elif cmd == 'getAllowedFileTypes':
response.update({
'status': 200,
'fileTypes': WsFileWatcher.allowedTypes
})
return send_response(client, response)
elif cmd == 'putTextFile':
text = request.get('text')
if text is None:
errStr = 'PutTextFile: Missing text field'
return send_error_response(client, response, errStr)
elif type(text) is not str:
errStr = "PutTextFile: Only text data allowed"
return send_error_response(client, response, errStr)
fullPath = os.path.join(dir, filename)
with open(fullPath, 'w') as volFile:
volFile.write(text)
response.update({'status': 200})
return send_response(client, response)
elif cmd == 'putBinaryFile':
try:
data = unpackDataMessage(request)
except Exception as err:
errStr = 'putBinaryFile: {}'.format(err)
return send_error_response(client, response, errStr)
# If data is None - Incomplete multipart data, more will follow
if data is not None:
fullPath = os.path.join(dir, filename)
with open(fullPath, 'wb') as binFile:
binFile.write(data)
response.update({'status': 200})
return send_response(client, response)
elif cmd == 'dataLog':
logLine = request.get('logLine')
if logLine is None:
errStr = 'DataLog: Missing logLine field'
return send_error_response(client, response, errStr)
fullPath = os.path.join(dir, filename)
with open(fullPath, 'a') as logFile:
logFile.write(logLine + '\n')
response.update({'status': 200})
return send_response(client, response)
elif cmd == 'ping':
response.update({'status': 200})
return send_response(client, response)
elif cmd == 'error':
errorCode = request.get('status', 400)
errorMsg = request.get('error', 'missing error msg')
if errorCode == 401:
WsFileWatcher.needLogin = True
WsFileWatcher.sessionCookie = None
errStr = 'Error {}: {}'.format(errorCode, errorMsg)
logging.log(logging.ERROR, errStr)
return
else:
errStr = 'OnMessage: Unrecognized command {}'.format(cmd)
return send_error_response(client, response, errStr)
except Exception as err:
errStr = "OnMessage Exception: {}: {}".format(cmd, err)
send_error_response(client, response, errStr)
if cmd == 'error':
sys.exit()
return
errStr = 'unhandled request'
send_error_response(client, response, errStr)
return
print(r,s)
# now load the same information
cmap=plt.get_cmap('cool')
colors=cmap(np.linspace(0,1,nStations))
brainiak_path='/jukebox/norman/amennen/github/brainiak/rt-cloud/projects/greenEyes/data/'
# now load newestfile
sys.path.append('/jukebox/norman/amennen/github/brainiak/rt-cloud')
from rtCommon.utils import findNewestFile, loadMatFile
s = 0
subject_path = brainiak_path + 'sub-' + str(allSubjects[s]) + '/' + 'ses-02' + '/'
run = 1
filePattern = 'patternsData_r{}*'.format(run)
fn = findNewestFile(subject_path,filePattern)
test_data = loadMatFile(fn)
test_prob = test_data.correct_prob
x = correct_prob[:,run-1,s]
y = test_prob[0,:]
corr = sstats.pearsonr(x,y)[0]
plt.figure(figsize=(10,10))
for st in np.arange(nStations):
plt.plot(x[st],y[st], '.', ms=20, color=colors[st],label=st)
plt.plot([0,1],[0,1], '--', color='r', lw=3)
plt.title('Subj %i, Run %i, Total corr = %3.3f' % (allSubjects[s],run,corr))
plt.xlim([0,1])
plt.ylim([0,1])
plt.xlabel('Offline prediction')
