def getNewestFile(self, filepattern: str) -> bytes: """Searches for files matching filePattern and returns the data from the newest one.""" data = None baseDir, filePattern = os.path.split(filepattern) self._checkAllowedDirs(baseDir) if not os.path.isabs(baseDir): # TODO - handle relative paths pass filename = utils.findNewestFile(baseDir, filePattern) self._checkAllowedFileTypes(filename) if filename is None: # No file matching pattern raise FileNotFoundError( 'No file found matching pattern {}'.format(filePattern)) elif not os.path.exists(filename): raise FileNotFoundError( 'File missing after match {}'.format(filePattern)) else: with open(filename, 'rb') as fp: data = fp.read() return data
def getNewestFile(self, filePattern): data = None if self.local: baseDir, filePattern = os.path.split(filePattern) if not os.path.isabs(baseDir): # TODO - handle relative paths pass filename = findNewestFile(baseDir, filePattern) if filename is None: # No file matching pattern raise FileNotFoundError('No file found matching pattern {}'.format(filePattern)) elif not os.path.exists(filename): raise FileNotFoundError('File missing after match {}'.format(filePattern)) else: with open(filename, 'rb') as fp: data = fp.read() else: getNewestFileCmd = projUtils.getNewestFileReqStruct(filePattern) retVals = projUtils.clientSendCmd(self.commPipes, getNewestFileCmd) data = retVals.data return data
def test_noMatchingFiles(self): print("Test findNewestFile no matching files:") filename = utils.findNewestFile('/tmp/testdir/', 'no_such_file') assert filename is None
def test_pathPartiallyInPattern(self): print("Test findNewestFile path partially in pattern:") filename = utils.findNewestFile('/tmp', 'testdir/file1_20170101*') self.assert_result_matches_filename(filename)
def test_pathInPattern(self): print("Test findNewestFile path embedded in pattern:") filename = utils.findNewestFile('/tmp/testdir', '/tmp/testdir/file1_20170101*') self.assert_result_matches_filename(filename)
def test_emptyPath(self): print("Test findNewestFile empty path:") filename = utils.findNewestFile('', '/tmp/testdir/file1_20170101*') self.assert_result_matches_filename(filename)
def test_normalCase(self): print("Test findNewestFile normal case:") filename = utils.findNewestFile('/tmp/testdir', 'file1_20170101*') self.assert_result_matches_filename(filename)
def on_message(client, message): fileWatcher = WsFileWatcher.fileWatcher response = {'status': 400, 'error': 'unhandled request'} try: request = json.loads(message) response = request.copy() if 'data' in response: del response['data'] cmd = request.get('cmd') dir = request.get('dir') filename = request.get('filename') timeout = request.get('timeout', 0) compress = request.get('compress', False) logging.log(logging.INFO, "{}: {} {}".format(cmd, dir, filename)) # Do Validation Checks if cmd not in ['getAllowedFileTypes', 'ping', 'error']: # All other commands must have a filename or directory parameter if dir is None and filename is not None: dir, filename = os.path.split(filename) if filename is None: errStr = "{}: Missing filename param".format(cmd) return send_error_response(client, response, errStr) if dir is None: errStr = "{}: Missing dir param".format(cmd) return send_error_response(client, response, errStr) if cmd in ('watchFile', 'getFile', 'getNewestFile'): if not os.path.isabs(dir): # make path relative to the watch dir dir = os.path.join(fileWatcher.watchDir, dir) if WsFileWatcher.validateRequestedFile(dir, filename, cmd) is False: errStr = '{}: {}'.format(cmd, WsFileWatcher.validationError) return send_error_response(client, response, errStr) if cmd in ('putTextFile', 'putBinaryFile', 'dataLog'): if not os.path.exists(dir): os.makedirs(dir) if not os.path.exists(dir): errStr = '{}: No such directory: {}'.format(cmd, dir) return send_error_response(client, response, errStr) # Now handle requests if cmd == 'initWatch': minFileSize = request.get('minFileSize') demoStep = request.get('demoStep') if minFileSize is None: errStr = "InitWatch: Missing minFileSize param" return send_error_response(client, response, errStr) WsFileWatcher.fileWatchLock.acquire() try: fileWatcher.initFileNotifier(dir, filename, minFileSize, demoStep) finally: WsFileWatcher.fileWatchLock.release() response.update({'status': 200}) return send_response(client, response) elif cmd == 'watchFile': WsFileWatcher.fileWatchLock.acquire() filename = os.path.join(dir, filename) try: retVal = fileWatcher.waitForFile(filename, timeout=timeout) finally: WsFileWatcher.fileWatchLock.release() if retVal is None: errStr = "WatchFile: 408 Timeout {}s: {}".format( timeout, filename) response.update({'status': 408, 'error': errStr}) logging.log(logging.WARNING, errStr) return send_response(client, response) else: response.update({'status': 200, 'filename': filename}) return send_data_response(client, response, compress) elif cmd == 'getFile': filename = os.path.join(dir, filename) if not os.path.exists(filename): errStr = "GetFile: File not found {}".format(filename) return send_error_response(client, response, errStr) response.update({'status': 200, 'filename': filename}) return send_data_response(client, response, compress) elif cmd == 'getNewestFile': resultFilename = findNewestFile(dir, filename) if resultFilename is None or not os.path.exists( resultFilename): errStr = 'GetNewestFile: file not found: {}'.format( os.path.join(dir, filename)) return send_error_response(client, response, errStr) response.update({'status': 200, 'filename': resultFilename}) return send_data_response(client, response, compress) elif cmd == 'listFiles': if not os.path.isabs(dir): errStr = "listFiles must have an absolute path: {}".format( dir) return send_error_response(client, response, errStr) filePattern = os.path.join(dir, filename) fileList = [x for x in glob.iglob(filePattern, recursive=True)] fileList = WsFileWatcher.filterFileList(fileList) response.update({ 'status': 200, 'filePattern': filePattern, 'fileList': fileList }) return send_response(client, response) elif cmd == 'getAllowedFileTypes': response.update({ 'status': 200, 'fileTypes': WsFileWatcher.allowedTypes }) return send_response(client, response) elif cmd == 'putTextFile': text = request.get('text') if text is None: errStr = 'PutTextFile: Missing text field' return send_error_response(client, response, errStr) elif type(text) is not str: errStr = "PutTextFile: Only text data allowed" return send_error_response(client, response, errStr) fullPath = os.path.join(dir, filename) with open(fullPath, 'w') as volFile: volFile.write(text) response.update({'status': 200}) return send_response(client, response) elif cmd == 'putBinaryFile': try: data = unpackDataMessage(request) except Exception as err: errStr = 'putBinaryFile: {}'.format(err) return send_error_response(client, response, errStr) # If data is None - Incomplete multipart data, more will follow if data is not None: fullPath = os.path.join(dir, filename) with open(fullPath, 'wb') as binFile: binFile.write(data) response.update({'status': 200}) return send_response(client, response) elif cmd == 'dataLog': logLine = request.get('logLine') if logLine is None: errStr = 'DataLog: Missing logLine field' return send_error_response(client, response, errStr) fullPath = os.path.join(dir, filename) with open(fullPath, 'a') as logFile: logFile.write(logLine + '\n') response.update({'status': 200}) return send_response(client, response) elif cmd == 'ping': response.update({'status': 200}) return send_response(client, response) elif cmd == 'error': errorCode = request.get('status', 400) errorMsg = request.get('error', 'missing error msg') if errorCode == 401: WsFileWatcher.needLogin = True WsFileWatcher.sessionCookie = None errStr = 'Error {}: {}'.format(errorCode, errorMsg) logging.log(logging.ERROR, errStr) return else: errStr = 'OnMessage: Unrecognized command {}'.format(cmd) return send_error_response(client, response, errStr) except Exception as err: errStr = "OnMessage Exception: {}: {}".format(cmd, err) send_error_response(client, response, errStr) if cmd == 'error': sys.exit() return errStr = 'unhandled request' send_error_response(client, response, errStr) return
print(r,s) # now load the same information cmap=plt.get_cmap('cool') colors=cmap(np.linspace(0,1,nStations)) brainiak_path='/jukebox/norman/amennen/github/brainiak/rt-cloud/projects/greenEyes/data/' # now load newestfile sys.path.append('/jukebox/norman/amennen/github/brainiak/rt-cloud') from rtCommon.utils import findNewestFile, loadMatFile s = 0 subject_path = brainiak_path + 'sub-' + str(allSubjects[s]) + '/' + 'ses-02' + '/' run = 1 filePattern = 'patternsData_r{}*'.format(run) fn = findNewestFile(subject_path,filePattern) test_data = loadMatFile(fn) test_prob = test_data.correct_prob x = correct_prob[:,run-1,s] y = test_prob[0,:] corr = sstats.pearsonr(x,y)[0] plt.figure(figsize=(10,10)) for st in np.arange(nStations): plt.plot(x[st],y[st], '.', ms=20, color=colors[st],label=st) plt.plot([0,1],[0,1], '--', color='r', lw=3) plt.title('Subj %i, Run %i, Total corr = %3.3f' % (allSubjects[s],run,corr)) plt.xlim([0,1]) plt.ylim([0,1]) plt.xlabel('Offline prediction')