def readNextBlock(self): while True: if not self.online and self.counter_records == self.nrecords: self.flagIsNewFile = 1 if not self.setNextFile(): return 0 try: pointer = self.fp.tell() self.readBlock() except: if self.online and self.waitDataBlock(pointer, 38512) == 1: continue else: if not self.setNextFile(): return 0 if (self.datatime < datetime.datetime.combine(self.startDate, self.startTime)) or \ (self.datatime > datetime.datetime.combine(self.endDate, self.endTime)): log.warning( 'Reading Record No. {}/{} -> {} [Skipping]'.format( self.counter_records, self.nrecords, self.datatime.ctime()), 'BLTRParamReader') continue break log.log( 'Reading Record No. {} -> {}'.format(self.counter_records, self.datatime.ctime()), 'BLTRParamReader') return 1
def waitDataBlock(self, pointer_location, blocksize=None): """ """ currentPointer = pointer_location if blocksize is None: neededSize = self.processingHeaderObj.blockSize # + self.basicHeaderSize else: neededSize = blocksize for nTries in range(self.nTries): self.fp.close() self.fp = open(self.filename, 'rb') self.fp.seek(currentPointer) self.fileSize = os.path.getsize(self.filename) currentSize = self.fileSize - currentPointer if (currentSize >= neededSize): return 1 log.warning( "Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries + 1), self.name) time.sleep(self.delay) return 0
def setup(self, path=None, startDate=None, endDate=None, ext=None, startTime=datetime.time(0, 0, 0), endTime=datetime.time(23, 59, 59), timezone=0, format=None, **kwargs): self.path = path self.startDate = startDate self.endDate = endDate self.startTime = startTime self.endTime = endTime self.datatime = datetime.datetime(1900, 1, 1) self.format = format if self.path is None: raise ValueError("The path is not valid") if ext is None: ext = self.ext self.search_files(self.path, startDate, endDate, ext) self.timezone = timezone self.fileIndex = 0 if not self.fileList: log.warning( 'There is no files matching these date in the folder: {}'. format(path), self.name) self.setNextFile()
def readNextFile(self): while True: self.flagDiscontinuousBlock = 0 if not self.setNextFile(): return 0 self.datatime = datetime.datetime.utcfromtimestamp( self.header[0]['Time']) if self.online: break if (self.datatime < datetime.datetime.combine(self.startDate, self.startTime)) or \ (self.datatime > datetime.datetime.combine(self.endDate, self.endTime)): log.warning( 'Reading Record No. {}/{} -> {} [Skipping]'.format( self.counter_records, self.nrecords, self.datatime.ctime()), 'PXReader') continue break log.log( 'Reading Record No. {}/{} -> {}'.format(self.counter_records, self.nrecords, self.datatime.ctime()), 'PXReader') return 1
def call(self, **kwargs): ''' ''' try: if self.dataIn is not None and self.dataIn.flagNoData and not self.dataIn.error: return self.dataIn.isReady() elif self.dataIn is None or not self.dataIn.error: self.run(**kwargs) elif self.dataIn.error: self.dataOut.error = self.dataIn.error self.dataOut.flagNoData = True except: err = traceback.format_exc() if 'SchainWarning' in err: log.warning( err.split('SchainWarning:')[-1].split('\n')[0].strip(), self.name) elif 'SchainError' in err: log.error( err.split('SchainError:')[-1].split('\n')[0].strip(), self.name) else: log.error(err, self.name) self.dataOut.error = True for op, optype, opkwargs in self.operations: if optype == 'other' and not self.dataOut.flagNoData: self.dataOut = op.run(self.dataOut, **opkwargs) elif optype == 'external' and not self.dataOut.flagNoData: op.queue.put(self.dataOut) elif optype == 'external' and self.dataOut.error: op.queue.put(self.dataOut) return 'Error' if self.dataOut.error else self.dataOut.isReady()
def send_files(self): for x, pattern in enumerate(self.patterns): local, remote, ext, period, exp_code, sub_exp_code = pattern if (self.dataOut.utctime - self.times[x]) < int(period): continue srcname = self.find_files(local, ext) if srcname is None: continue if srcname == self.latest[x]: log.warning('File alreday uploaded {}'.format(srcname)) continue if exp_code.strip(): dstname = self.getftpname(srcname, int(exp_code), int(sub_exp_code)) else: dstname = srcname src = os.path.join(local, srcname) dst = os.path.join(remote.strip(), dstname) if self.upload(src, dst): self.times[x] = self.dataOut.utctime self.latest[x] = srcname
def __plot(self): ''' Main function to plot, format and save figures ''' self.plot() self.format() for n, fig in enumerate(self.figures): if self.nrows == 0 or self.nplots == 0: log.warning('No data', self.name) fig.text(0.5, 0.5, 'No Data', fontsize='large', ha='center') fig.canvas.manager.set_window_title(self.CODE) continue fig.canvas.manager.set_window_title('{} - {}'.format( self.title, self.getDateTime(self.data.max_time).strftime('%Y/%m/%d'))) fig.canvas.draw() if self.show: fig.show() figpause(0.01) if self.save: self.save_figure(n) if self.server: self.send_to_server()
def setFile(self): ''' Create new cedar file object ''' self.mnemonic = MNEMONICS[self.kinst] #TODO get mnemonic from madrigal date = datetime.datetime.utcfromtimestamp(self.dataOut.utctime) filename = '{}{}{}'.format(self.mnemonic, date.strftime('%Y%m%d_%H%M%S'), self.ext) self.fullname = os.path.join(self.path, filename) if os.path.isfile(self.fullname): log.warning( 'Destination file {} already exists, previous file deleted.'. format(self.fullname), 'MADWriter') os.remove(self.fullname) try: log.success('Creating file: {}'.format(self.fullname), 'MADWriter') if not os.path.exists(self.path): os.makedirs(self.path) self.fp = madrigal.cedar.MadrigalCedarFile(self.fullname, True) except ValueError as e: log.error( 'Impossible to create a cedar object with "madrigal.cedar.MadrigalCedarFile"', 'MADWriter') return return 1
def readNextBlock(self): while True: self.flagDiscontinuousBlock = 0 if self.counter_records == self.nrecords: self.setNextFile() self.readBlock() if (self.datatime < datetime.datetime.combine(self.startDate, self.startTime)) or \ (self.datatime > datetime.datetime.combine(self.endDate, self.endTime)): log.warning( 'Reading Record No. {}/{} -> {} [Skipping]'.format( self.counter_records, self.nrecords, self.datatime.ctime()), 'MADReader') continue break log.log( 'Reading Record No. {}/{} -> {}'.format(self.counter_records, self.nrecords, self.datatime.ctime()), 'MADReader') return 1
def play_sound(): sound = os.path.join(get_path(), 'alarm1.oga') if os.path.exists(sound): for __ in range(2): os.system('paplay {}'.format(sound)) time.sleep(0.5) else: log.warning('Unable to play alarm, sound file not found', 'ADMIN')
def setup(self, **kwargs): self.set_kwargs(**kwargs) self.oneDDict = load_json(self.oneDDict) self.twoDDict = load_json(self.twoDDict) self.ind2DList = load_json(self.ind2DList) self.independentParam = self.ind2DList[0] if self.path is None: raise ValueError('The path is not valid') self.open_file = open self.open_mode = 'rb' if self.format is None: raise ValueError('The format is not valid choose simple or hdf5') elif self.format.lower() in ('simple', 'txt'): self.ext = '.txt' elif self.format.lower() in ('cedar', ): self.ext = '.001' else: self.ext = '.hdf5' self.open_file = h5py.File self.open_mode = 'r' if self.online: log.log("Searching files in online mode...", self.name) for nTries in range(self.nTries): fullpath = self.searchFilesOnLine(self.path, self.startDate, self.endDate, self.expLabel, self.ext, self.walk, self.filefmt, self.folderfmt) try: fullpath = next(fullpath) except: fullpath = None if fullpath: break log.warning( 'Waiting {} sec for a valid file in {}: try {} ...'.format( self.delay, self.path, nTries + 1), self.name) time.sleep(self.delay) if not (fullpath): raise schainpy.admin.SchainError( 'There isn\'t any valid file in {}'.format(self.path)) else: log.log("Searching files in {}".format(self.path), self.name) self.filenameList = self.searchFilesOffLine( self.path, self.startDate, self.endDate, self.expLabel, self.ext, self.walk, self.filefmt, self.folderfmt) self.setNextFile()
def setNextFileOnline(self): """Check for the next file to be readed in online mode. Set: self.filename self.fp self.filesize Return: boolean """ nextFile = True nextDay = False for nFiles in range(self.nFiles + 1): for nTries in range(self.nTries): fullfilename, filename = self.checkForRealPath( nextFile, nextDay) if fullfilename is not None: break log.warning( "Waiting %0.2f sec for the next file: \"%s\" , try %02d ..." % (self.delay, filename, nTries + 1), self.name) time.sleep(self.delay) nextFile = False continue if fullfilename is not None: break self.nTries = 1 nextFile = True if nFiles == (self.nFiles - 1): log.log('Trying with next day...', self.name) nextDay = True self.nTries = 3 if fullfilename: self.fileSize = os.path.getsize(fullfilename) self.filename = fullfilename self.flagIsNewFile = 1 if self.fp != None: self.fp.close() self.fp = self.open_file(fullfilename, self.open_mode) self.flagNoMoreFiles = 0 self.fileIndex += 1 return 1 else: return 0
def check(self): try: if not self.ready: if time.time() - self.current_time < self.timeout: return else: self.current_time = time.time() self.ftp.voidcmd("NOOP") except: log.warning('Connection lost... trying to reconnect', self.name) if self.ftp is not None: self.ftp.close() self.ftp = None self.connect()
def parseHeader(self): ''' ''' self.output = {} self.version = '2' s_parameters = None if self.ext == '.txt': self.parameters = [ s.strip().lower() for s in self.fp.readline().decode().strip().split(' ') if s ] elif self.ext == '.hdf5': self.metadata = self.fp['Metadata'] if '_record_layout' in self.metadata: s_parameters = [ s[0].lower().decode() for s in self.metadata['Independent Spatial Parameters'] ] self.version = '3' self.parameters = [ s[0].lower().decode() for s in self.metadata['Data Parameters'] ] log.success('Parameters found: {}'.format(self.parameters), 'MADReader') if s_parameters: log.success('Spatial parameters found: {}'.format(s_parameters), 'MADReader') for param in list(self.oneDDict.keys()): if param.lower() not in self.parameters: log.warning( 'Parameter {} not found will be ignored'.format(param), 'MADReader') self.oneDDict.pop(param, None) for param, value in list(self.twoDDict.items()): if param.lower() not in self.parameters: log.warning( 'Parameter {} not found, it will be ignored'.format(param), 'MADReader') self.twoDDict.pop(param, None) continue if isinstance(value, list): if value[0] not in self.output: self.output[value[0]] = [] self.output[value[0]].append([])
def setup(self, **kwargs): self.set_kwargs(**kwargs) if self.path is None: raise ValueError("The path is not valid") if self.online: log.log("Searching files in online mode...", self.name) for nTries in range(self.nTries): fullpath = self.searchFilesOnLine(self.path, self.startDate, self.endDate, self.expLabel, self.ext, self.walk, self.filefmt, self.folderfmt) try: fullpath = next(fullpath) except: fullpath = None if fullpath: self.fileSize = os.path.getsize(fullpath) self.filename = fullpath self.flagIsNewFile = 1 if self.fp != None: self.fp.close() self.fp = self.open_file(fullpath, self.open_mode) self.flagNoMoreFiles = 0 break log.warning( 'Waiting {} sec for a valid file in {}: try {} ...'.format( self.delay, self.path, nTries + 1), self.name) time.sleep(self.delay) if not (fullpath): raise schainpy.admin.SchainError( 'There isn\'t any valid file in {}'.format(self.path)) self.readFirstHeader() else: log.log("Searching files in {}".format(self.path), self.name) self.filenameList = self.searchFilesOffLine( self.path, self.startDate, self.endDate, self.expLabel, self.ext, self.walk, self.filefmt, self.folderfmt) self.setNextFile()
def setup(self, **kwargs): self.set_kwargs(**kwargs) if not self.ext.startswith('.'): self.ext = '.{}'.format(self.ext) if self.online: log.log("Searching files in online mode...", self.name) for nTries in range(self.nTries): fullpath = self.searchFilesOnLine(self.path, self.startDate, self.endDate, self.expLabel, self.ext, self.walk, self.filefmt, self.folderfmt) try: fullpath = next(fullpath) except: fullpath = None if fullpath: break log.warning( 'Waiting {} sec for a valid file in {}: try {} ...'.format( self.delay, self.path, nTries + 1), self.name) time.sleep(self.delay) if not(fullpath): raise schainpy.admin.SchainError( 'There isn\'t any valid file in {}'.format(self.path)) pathname, filename = os.path.split(fullpath) self.year = int(filename[1:5]) self.doy = int(filename[5:8]) self.set = int(filename[8:11]) - 1 else: log.log("Searching files in {}".format(self.path), self.name) self.filenameList = self.searchFilesOffLine(self.path, self.startDate, self.endDate, self.expLabel, self.ext, self.walk, self.filefmt, self.folderfmt) self.setNextFile() return
def run(self, dataOut, headers=[ 'systemHeaderObj', 'radarControllerHeaderObj', 'processingHeaderObj' ]): if self.__printInfo == False: return for header in headers: if hasattr(dataOut, header): obj = getattr(dataOut, header) if hasattr(obj, 'printInfo'): obj.printInfo() else: print(obj) else: log.warning('Header {} Not found in object'.format(header)) self.__printInfo = False
def __readData(self): data = {} if self.description: for key, value in self.description['Data'].items(): if isinstance(value, str): if isinstance(self.fp[value], h5py.Dataset): data[key] = self.fp[value].value elif isinstance(self.fp[value], h5py.Group): array = [] for ch in self.fp[value]: array.append(self.fp[value][ch].value) data[key] = numpy.array(array) elif isinstance(value, list): array = [] for ch in value: array.append(self.fp[ch].value) data[key] = numpy.array(array) else: grp = self.fp['Data'] for name in grp: if isinstance(grp[name], h5py.Dataset): array = grp[name].value elif isinstance(grp[name], h5py.Group): array = [] for ch in grp[name]: array.append(grp[name][ch].value) array = numpy.array(array) else: log.warning('Unknown type: {}'.format(name)) if name in self.description: key = self.description[name] else: key = name data[key] = array self.data = data return
def send_to_server(self): ''' ''' if self.exp_code == None: log.warning('Missing `exp_code` skipping sending to server...') last_time = self.data.max_time interval = last_time - self.sender_time if interval < self.sender_period: return self.sender_time = last_time attrs = ['titles', 'zmin', 'zmax', 'tag', 'ymin', 'ymax'] for attr in attrs: value = getattr(self, attr) if value: if isinstance(value, (numpy.float32, numpy.float64)): value = round(float(value), 2) self.data.meta[attr] = value if self.colormap == 'jet': self.data.meta['colormap'] = 'Jet' elif 'RdBu' in self.colormap: self.data.meta['colormap'] = 'RdBu' else: self.data.meta['colormap'] = 'Viridis' self.data.meta['interval'] = int(interval) self.sender_queue.append(last_time) while True: try: tm = self.sender_queue.popleft() except IndexError: break msg = self.data.jsonify(tm, self.save_code, self.plot_type) self.socket.send_string(msg) socks = dict(self.poll.poll(2000)) if socks.get(self.socket) == zmq.POLLIN: reply = self.socket.recv_string() if reply == 'ok': log.log("Response from server ok", self.name) time.sleep(0.1) continue else: log.warning( "Malformed reply from server: {}".format(reply), self.name) else: log.warning("No response from server, retrying...", self.name) self.sender_queue.appendleft(tm) self.socket.setsockopt(zmq.LINGER, 0) self.socket.close() self.poll.unregister(self.socket) self.socket = self.context.socket(zmq.REQ) self.socket.connect(self.server) self.poll.register(self.socket, zmq.POLLIN) break
def writeMetadata(self, fp): if self.description: if 'Metadata' in self.description: grp = fp.create_group('Metadata') else: grp = fp else: grp = fp.create_group('Metadata') for i in range(len(self.metadataList)): if not hasattr(self.dataOut, self.metadataList[i]): log.warning('Metadata: `{}` not found'.format(self.metadataList[i]), self.name) continue value = getattr(self.dataOut, self.metadataList[i]) if isinstance(value, bool): if value is True: value = 1 else: value = 0 grp.create_dataset(self.getLabel(self.metadataList[i]), data=value) return
def readNextBlock(self): while True: if not self.readBlock(): self.flagIsNewFile = 1 if not self.setNextFile(): return 0 if (self.datatime < datetime.datetime.combine(self.startDate, self.startTime)) or \ (self.datatime > datetime.datetime.combine(self.endDate, self.endTime)): log.warning( 'Reading Record No. {} -> {} [Skipping]'.format( self.counter_records, self.datatime.ctime()), self.name) continue break log.log( 'Reading Record No. {} -> {}'.format(self.counter_records, self.datatime.ctime()), self.name) return 1
def setup(self, path=None, blocksPerFile=10, metadataList=None, dataList=None, setType=None, description=None): self.path = path self.blocksPerFile = blocksPerFile self.metadataList = metadataList self.dataList = [s.strip() for s in dataList] self.setType = setType self.description = description if self.metadataList is None: self.metadataList = self.dataOut.metadata_list tableList = [] dsList = [] for i in range(len(self.dataList)): dsDict = {} if hasattr(self.dataOut, self.dataList[i]): dataAux = getattr(self.dataOut, self.dataList[i]) dsDict['variable'] = self.dataList[i] else: log.warning('Attribute {} not found in dataOut', self.name) continue if dataAux is None: continue elif isinstance(dataAux, (int, float, numpy.integer, numpy.float)): dsDict['nDim'] = 0 else: dsDict['nDim'] = len(dataAux.shape) dsDict['shape'] = dataAux.shape dsDict['dsNumber'] = dataAux.shape[0] dsDict['dtype'] = dataAux.dtype dsList.append(dsDict) self.dsList = dsList self.currentDay = self.dataOut.datatime.date()
def search_files_online(self): ''' Searching for NCDF files in online mode path Creating a list of files to procces included in [startDate,endDate] Input: path - Path to find files ''' self.files = {} for n in range(self.nTries): if self.walk: paths = [ os.path.join(self.path, p) for p in os.listdir(self.path) if os.path.isdir(os.path.join(self.path, p)) ] paths.sort() path = paths[-1] else: path = self.path new_files = [ os.path.join(path, s) for s in glob.glob1(path, '*') if os.path.splitext(s)[-1] in self.ext and '{}'.format(self.ele) in s ] new_files.sort() for fullname in new_files: thisFile = fullname.split('/')[-1] year = thisFile[3:7] if not year.isdigit(): continue month = thisFile[7:9] if not month.isdigit(): continue day = thisFile[9:11] if not day.isdigit(): continue year, month, day = int(year), int(month), int(day) dateFile = datetime.date(year, month, day) timeFile = datetime.time(int(thisFile[12:14]), int(thisFile[14:16]), int(thisFile[16:18])) dt = datetime.datetime.combine(dateFile, timeFile) if self.dt >= dt: continue if dt not in self.files: self.dt = dt self.files[dt] = [] self.files[dt].append(fullname) break if self.files: break else: log.warning( 'Waiting {} seconds for the next file, try {} ...'.format( self.delay, n + 1), 'PXReader') time.sleep(self.delay) if not self.files: return 0 self.dates = list(self.files.keys()) self.dates.sort() self.cursor = 0 return 1
def _monitor(self, queue, ctx): import socket procs = 0 err_msg = '' while True: msg = queue.get() if '#_start_#' in msg: procs += 1 elif '#_end_#' in msg: procs -= 1 else: err_msg = msg if procs == 0 or 'Traceback' in err_msg: break time.sleep(0.1) if '|' in err_msg: name, err = err_msg.split('|') if 'SchainWarning' in err: log.warning( err.split('SchainWarning:')[-1].split('\n')[0].strip(), name) elif 'SchainError' in err: log.error( err.split('SchainError:')[-1].split('\n')[0].strip(), name) else: log.error(err, name) else: name, err = self.name, err_msg time.sleep(1) ctx.term() message = ''.join(err) if err_msg: subject = 'SChain v%s: Error running %s\n' % (schainpy.__version__, self.name) subtitle = 'Hostname: %s\n' % socket.gethostbyname( socket.gethostname()) subtitle += 'Working directory: %s\n' % os.path.abspath('./') subtitle += 'Configuration file: %s\n' % self.filename subtitle += 'Time: %s\n' % str(datetime.datetime.now()) readUnitConfObj = self.getReadUnit() if readUnitConfObj: subtitle += '\nInput parameters:\n' subtitle += '[Data path = %s]\n' % readUnitConfObj.parameters[ 'path'] subtitle += '[Start date = %s]\n' % readUnitConfObj.parameters[ 'startDate'] subtitle += '[End date = %s]\n' % readUnitConfObj.parameters[ 'endDate'] subtitle += '[Start time = %s]\n' % readUnitConfObj.parameters[ 'startTime'] subtitle += '[End time = %s]\n' % readUnitConfObj.parameters[ 'endTime'] a = Alarm(modes=self.alarm, email=self.email, message=message, subject=subject, subtitle=subtitle, filename=self.filename) a.start()
import numpy import datetime from collections import deque from functools import wraps from threading import Thread import matplotlib if 'BACKEND' in os.environ: matplotlib.use(os.environ['BACKEND']) elif 'linux' in sys.platform: matplotlib.use("TkAgg") elif 'darwin' in sys.platform: matplotlib.use('MacOSX') else: from schainpy.utils import log log.warning('Using default Backend="Agg"', 'INFO') matplotlib.use('Agg') import matplotlib.pyplot as plt from matplotlib.patches import Polygon from mpl_toolkits.axes_grid1 import make_axes_locatable from matplotlib.ticker import FuncFormatter, LinearLocator, MultipleLocator from schainpy.model.data.jrodata import PlotterData from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator from schainpy.utils import log jet_values = matplotlib.pyplot.get_cmap('jet', 100)(numpy.arange(100))[10:90] blu_values = matplotlib.pyplot.get_cmap('seismic_r', 20)(numpy.arange(20))[10:15] ncmap = matplotlib.colors.LinearSegmentedColormap.from_list(
def setup(self, **kwargs): self.set_kwargs(**kwargs) if not self.ext.startswith('.'): self.ext = '.{}'.format(self.ext) if self.server is not None: if 'tcp://' in self.server: address = server else: address = 'ipc:///tmp/%s' % self.server self.server = address self.context = zmq.Context() self.receiver = self.context.socket(zmq.PULL) self.receiver.connect(self.server) time.sleep(0.5) print('[Starting] ReceiverData from {}'.format(self.server)) else: self.server = None if self.path == None: raise ValueError("[Reading] The path is not valid") if self.online: log.log("[Reading] Searching files in online mode...", self.name) for nTries in range(self.nTries): fullpath = self.searchFilesOnLine( self.path, self.startDate, self.endDate, self.expLabel, self.ext, self.walk, self.filefmt, self.folderfmt) try: fullpath = next(fullpath) except: fullpath = None if fullpath: break log.warning( 'Waiting {} sec for a valid file in {}: try {} ...'. format(self.delay, self.path, nTries + 1), self.name) time.sleep(self.delay) if not (fullpath): raise schainpy.admin.SchainError( 'There isn\'t any valid file in {}'.format(self.path)) pathname, filename = os.path.split(fullpath) self.year = int(filename[1:5]) self.doy = int(filename[5:8]) self.set = int(filename[8:11]) - 1 else: log.log("Searching files in {}".format(self.path), self.name) self.filenameList = self.searchFilesOffLine( self.path, self.startDate, self.endDate, self.expLabel, self.ext, self.walk, self.filefmt, self.folderfmt) self.setNextFile() return
def test(): log.warning('testing')