def connect(self):
'''
'''
log.log('Connecting to ftp://{}'.format(self.server), self.name)
try:
self.ftp = ftplib.FTP(self.server, timeout=1)
except ftplib.all_errors:
log.error('Server connection fail: {}'.format(self.server),
self.name)
if self.ftp is not None:
self.ftp.close()
self.ftp = None
self.ready = False
return
try:
self.ftp.login(self.username, self.password)
except ftplib.all_errors:
log.error('The given username y/o password are incorrect',
self.name)
if self.ftp is not None:
self.ftp.close()
self.ftp = None
self.ready = False
return
log.success('Connection success', self.name)
self.ready = True
return
def upload(self, src, dst):
log.log('Uploading {} -> {} '.format(
src.split('/')[-1],
dst.split('/')[-1]),
self.name,
nl=False)
fp = open(src, 'rb')
command = 'STOR {}'.format(dst)
try:
self.ftp.storbinary(command, fp, blocksize=1024)
except Exception as e:
log.error('{}'.format(e), self.name)
return 0
try:
self.ftp.sendcmd('SITE CHMOD 755 {}'.format(dst))
except Exception as e:
log.error('{}'.format(e), self.name)
return 0
fp.close()
log.success('OK', tag='')
return 1
def readNextFile(self):
while True:
self.flagDiscontinuousBlock = 0
if not self.setNextFile():
return 0
self.datatime = datetime.datetime.utcfromtimestamp(
self.header[0]['Time'])
if self.online:
break
if (self.datatime < datetime.datetime.combine(self.startDate, self.startTime)) or \
(self.datatime > datetime.datetime.combine(self.endDate, self.endTime)):
log.warning(
'Reading Record No. {}/{} -> {} [Skipping]'.format(
self.counter_records, self.nrecords,
self.datatime.ctime()), 'PXReader')
continue
break
log.log(
'Reading Record No. {}/{} -> {}'.format(self.counter_records,
self.nrecords,
self.datatime.ctime()),
'PXReader')
return 1
def setNextFile(self):
"""Set the next file to be readed open it and parse de file header"""
while True:
if self.fp != None:
self.fp.close()
if self.online:
newFile = self.setNextFileOnline()
else:
newFile = self.setNextFileOffline()
if not (newFile):
if self.online:
raise schainpy.admin.SchainError(
'Time to wait for new files reach')
else:
if self.fileIndex == -1:
raise schainpy.admin.SchainWarning(
'No files found in the given path')
else:
raise schainpy.admin.SchainWarning(
'No more files to read')
if self.verifyFile(self.filename):
break
log.log('Opening file: %s' % self.filename, self.name)
self.readFirstHeader()
self.nReadBlocks = 0
def readNextBlock(self):
while True:
self.flagDiscontinuousBlock = 0
if self.counter_records == self.nrecords:
self.setNextFile()
self.readBlock()
if (self.datatime < datetime.datetime.combine(self.startDate, self.startTime)) or \
(self.datatime > datetime.datetime.combine(self.endDate, self.endTime)):
log.warning(
'Reading Record No. {}/{} -> {} [Skipping]'.format(
self.counter_records, self.nrecords,
self.datatime.ctime()), 'MADReader')
continue
break
log.log(
'Reading Record No. {}/{} -> {}'.format(self.counter_records,
self.nrecords,
self.datatime.ctime()),
'MADReader')
return 1
def save_figure(self, n):
'''
'''
if (self.data.max_time - self.save_time) <= self.save_period:
return
self.save_time = self.data.max_time
fig = self.figures[n]
figname = os.path.join(
self.save, self.save_code, '{}_{}.png'.format(
self.save_code,
self.getDateTime(self.data.max_time).strftime('%Y%m%d_%H%M%S'),
))
log.log('Saving figure: {}'.format(figname), self.name)
if not os.path.isdir(os.path.dirname(figname)):
os.makedirs(os.path.dirname(figname))
fig.savefig(figname)
if self.throttle == 0:
figname = os.path.join(
self.save, '{}_{}.png'.format(
self.save_code,
self.getDateTime(self.data.min_time).strftime('%Y%m%d'),
))
fig.savefig(figname)
def set_output(self):
'''
Storing data from buffer to dataOut object
'''
self.data['Elevation'] = numpy.array(self.fp.variables['Elevation'])
self.data['Azimuth'] = numpy.array(self.fp.variables['Azimuth'])
self.dataOut.range = numpy.array(self.fp.variables['GateWidth'])
self.dataOut.data = self.data
self.dataOut.units = [h['Unit-value'] for h in self.header]
self.dataOut.parameters = [h['TypeName'] for h in self.header]
self.dataOut.missing = self.header[0]['MissingData']
self.dataOut.max_range = self.header[0]['MaximumRange-value']
self.dataOut.elevation = self.header[0]['Elevation']
self.dataOut.azimuth = self.header[0]['Azimuth']
self.dataOut.latitude = self.header[0]['Latitude']
self.dataOut.longitude = self.header[0]['Longitude']
self.dataOut.utctime = self.header[0]['Time']
self.dataOut.utctimeInit = self.dataOut.utctime
self.dataOut.useLocalTime = True
self.dataOut.flagNoData = False
self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
log.log(
'Parameters found: {}'.format(','.join(self.dataOut.parameters)),
'PXReader')
def readNextBlock(self):
while True:
if not self.online and self.counter_records == self.nrecords:
self.flagIsNewFile = 1
if not self.setNextFile():
return 0
try:
pointer = self.fp.tell()
self.readBlock()
except:
if self.online and self.waitDataBlock(pointer, 38512) == 1:
continue
else:
if not self.setNextFile():
return 0
if (self.datatime < datetime.datetime.combine(self.startDate, self.startTime)) or \
(self.datatime > datetime.datetime.combine(self.endDate, self.endTime)):
log.warning(
'Reading Record No. {}/{} -> {} [Skipping]'.format(
self.counter_records, self.nrecords,
self.datatime.ctime()), 'BLTRParamReader')
continue
break
log.log(
'Reading Record No. {} -> {}'.format(self.counter_records,
self.datatime.ctime()),
'BLTRParamReader')
return 1
def send_to_server(self):
'''
'''
if self.exp_code == None:
log.warning('Missing `exp_code` skipping sending to server...')
last_time = self.data.max_time
interval = last_time - self.sender_time
if interval < self.sender_period:
return
self.sender_time = last_time
attrs = ['titles', 'zmin', 'zmax', 'tag', 'ymin', 'ymax']
for attr in attrs:
value = getattr(self, attr)
if value:
if isinstance(value, (numpy.float32, numpy.float64)):
value = round(float(value), 2)
self.data.meta[attr] = value
if self.colormap == 'jet':
self.data.meta['colormap'] = 'Jet'
elif 'RdBu' in self.colormap:
self.data.meta['colormap'] = 'RdBu'
else:
self.data.meta['colormap'] = 'Viridis'
self.data.meta['interval'] = int(interval)
self.sender_queue.append(last_time)
while True:
try:
tm = self.sender_queue.popleft()
except IndexError:
break
msg = self.data.jsonify(tm, self.save_code, self.plot_type)
self.socket.send_string(msg)
socks = dict(self.poll.poll(2000))
if socks.get(self.socket) == zmq.POLLIN:
reply = self.socket.recv_string()
if reply == 'ok':
log.log("Response from server ok", self.name)
time.sleep(0.1)
continue
else:
log.warning(
"Malformed reply from server: {}".format(reply),
self.name)
else:
log.warning("No response from server, retrying...", self.name)
self.sender_queue.appendleft(tm)
self.socket.setsockopt(zmq.LINGER, 0)
self.socket.close()
self.poll.unregister(self.socket)
self.socket = self.context.socket(zmq.REQ)
self.socket.connect(self.server)
self.poll.register(self.socket, zmq.POLLIN)
break
def publish_data(self):
self.dataOut.finished = False
if self.verbose:
log.log(
'Sending {} - {}'.format(self.dataOut.type,
self.dataOut.datatime), self.name)
self.zmq_socket.send_pyobj(self.dataOut)
def setup(self, **kwargs):
self.set_kwargs(**kwargs)
self.oneDDict = load_json(self.oneDDict)
self.twoDDict = load_json(self.twoDDict)
self.ind2DList = load_json(self.ind2DList)
self.independentParam = self.ind2DList[0]
if self.path is None:
raise ValueError('The path is not valid')
self.open_file = open
self.open_mode = 'rb'
if self.format is None:
raise ValueError('The format is not valid choose simple or hdf5')
elif self.format.lower() in ('simple', 'txt'):
self.ext = '.txt'
elif self.format.lower() in ('cedar', ):
self.ext = '.001'
else:
self.ext = '.hdf5'
self.open_file = h5py.File
self.open_mode = 'r'
if self.online:
log.log("Searching files in online mode...", self.name)
for nTries in range(self.nTries):
fullpath = self.searchFilesOnLine(self.path, self.startDate,
self.endDate, self.expLabel,
self.ext, self.walk,
self.filefmt, self.folderfmt)
try:
fullpath = next(fullpath)
except:
fullpath = None
if fullpath:
break
log.warning(
'Waiting {} sec for a valid file in {}: try {} ...'.format(
self.delay, self.path, nTries + 1), self.name)
time.sleep(self.delay)
if not (fullpath):
raise schainpy.admin.SchainError(
'There isn\'t any valid file in {}'.format(self.path))
else:
log.log("Searching files in {}".format(self.path), self.name)
self.filenameList = self.searchFilesOffLine(
self.path, self.startDate, self.endDate, self.expLabel,
self.ext, self.walk, self.filefmt, self.folderfmt)
self.setNextFile()
def run(self):
if not self.isConfig:
self.setup()
self.isConfig = True
self.dataOut = self.receiver.recv_pyobj()
log.log(
'{} - {}'.format(
self.dataOut.type,
self.dataOut.datatime.ctime(),
), 'Receiving')
def setNextFile(self):
'''
Open next files for the current datetime
'''
cursor = self.cursor
if not self.online_mode:
if cursor == len(self.dates):
if self.online:
cursor = 0
self.dt = self.dates[cursor]
self.online_mode = True
if not self.search_files_online():
log.success('No more files', 'PXReader')
return 0
else:
log.success('No more files', 'PXReader')
self.flagNoMoreFiles = 1
return 0
else:
if not self.search_files_online():
return 0
cursor = self.cursor
self.data = {}
self.header = []
for fullname in self.files[self.dates[cursor]]:
log.log('Opening: {}'.format(fullname), 'PXReader')
if os.path.splitext(fullname)[-1] == '.tgz':
tar = tarfile.open(fullname, 'r:gz')
tar.extractall('/tmp')
files = [
os.path.join('/tmp', member.name)
for member in tar.getmembers()
]
else:
files = [fullname]
for filename in files:
if self.filename is not None:
self.fp.close()
self.filename = filename
self.filedate = self.dates[cursor]
self.fp = Dataset(self.filename, 'r')
self.parseFile()
self.counter_records += 1
self.cursor += 1
return 1
def setNextFileOnline(self):
"""Check for the next file to be readed in online mode.
Set:
self.filename
self.fp
self.filesize
Return:
boolean
"""
nextFile = True
nextDay = False
for nFiles in range(self.nFiles + 1):
for nTries in range(self.nTries):
fullfilename, filename = self.checkForRealPath(
nextFile, nextDay)
if fullfilename is not None:
break
log.warning(
"Waiting %0.2f sec for the next file: \"%s\" , try %02d ..."
% (self.delay, filename, nTries + 1), self.name)
time.sleep(self.delay)
nextFile = False
continue
if fullfilename is not None:
break
self.nTries = 1
nextFile = True
if nFiles == (self.nFiles - 1):
log.log('Trying with next day...', self.name)
nextDay = True
self.nTries = 3
if fullfilename:
self.fileSize = os.path.getsize(fullfilename)
self.filename = fullfilename
self.flagIsNewFile = 1
if self.fp != None:
self.fp.close()
self.fp = self.open_file(fullfilename, self.open_mode)
self.flagNoMoreFiles = 0
self.fileIndex += 1
return 1
else:
return 0
def getData(self):
for attr in self.data:
if self.data[attr].ndim == 1:
setattr(self.dataOut, attr, self.data[attr][self.blockIndex])
else:
setattr(self.dataOut, attr, self.data[attr][:, self.blockIndex])
self.dataOut.flagNoData = False
self.blockIndex += 1
log.log("Block No. {}/{} -> {}".format(
self.blockIndex,
self.blocksPerFile,
self.dataOut.datatime.ctime()), self.name)
return
def setup(self, **kwargs):
self.set_kwargs(**kwargs)
if self.path is None:
raise ValueError("The path is not valid")
if self.online:
log.log("Searching files in online mode...", self.name)
for nTries in range(self.nTries):
fullpath = self.searchFilesOnLine(self.path, self.startDate,
self.endDate, self.expLabel,
self.ext, self.walk,
self.filefmt, self.folderfmt)
try:
fullpath = next(fullpath)
except:
fullpath = None
if fullpath:
self.fileSize = os.path.getsize(fullpath)
self.filename = fullpath
self.flagIsNewFile = 1
if self.fp != None:
self.fp.close()
self.fp = self.open_file(fullpath, self.open_mode)
self.flagNoMoreFiles = 0
break
log.warning(
'Waiting {} sec for a valid file in {}: try {} ...'.format(
self.delay, self.path, nTries + 1), self.name)
time.sleep(self.delay)
if not (fullpath):
raise schainpy.admin.SchainError(
'There isn\'t any valid file in {}'.format(self.path))
self.readFirstHeader()
else:
log.log("Searching files in {}".format(self.path), self.name)
self.filenameList = self.searchFilesOffLine(
self.path, self.startDate, self.endDate, self.expLabel,
self.ext, self.walk, self.filefmt, self.folderfmt)
self.setNextFile()
def writeData(self, fp):
if self.description:
if 'Data' in self.description:
grp = fp.create_group('Data')
else:
grp = fp
else:
grp = fp.create_group('Data')
dtsets = []
data = []
for dsInfo in self.dsList:
if dsInfo['nDim'] == 0:
ds = grp.create_dataset(
self.getLabel(dsInfo['variable']),
(self.blocksPerFile, ),
chunks=True,
dtype=numpy.float64)
dtsets.append(ds)
data.append((dsInfo['variable'], -1))
else:
label = self.getLabel(dsInfo['variable'])
if label is not None:
sgrp = grp.create_group(label)
else:
sgrp = grp
for i in range(dsInfo['dsNumber']):
ds = sgrp.create_dataset(
self.getLabel(dsInfo['variable'], i),
(self.blocksPerFile, ) + dsInfo['shape'][1:],
chunks=True,
dtype=dsInfo['dtype'])
dtsets.append(ds)
data.append((dsInfo['variable'], i))
fp.flush()
log.log('Creating file: {}'.format(fp.filename), self.name)
self.ds = dtsets
self.data = data
self.firsttime = True
self.blockIndex = 0
return
def putData(self):
if (self.blockIndex == self.blocksPerFile) or self.timeFlag():
self.closeFile()
self.setNextFile()
for i, ds in enumerate(self.ds):
attr, ch = self.data[i]
if ch == -1:
ds[self.blockIndex] = getattr(self.dataOut, attr)
else:
ds[self.blockIndex] = getattr(self.dataOut, attr)[ch]
self.fp.flush()
self.blockIndex += 1
log.log('Block No. {}/{}'.format(self.blockIndex, self.blocksPerFile), self.name)
return
def setup(self, **kwargs):
self.set_kwargs(**kwargs)
if not self.ext.startswith('.'):
self.ext = '.{}'.format(self.ext)
if self.online:
log.log("Searching files in online mode...", self.name)
for nTries in range(self.nTries):
fullpath = self.searchFilesOnLine(self.path, self.startDate,
self.endDate, self.expLabel, self.ext, self.walk,
self.filefmt, self.folderfmt)
try:
fullpath = next(fullpath)
except:
fullpath = None
if fullpath:
break
log.warning(
'Waiting {} sec for a valid file in {}: try {} ...'.format(
self.delay, self.path, nTries + 1),
self.name)
time.sleep(self.delay)
if not(fullpath):
raise schainpy.admin.SchainError(
'There isn\'t any valid file in {}'.format(self.path))
pathname, filename = os.path.split(fullpath)
self.year = int(filename[1:5])
self.doy = int(filename[5:8])
self.set = int(filename[8:11]) - 1
else:
log.log("Searching files in {}".format(self.path), self.name)
self.filenameList = self.searchFilesOffLine(self.path, self.startDate,
self.endDate, self.expLabel, self.ext, self.walk, self.filefmt, self.folderfmt)
self.setNextFile()
return
def find_folders(self, path, startDate, endDate, folderfmt, last=False):
folders = [
x for f in path.split(',') for x in os.listdir(f)
if os.path.isdir(os.path.join(f, x))
]
folders.sort()
if last:
folders = [folders[-1]]
for folder in folders:
try:
dt = datetime.datetime.strptime(
parse_format(folder, folderfmt), folderfmt).date()
if dt >= startDate and dt <= endDate:
yield os.path.join(path, folder)
else:
log.log('Skiping folder {}'.format(folder), self.name)
except Exception as e:
log.log('Skiping folder {}'.format(folder), self.name)
continue
return
def readNextBlock(self):
while True:
if not self.readBlock():
self.flagIsNewFile = 1
if not self.setNextFile():
return 0
if (self.datatime < datetime.datetime.combine(self.startDate, self.startTime)) or \
(self.datatime > datetime.datetime.combine(self.endDate, self.endTime)):
log.warning(
'Reading Record No. {} -> {} [Skipping]'.format(
self.counter_records, self.datatime.ctime()),
self.name)
continue
break
log.log(
'Reading Record No. {} -> {}'.format(self.counter_records,
self.datatime.ctime()),
self.name)
return 1
def find_files(self,
folders,
ext,
filefmt,
startDate=None,
endDate=None,
expLabel='',
last=False):
for path in folders:
files = glob.glob1(path, '*{}'.format(ext))
files.sort()
if last:
if files:
fo = files[-1]
try:
dt = datetime.datetime.strptime(
parse_format(fo, filefmt), filefmt).date()
yield os.path.join(path, expLabel, fo)
except Exception as e:
pass
return
else:
return
for fo in files:
try:
dt = datetime.datetime.strptime(parse_format(fo, filefmt),
filefmt).date()
if dt >= startDate and dt <= endDate:
yield os.path.join(path, expLabel, fo)
else:
log.log('Skiping file {}'.format(fo), self.name)
except Exception as e:
log.log('Skiping file {}'.format(fo), self.name)
continue
def search_files(self, path, startDate, endDate, startTime, endTime, walk):
'''
Searching for NCDF files in path
Creating a list of files to procces included in [startDate,endDate]
Input:
path - Path to find files
'''
log.log('Searching files {} in {} '.format(self.ext, path), 'PXReader')
if walk:
paths = [
os.path.join(path, p) for p in os.listdir(path)
if os.path.isdir(os.path.join(path, p))
]
paths.sort()
else:
paths = [path]
fileList0 = []
for subpath in paths:
if not folder_in_range(
subpath.split('/')[-1], startDate, endDate, '%Y%m%d'):
continue
fileList0 += [
os.path.join(subpath, s) for s in glob.glob1(subpath, '*')
if os.path.splitext(s)[-1] in self.ext
and '{}'.format(self.ele) in s
]
fileList0.sort()
if self.online:
fileList0 = fileList0[-1:]
self.files = {}
startDate = startDate - datetime.timedelta(1)
endDate = endDate + datetime.timedelta(1)
for fullname in fileList0:
thisFile = fullname.split('/')[-1]
year = thisFile[3:7]
if not year.isdigit():
continue
month = thisFile[7:9]
if not month.isdigit():
continue
day = thisFile[9:11]
if not day.isdigit():
continue
year, month, day = int(year), int(month), int(day)
dateFile = datetime.date(year, month, day)
timeFile = datetime.time(int(thisFile[12:14]),
int(thisFile[14:16]),
int(thisFile[16:18]))
if (startDate > dateFile) or (endDate < dateFile):
continue
dt = datetime.datetime.combine(dateFile, timeFile)
if dt not in self.files:
self.files[dt] = []
self.files[dt].append(fullname)
self.dates = list(self.files.keys())
self.dates.sort()
return
def readBlock(self):
'''
It should be checked if the block has data, if it is not passed to the next file.
Then the following is done:
1. Read the RecordHeader
2. Fill the buffer with the current block number.
'''
if self.BlockCounter == self.nFDTdataRecors:
if not self.readFile():
return
if self.mode == 1:
self.rheader.read(self.BlockCounter + 1)
elif self.mode == 0:
self.rheader.read(self.BlockCounter)
self.RecCounter = self.rheader.RecCounter
self.OffsetStartHeader = self.rheader.OffsetStartHeader
self.Off2StartNxtRec = self.rheader.Off2StartNxtRec
self.Off2StartData = self.rheader.Off2StartData
self.nProfiles = self.rheader.nProfiles
self.nChannels = self.rheader.nChannels
self.nHeights = self.rheader.nHeights
self.frequency = self.rheader.TransmitFrec
self.DualModeIndex = self.rheader.DualModeIndex
self.pairsList = [(0, 1), (0, 2), (1, 2)]
self.dataOut.pairsList = self.pairsList
self.nRdPairs = len(self.dataOut.pairsList)
self.dataOut.nRdPairs = self.nRdPairs
self.dataOut.heightList = (
self.rheader.StartRangeSamp +
numpy.arange(self.nHeights) * self.rheader.SampResolution) / 1000.
self.dataOut.channelList = range(self.nChannels)
self.dataOut.nProfiles = self.rheader.nProfiles
self.dataOut.nIncohInt = self.rheader.nIncohInt
self.dataOut.nCohInt = self.rheader.nCohInt
self.dataOut.ippSeconds = 1 / float(self.rheader.PRFhz)
self.dataOut.PRF = self.rheader.PRFhz
self.dataOut.nFFTPoints = self.rheader.nProfiles
self.dataOut.utctime = self.rheader.nUtime + self.rheader.nMilisec / 1000.
self.dataOut.timeZone = 0
self.dataOut.useLocalTime = False
self.dataOut.nmodes = 2
log.log(
'Reading block {} - {}'.format(self.BlockCounter,
self.dataOut.datatime), self.name)
OffDATA = self.OffsetStartHeader + self.RecCounter * \
self.Off2StartNxtRec + self.Off2StartData
self.fp.seek(OffDATA, os.SEEK_SET)
self.data_fft = numpy.fromfile(
self.fp, [('complex', '<c8')],
self.nProfiles * self.nChannels * self.nHeights)
self.data_fft = self.data_fft.astype(numpy.dtype('complex'))
self.data_block = numpy.reshape(
self.data_fft, (self.nHeights, self.nChannels, self.nProfiles))
self.data_block = numpy.transpose(self.data_block, (1, 2, 0))
copy = self.data_block.copy()
spc = copy * numpy.conjugate(copy)
self.data_spc = numpy.absolute(spc) # valor absoluto o magnitud
self.dataOut.data_spc = self.data_spc
cspc = self.data_block.copy()
self.data_cspc = self.data_block.copy()
for i in range(self.nRdPairs):
chan_index0 = self.dataOut.pairsList[i][0]
chan_index1 = self.dataOut.pairsList[i][1]
self.data_cspc[i, :, :] = cspc[
chan_index0, :, :] * numpy.conjugate(cspc[chan_index1, :, :])
'''Getting Eij and Nij'''
(AntennaX0,
AntennaY0) = pol2cart(self.rheader.AntennaCoord0,
self.rheader.AntennaAngl0 * numpy.pi / 180)
(AntennaX1,
AntennaY1) = pol2cart(self.rheader.AntennaCoord1,
self.rheader.AntennaAngl1 * numpy.pi / 180)
(AntennaX2,
AntennaY2) = pol2cart(self.rheader.AntennaCoord2,
self.rheader.AntennaAngl2 * numpy.pi / 180)
E01 = AntennaX0 - AntennaX1
N01 = AntennaY0 - AntennaY1
E02 = AntennaX0 - AntennaX2
N02 = AntennaY0 - AntennaY2
E12 = AntennaX1 - AntennaX2
N12 = AntennaY1 - AntennaY2
self.ChanDist = numpy.array([[E01, N01], [E02, N02], [E12, N12]])
self.dataOut.ChanDist = self.ChanDist
self.BlockCounter += 2
self.dataOut.data_spc = self.data_spc
self.dataOut.data_cspc = self.data_cspc
def setup(self, **kwargs):
self.set_kwargs(**kwargs)
if not self.ext.startswith('.'):
self.ext = '.{}'.format(self.ext)
if self.server is not None:
if 'tcp://' in self.server:
address = server
else:
address = 'ipc:///tmp/%s' % self.server
self.server = address
self.context = zmq.Context()
self.receiver = self.context.socket(zmq.PULL)
self.receiver.connect(self.server)
time.sleep(0.5)
print('[Starting] ReceiverData from {}'.format(self.server))
else:
self.server = None
if self.path == None:
raise ValueError("[Reading] The path is not valid")
if self.online:
log.log("[Reading] Searching files in online mode...",
self.name)
for nTries in range(self.nTries):
fullpath = self.searchFilesOnLine(
self.path, self.startDate, self.endDate, self.expLabel,
self.ext, self.walk, self.filefmt, self.folderfmt)
try:
fullpath = next(fullpath)
except:
fullpath = None
if fullpath:
break
log.warning(
'Waiting {} sec for a valid file in {}: try {} ...'.
format(self.delay, self.path, nTries + 1), self.name)
time.sleep(self.delay)
if not (fullpath):
raise schainpy.admin.SchainError(
'There isn\'t any valid file in {}'.format(self.path))
pathname, filename = os.path.split(fullpath)
self.year = int(filename[1:5])
self.doy = int(filename[5:8])
self.set = int(filename[8:11]) - 1
else:
log.log("Searching files in {}".format(self.path), self.name)
self.filenameList = self.searchFilesOffLine(
self.path, self.startDate, self.endDate, self.expLabel,
self.ext, self.walk, self.filefmt, self.folderfmt)
self.setNextFile()
return
def writeBlock(self):
'''
Add data records to cedar file taking data from oneDDict and twoDDict
attributes.
Allowed parameters in: parcodes.tab
'''
startTime = datetime.datetime.utcfromtimestamp(self.dataOut.utctime)
endTime = startTime + datetime.timedelta(
seconds=self.dataOut.paramInterval)
heights = self.dataOut.heightList
if self.ext == '.dat':
for key, value in list(self.twoDDict.items()):
if isinstance(value, str):
data = getattr(self.dataOut, value)
invalid = numpy.isnan(data)
data[invalid] = self.missing
elif isinstance(value, (tuple, list)):
attr, key = value
data = getattr(self.dataOut, attr)
invalid = numpy.isnan(data)
data[invalid] = self.missing
out = {}
for key, value in list(self.twoDDict.items()):
key = key.lower()
if isinstance(value, str):
if 'db' in value.lower():
tmp = getattr(self.dataOut, value.replace('_db', ''))
SNRavg = numpy.average(tmp, axis=0)
tmp = 10 * numpy.log10(SNRavg)
else:
tmp = getattr(self.dataOut, value)
out[key] = tmp.flatten()[:len(heights)]
elif isinstance(value, (tuple, list)):
attr, x = value
data = getattr(self.dataOut, attr)
out[key] = data[int(x)][:len(heights)]
a = numpy.array([out[k] for k in self.keys])
nrows = numpy.array(
[numpy.isnan(a[:, x]).all() for x in range(len(heights))])
index = numpy.where(nrows == False)[0]
rec = madrigal.cedar.MadrigalDataRecord(
self.kinst, self.kindat, startTime.year, startTime.month,
startTime.day, startTime.hour, startTime.minute, startTime.second,
startTime.microsecond / 10000, endTime.year, endTime.month,
endTime.day, endTime.hour, endTime.minute, endTime.second,
endTime.microsecond / 10000, list(self.oneDDict.keys()),
list(self.twoDDict.keys()), len(index), **self.extra_args)
# Setting 1d values
for key in self.oneDDict:
rec.set1D(key, getattr(self.dataOut, self.oneDDict[key]))
# Setting 2d values
nrec = 0
for n in index:
for key in out:
rec.set2D(key, nrec, out[key][n])
nrec += 1
self.fp.append(rec)
if self.ext == '.hdf5' and self.counter % 500 == 0 and self.counter > 0:
self.fp.dump()
if self.counter % 20 == 0 and self.counter > 0:
log.log('Writing {} records'.format(self.counter), 'MADWriter')