def guard_and_time(msg, fn, tic, *args, **kwargs):
info('Starting', msg, 'at', tic)
try:
fn(*args, **kwargs)
except Exception:
email_exc()
info('Done at', np.datetime64('now'))
def __init__(self, dir, filename=None, parser=parseint89, elmin=None):
UserList.__init__(self)
if filename is None:
if not canread(dir):
raise ValueError('Given path ' + dir +
' does not lead to a readable file.')
filename = os.path.basename(dir)
dir = os.path.dirname(dir)
self.site = filename[:4]
self.doy = int(filename[4:7])
self.year = fullyear(int(filename[9:11]))
try:
self.rxloc = sitelocs[self.site]
except KeyError:
info(
'First four characters of filename, ' + self.site +
', are not a recognized site name. Receiver location unknown.')
with fileread(os.path.join(dir, filename)) as fid:
for l in fid:
try:
rec = parser(l)
except ValueError as e:
info(e, end=' on line ' + str(fid.lineno) + '\n')
continue
if elmin is not None and rec.el < elmin:
continue
if rec.snr > 0:
self.append(rec)
self.sort(key=lambda x: x.sod)
def read_rinex_nav(file):
"""Parse RINEX nav message file to Pandas DataFrame.
file: URL, file path, or open (binary-mode) file object.
"""
lines = getlines(file)
try:
idx, = [
i for i, l in enumerate(lines) if l.strip() == b'END OF HEADER'
]
except ValueError:
info(
'Incorrectly formatted RINEX 2 nav file (END OF HEADER does not appear once).'
)
return
lines = [
b''.join(lines[i:i + 8]).replace(b'D', b'E')
for i in range(idx + 1, len(lines), 8)
]
names = [
'PRN', 'ToC', 'Bias', 'Drift', 'Rate', 'IODE', 'Crs', 'Δn', 'M0',
'Cuc', 'Ecc', 'Cus', 'SqrtA', 'ToE', 'Cic', 'Ω', 'CIS', 'i0', 'Crc',
'ω', 'dΩ', 'di', 'CL2', 'Week', 'L2P', 'Acc', 'Health', 'TGD', 'IODC',
'ToM', 'Fit'
]
widths = np.array([2, 20] + [19] * 29)
widths[5::4] += 3
dtype = [np.uint8, 'M8[ms]'] + [np.float] * 29
nnav = np.genfromtxt(lines,
names=names,
delimiter=widths,
comments=None,
converters={1: epoc2dt64},
dtype=dtype)
return pd.DataFrame(nnav).set_index(['ToC', 'PRN'])
def __callback(self, ip, port, message = '', daemon = False):
try:
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
if not daemon: utility.info("Responding to [" + ip + ":" + str(port) + "]")
s.sendto(str(message).encode('utf-8'), (ip, port))
except Exception as e:
utility.warn(str(e) + "\n>>> Continue ...")
def getnewtime(line, curtime, name):
thetime = nptime(line)
if curtime is not None:
thegap = thetime - curtime
if thegap > gapthresh:
info('Gap of', thegap.astype('m8[m]'), 'in file', name)
return thetime
def send_email(self,
subject='Person Detected',
body='__PERSON_DETECTED__',
filename=None):
now = time()
if self.last_email_at is None or (now - self.last_email_at >
self.email_debounce):
msg = MIMEMultipart()
msg['From'] = self.username
msg['To'] = ', '.join(self.recipients)
msg['Subject'] = subject
msg.attach(MIMEText(body, 'plain'))
if filename:
attachment = open(filename, 'rb')
p = MIMEBase('application', 'octet-stream')
p.set_payload(attachment.read())
encoders.encode_base64(p)
p.add_header('Content-Disposition',
'attachment; filename={}'.format(filename))
msg.attach(p)
self.last_email_at = now
self.login()
email_text = msg.as_string()
self.server.sendmail(self.username, self.recipients, email_text)
self.logout()
info('__EMAIL_SENT__')
def setCID(cid, daemon=False):
if not daemon: utility.info('[MXA] Setting Cell ID to [' + str(cid) + ']')
if str(cid).isdigit():
connectMXA('EVM:DLINk:SYNC:CID:AUTO OFF', 5, True)
connectMXA('EVM:DLINk:SYNC:CID ' + str(cid), 5, True)
else:
connectMXA('EVM:DLINk:SYNC:CID:AUTO ON', 5, True)
def __discover(self, delay = 1, daemon = False):
while True:
try:
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
s.bind(('', self.this_device['PORT']))
while True:
message = utility.strToDict(s.recvfrom(self.buffer_size)[0].decode('utf-8'))
if message and message['HOST'] != self.this_device['HOST']:
if message['HOST'] not in self.device_list or (message['HOST'] in self.device_list and self.device_list[message['HOST']]['STATUS'] != 'REACHABLE'):
self.device_list[message['HOST']] = message
try:
hs = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
hs.settimeout(2)
hs.sendto('PING!'.encode('utf-8'), (message['IP'], message['PORT'] + 1))
return_message = hs.recvfrom(self.buffer_size)
hs.settimeout(None)
if return_message[0].decode('utf-8') != 'PONG!':
utility.warn("Handshake with device at [" + message['IP'] + ":" + str(message['PORT']) + "] Failed! Mark device [" + message['HOST'] + "] UNKNOWN" , False)
self.device_list[message['HOST']]['STATUS'] = 'UNKNOWN'
else:
self.device_list[message['HOST']]['STATUS'] = 'REACHABLE'
if not daemon: utility.info("Discovered device [" + message['HOST'] + "] at [" + message['IP'] + "]")
except Exception as e:
if not daemon: utility.warn("Handshake with device at [" + message['IP'] + ":" + str(message['PORT']) + "] Failed! Mark device [" + message['HOST'] + "] UNREACHABLE" , False)
self.device_list[message['HOST']]['STATUS'] = 'UNREACHABLE'
finally:
hs.close()
break
utility.sleep(delay, True)
except Exception as e:
utility.warn(str(e) + "\n>>> Continue ...")
finally:
s.close()
def bigjump(file, curtime):
"""Return true if the time difference between the next line of file and curtime is big."""
jump = nexttime(file) - curtime
if jump > thresh:
info('Jump of', jump.astype('m8[m]'), 'seen in file', file.name)
return True
return False
def __healthCheck(self, interval = 300, daemon = False):
while True:
utility.sleep(interval, True)
for key in self.device_list:
if key is 'SELF': next
## if self.device_list[key]['STATUS'] is 'UNREACHABLE': next # Skip the unreachable devices
else:
try:
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
s.settimeout(2)
s.sendto('PING!'.encode('utf-8'), (self.device_list[key]['IP'], self.device_list[key]['PORT'] + 1))
message = s.recvfrom(self.buffer_size)
s.settimeout(None)
if message[0].decode('utf-8') != 'PONG!':
utility.warn("Health check device at [" + self.device_list[key]['IP'] + ":" + str(self.device_list[key]['PORT']) + "] not responding correctly! Mark device [" + key+ "] UNKNOWN" , False)
self.device_list[key]['STATUS'] = 'UNKNOWN'
else:
self.device_list[key]['STATUS'] = 'REACHABLE'
except Exception as e:
if key in self.device_list and self.device_list[key]['STATUS'] == 'REACHABLE':
utility.warn("Health check device at [" + self.device_list[key]['IP'] + "] failed! Mark device [" + key + "] UNREACHABLE" , False)
self.device_list[key]['STATUS'] = 'UNREACHABLE'
finally:
s.close()
if not daemon:
utility.info(" Health Check Result:\n--------------------------------------------------------------------------------")
for device in self.device_list:
print(self.device_list[device]['STATUS'] + ' ' + self.device_list[device]['HOST'] + ': [' + self.device_list[device]['IP'] + ']')
def midnightplots(SNRs, HK, day=None, ddir=None):
"""Make 24-hour archive page in *ddir* (default config.DAILYDIR).
*day* is a numpy datetime64 in the desired day (default: the immediately
preceding day.)
*SNRs* and *HK* should include data for the desired 24 hours.
"""
if ddir is None:
ddir = config.DAILYDIR
if day is None:
day = np.datetime64('now') - np.timedelta64(6, 'h')
day = day.astype('M8[D]')
ddir = fmt_timesite(ddir, day)
try:
os.makedirs(ddir, exist_ok=True)
except FileExistsError as e:
info('Could not create directory', ddir, '\n ', e)
return
etime = day + np.timedelta64(1, 'D')
makeplots(SNRs, HK, domove=False, plotdir=ddir, snr_hours=24, hk_hours=24, endtime=etime,
figlength=14, doazel=False, snrminel=15)
index = os.path.join(fmt_timesite(config.PLOTDIR, day), 'index.html')
try:
shutil.copy(index, ddir)
except OSError:
info('Could not copy index.html from plot dir to daily dir.')
ylink = os.path.join(ddir, 'yesterlink.html')
yday = day - np.timedelta64(1, 'D')
with open(ylink, 'wt') as fid:
fid.write(yday.tolist().strftime('../../%Y/%j'))
def _move(srcs, suf):
"""Move each filename in *srcs* to the name obtained by removing *suf*."""
for src in srcs:
if not src:
continue
if src.count(suf) != 1:
info('Suffix', suf, 'not found uniquely in filename', src)
continue
os.replace(src, src.replace(suf, '', 1))
def getDevice(self, name = 'SELF', daemon = False):
if name in self.device_list:
if not daemon:
utility.info("Getting device " + name + " from devices list\n----------------------------------------------")
utility.pp(self.device_list[name])
return self.device_list[name]
else:
utility.warn("Device [" + str(name) + "] NOT found, return SELF", False)
return self.device_list['SELF']
def start_workers(self):
with self.glock:
if util.file_exists(self.pidfile):
util.error("pidfile already exists: " + self.pidfile)
sys.exit(-1)
for pname in self.workers.keys():
util.info("{}: worker starting...".format(pname))
self.workers[pname].start()
pass
def connectSQLite(db_path, daemon = False):
global sqlite_file
if db_path is None: db_path = sqlite_file
try:
if not daemon: utility.info("Connecting to DB: " + db_path)
conn = sqlite3.connect(db_path)
return conn
except sqlite3.Error:
utility.error("Connection to " + db_path + " failed")
exit(1)
def rewriteall(odir=vdir, ndir=ndir):
import glob
files = glob.glob(os.path.join(odir, '*.snr89'))
for path in files:
file = os.path.basename(path)
nfile = os.path.join(ndir, file)
if os.path.isfile(nfile):
info(file + ' already present in ' + ndir + '. Skipping...')
continue
rewrite(odir, ndir, file)
def setRefCRSPort(port, daemon=False):
if str(port).isdigit(): port = 'P' + str(port)
if not daemon:
utility.info('[MXA] Setting the Reference C-RS Port [' + str(port) +
'] to use')
if 'AUTO' in str(port) or 'Auto' in str(port) or 'auto' in str(port):
connectMXA('EVM:DLINk:SYNC:ANTenna:PORT:AUTO ON', 5, True)
else:
connectMXA('EVM:DLIN:SYNC:ANT:PORT:AUTO OFF', 5, True)
connectMXA('EVM:DLIN:SYNC:ANT:PORT ' + str(port), 5, True)
def readsync(strm):
"""Read a sync byte from the stream. If we don't see one, scan forward for it."""
read = b''
r = strm.read(1)
while r != SYNC:
if not r:
raise EOFError('End of file encountered before sync byte')
read += r
r = strm.read(1)
if read:
info("Skipped", len(read), "bytes before SYNC:", read.hex())
def readhk(file):
"""Read housekeeping (HK) line. Check for proper format, then discard."""
hk = file.read(50)
if not hk:
info("Housekeeping line expected; instead found end of file?",
file.name)
return
if not hkline.fullmatch(hk):
info(
'The 50 characters following a blank line do not match expected HK format:\n',
hk.decode())
def getMXAResult(daemon=False):
if not daemon:
utility.info("############# " + Fore.YELLOW + "Reading MXA Report" +
Style.RESET_ALL + " #############")
name = (connectMXA('CALC:EVM:DATA4:TABL:NAM?',
daemon=True).replace('"', '')).split(',')
value = (connectMXA('CALC:EVM:DATA4:TABL:STR?',
daemon=True).replace('"', '')).split(',')
result = {}
for i in range(0, len(name)):
result[name[i]] = value[i]
return result
def __init__(self, lock, appfile):
self.glock = lock
self.workers = {}
self.stopfile = appfile + ".FORCESTOP"
self.pidfile = appfile + ".PID"
util.info("stopfile: {}".format(self.stopfile))
util.info("pidfile: {}".format(self.pidfile))
self.curtime = time.time()
pass
def write_all_snr88(SNRs, prefix='rx'):
"""Write out each receiver's snr records in a file in the current directory.
Filename is rx##DOY0.YY.snr88. This assumes that all the data is from the same
GPS day.
"""
for rxid, SNR in SNRs.items():
epoch = SNR[0]['time']
fname = prefix + '{:02}'.format(rxid) + epoch.tolist().strftime(
'%j0.%y.snr88')
info('Writing', fname, '...')
out_snr88(SNR, fname)
def __broadcast(self, delay = 1, daemon = False):
try:
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
s.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
if not daemon: utility.info("Starting broadcasting at: [" + self.this_device['BROADCAST_IP'] + ":" + str(self.this_device['PORT']) + "]")
while True:
self.__loadConfig(self.this_device['PORT'])
s.sendto(str(self.this_device).encode('utf-8'), (self.this_device['BROADCAST_IP'], int(self.this_device['PORT'])))
utility.sleep(delay, True)
except Exception as e:
utility.warn(str(e) + "\n>>> Continue ...")
finally:
s.close()
def translate(fid):
"""Translate BINEX data to ASCII formats.
Read in data from the provided file object, and write out snr88 files
and a housekeeping report in the current directory.
"""
SNRs, HK = readall(fid)
write_all_snr88(SNRs)
hkfile = HK[0]['time'].tolist().strftime(
'HK_%j.%y.txt') # we need a site identifier!
info('Writing', hkfile, '...')
with open(hkfile, 'wt') as fid:
hkreport(HK, fid)
def healthCheck(command='RAA', daemon=False):
if not daemon:
utility.info("################ " + Fore.YELLOW + 'JFW Health Check' +
Style.RESET_ALL + " ###############")
data = connectJFW(command, 2, True)
result = {}
for line in data.split('\n'):
keys = utility.regex(line, 'Atten\s*#*(\d+)\s*=*\s*(\d+)..')
if keys:
if not daemon:
utility.info("Attenuator #" + keys[0] + " - " + keys[1] + "dB")
result[keys[0]] = keys[1]
return result
def posneg(arr):
"""Check that input consists of some positive entries followed by negative entries,
with no mixing. Return index of first negative entry, or None."""
neg = False
ind = None
for i, k in enumerate(arr):
if not neg and k < 0:
ind = i
neg = True
if neg and k > 0:
info("{} (entry {}) is positive, after negative entry at {}!".
format(k, i, ind))
return ind
return ind
def checkline(line, name):
if b'HK' in line:
info('HK found in line not following a blank line in', name,
'-- What?')
#raise OSError('Gave up parsing these files')
return False
elif len(line) < 20:
return False
elif not goodline.fullmatch(line):
info('Bad line in', name, line.decode().strip())
return False
elif int(line[15:17]) >= 70:
return False
return True
def executeSQLite(command, conn = None, daemon = False):
if not daemon: utility.info("############### " + Fore.YELLOW + "Connecting and Executing SQLite Query" + Style.RESET_ALL+ " ###############")
if conn is None: conn = __init(None, daemon)
try:
c = conn.cursor()
if not daemon: utility.info("Executing query: " + command)
result = c.execute(command).fetchall()
if not daemon:
for row in result:
i = 0
utility.info("======= Entry " + str(i+1) + " =======")
for name in names:
utility.info(name + ': ' + str(row[i]))
i += 1
conn.commit()
conn.close()
except sqlite3.Error as e:
utility.error(str(e) + "\n Query " + command + " executeion failed")
return None
if not result:
return None
else:
names = list(map(lambda x:x[0], c.description))
return buildJson(names, result)
def execute(self, device, command, timeout = 20, daemon = False):
try:
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
s.settimeout(timeout)
if not daemon: utility.info("Sending command: [" + str(command) + "] to " + device['IP'] + ":" + str(device['PORT'] + 1))
s.sendto(str(command).encode('utf-8') ,(device['IP'], device['PORT'] + 1))
response = s.recvfrom(self.buffer_size)
s.settimeout(None)
if response: utility.info("Response from [" + response[1][0] + ":" + str(response[1][1]) + "]\n" + response[0].decode('utf-8'))
except Exception as e:
utility.warn(str(e) + "\n>>> Continue ...")
finally:
s.close()
return response[0].decode('utf-8')
def google_ht(lat, lon, bestguess=0):
"""Fetch height above the geoid (m) from Google Maps at the given coordinates (decimal degrees.)
Falls back to third parameter bestguess (default 0) if the HTTP request fails.
"""
gurl = 'https://maps.googleapis.com/maps/api/elevation/json'
params = {'locations': '{},{}'.format(lat, lon)}
if config.GKEY:
params['key'] = config.GKEY
try:
resp = jsonfetch(gurl, **params)
return resp['results'][0]['elevation']
except (HTTPError, KeyError, IndexError) as e:
info(e)
return bestguess