def main():
import argparse
import re
import sys
# Parse arguments
parser = argparse.ArgumentParser(
description='Beacon Reception Graph Generator')
parser.add_argument('-d',
'--debug',
action='store_true',
default=False,
help='enable debug')
parser.add_argument('-o',
'--output',
required=True,
help='output file name (PNG graphic)')
parser.add_argument('datestr', help='datestr (e.g. 20171028)')
args = parser.parse_args()
m = re.match(r'[0-9]{8}$', args.datestr)
if not m:
eprint("Illegal datestr '%s' specified" % (args.datestr))
sys.exit(1)
gen_graph(datestr=args.datestr, outfile_name=args.output, debug=args.debug)
def keep_task(func, name, exit_return=False, sleep_sec=1):
"""
Iterate function func and restart when it exited or raised an exception
if the function completed, it will be restarted if exit_return is True
"""
from time import sleep
def datestr():
from datetime import datetime
return datetime.now().strftime('%Y-%m-%d %H:%M:%S')
while True:
try:
func()
if exit_return:
return
eprint('Function %s exited at %s. Continued.' % (name, datestr()))
logging.error('Function %s exited at %s. Continued.' % \
(name, datestr()))
except KeyboardInterrupt:
break
except:
eprint('Function %s raised an exception at %s. Continued.' % \
(name, datestr()))
logging.exception(name + ' at ' + datestr())
sleep(sleep_sec)
def startrec(check_limit=False, debug=False):
from datetime import datetime
from multiprocessing import Queue
from softrock import initialize
import alsaaudio
import sys
# Initalize SoftRock
initialize(debug=debug)
bfo_offset_hz = config.getint('SignalRecorder', 'bfo_offset_hz')
device = config.get('SignalRecorder', 'alsa_dev')
samplerate = config.getint('Signal', 'samplerate')
inp = alsaaudio.PCM(alsaaudio.PCM_CAPTURE, device=device)
inp.setchannels(2)
truerate = inp.setrate(samplerate)
if truerate != samplerate:
eprint("Can't specify samplerate %d [Hz] to CODEC" % (samplerate))
sys.exit(1)
queue = Queue()
sigproc = SigProc(samplerate, queue)
sigproc.start()
periodsize = samplerate / 10 # XXX magic number
inp.setformat(alsaaudio.PCM_FORMAT_S16_LE)
inp.setperiodsize(periodsize)
# Continuously push the data to queue
first = True
while True:
readsize, samples = inp.read()
now = datetime.utcnow()
# If buffer overrun occurred, tell the SigProc
if readsize != periodsize:
eprint('Overrun occurred.')
samples = '\0' * periodsize * 2 * 2 # 2 ch * S16_LE
queue.put((samples, now))
# Change receiving frequency at appropriate timing
cur_sec_x10 = sec_x10(now)
if first:
last_sec_x10 = cur_sec_x10
if last_sec_x10 < FREQ_CHANGE_TIMING and \
cur_sec_x10 >= FREQ_CHANGE_TIMING:
change_freq(now, bfo_offset_hz, debug)
# Final processes for the next iteration
last_sec_x10 = cur_sec_x10
first = False
sigproc.join()
def run(self):
cutout = CutOutSamples(self.samplerate)
while True:
try:
new_samples, now = self.queue.get()
cutout.extend(now, bytearray(new_samples))
except KeyboardInterrupt:
eprint('Interrupted by user. Aborted.')
break
def main():
import argparse
import re
import sys
# Parse arguments
parser = argparse.ArgumentParser(
description='Signal Recorder for Migration from IBP Monitor-1')
parser.add_argument('-d',
'--debug',
action='store_true',
default=False,
help='enable debug')
parser.add_argument(
'--checklimit',
action='store_true',
default=False,
help='check if generated signal files are too many (it makes very slow'
)
parser.add_argument(
'--force',
action='store_true',
default=False,
help='ignore error even record and signal files already exist in' +
' database or directory')
parser.add_argument(
'-f',
'--from',
# required=True,
help='process from "new" (default), "today" (00:00:00 in UTC),'
' or datestr (e.g. 20171028)')
args = parser.parse_args()
args.arg_from = getattr(args, 'from')
# Check arguments
if args.arg_from == 'new':
pass
elif args.arg_from == 'today':
pass
elif args.arg_from is None:
args.arg_from = 'new'
else:
m = re.match(r'[0-9]{8}$', args.arg_from)
if not m:
eprint("Illegal datestr '%s' specified" % (args.arg_from))
sys.exit(1)
startrec(args.arg_from,
ignore_err=args.force,
check_limit=args.checklimit,
debug=args.debug)
def get_version():
# Check if version information is already cached
if not hasattr(get_version, 'version'):
try:
# Refer https://pe0fko.nl/SR-V9-Si570/
get_version.version = softrock_handle().controlRead(
usb1.TYPE_VENDOR | usb1.RECIPIENT_DEVICE | usb1.ENDPOINT_IN, 0,
0xe00, 0, 2, TIMEOUT)
except:
eprint('ERROR: Confirm SoftRock is attached to a USB port and also'
' you have a privilege\n to access the USB port.')
sys.exit(1)
# Notice that bytes are swapped
major = get_version.version[1]
minor = get_version.version[0]
return major, minor
def init_db(destroy='no', preserve=False, debug=False):
from lib.config import BeaconConfigParser
import os
if destroy != 'yes':
raise Exception('Not accepted by "yes"')
if not preserve:
try:
os.remove(BeaconConfigParser().getpath('Common', 'database'))
except OSError as err:
if err[1] != 'No such file or directory':
raise
conn = connect_database()
c = conn.cursor()
c.execute(SCHEMA_RECEIVED)
conn.commit()
conn.close()
eprint('Database is initialized and set up.')
def task_keeper(debug=False):
"""
Run all tasks listed in the config file and monitor them
"""
from lib.config import BeaconConfigParser
from multiprocessing import Process
from time import sleep
import sys
config = BeaconConfigParser()
logging.basicConfig(filename=config.getpath('TaskKeeper', 'logfile'))
task_names = config.get('TaskKeeper', 'tasks').split(',')
proc = {}
for task in task_names:
if task[0] == '@':
exit_return = True
task = task[1:]
else:
exit_return = False
exec 'from ' + task + ' import task as f'
proc[task] = Process(target=keep_task,
args=(eval('f'), task + '.task()', exit_return))
proc[task].start()
try:
spinner = spinning_cursor()
while True:
sys.stdout.write(spinner.next())
sys.stdout.write('\b')
sys.stdout.flush()
sleep(0.25)
except KeyboardInterrupt:
eprint('Interrupted by user. Aborted.')
for task in task_names:
if task[0] == '@':
task = task[1:]
proc[task].join()
def main():
import argparse
import re
import sys
# Parse arguments
parser = argparse.ArgumentParser(
description='Retrieve a 10-seconds .wav file')
parser.add_argument('-d',
'--debug',
action='store_true',
default=False,
help='enable debug')
parser.add_argument('--exactlen',
action='store_true',
default=False,
help='output exact length samples even data is short')
parser.add_argument('date', help='date string e.g. 20171028')
parser.add_argument('line',
type=int,
help='line# in the Monitor 1 .txt file')
parser.add_argument('output_file', help='output .wav file name')
args = parser.parse_args()
# Check arguments
m = re.match(r'[0-9]{8}$', args.date)
if not m:
eprint("Illegal date '%s' specified" % (args.date))
sys.exit(1)
if args.line < 1:
eprint("Illegal line '%d' specified" % (args.line))
sys.exit(1)
# Read signal data from raw file, and write it as .wav file
sig = retrieve_signal(args.date, args.line, debug=args.debug)
# Adjust signal length if required
if not args.exactlen:
sig = adjust_len(sig)
write_wav_file(args.output_file, sig)
def main():
import argparse
import re
import sys
# Parse arguments
parser = argparse.ArgumentParser(
description='Initialize (clear) database')
parser.add_argument('-d', '--debug',
action='store_true',
default=False,
help='enable debug')
parser.add_argument('--preserve',
action='store_true',
default=False,
help='do not erase database file')
parser.add_argument('agree',
help='say "agree" to initialize database')
args = parser.parse_args()
# Check arguments
m = re.match(r'agree$', args.agree)
if not m:
eprint('usage: "python initdb.py agree" to initialize (clear) database')
sys.exit(1)
# Ask again because can't undo
eprint("Final confirmation: database will be destroyed.")
s = raw_input("It's unrecoverable. Are you sure? (yes or no): ")
if s == 'yes':
init_db(destroy='yes', preserve=args.preserve, debug=args.debug)
else:
eprint('Aborted. (Not initialized.)')
def startrec_with_recover(check_limit=False, debug=False):
"""
Even startrec() failed, it will be relaunched
"""
global config
from time import sleep
import logging
from lib.config import BeaconConfigParser
config = BeaconConfigParser()
logging.basicConfig(filename='sigrec.log')
def datestr():
from datetime import datetime
return datetime.now().strftime('%Y-%m-%d %H:%M:%S')
while True:
try:
startrec(check_limit=check_limit, debug=debug)
eprint('startrec() exited at %s. Continued.' % (datestr()))
except KeyboardInterrupt:
eprint('Interrupted by user. Aborted.')
break
except:
eprint('startrec() raised an exception at %s. Continued.' % \
(datestr()))
logging.exception('startrec() at ' + datestr())
sleep(1)
def biashist_mig_band(dbconn, recorder, offset_ms, bfo_offset_hz, filename,
ignore_err=False):
"""
Read lines from given filename (Monitor-1 biashist file) and insert them as
database records.
"""
from lib.config import BeaconConfigParser
from lib.ibp import mhz_to_freq_khz
import re
import sqlite3
from datetime import datetime
m = re.search('_(20[0-9]+)\.log', filename)
date_str = m.group(1)
for line in open(filename, 'r').readlines():
if line.rstrip() == 'END':
break
if line.rstrip() == '':
eprint('Found empty line. Skipped')
continue
# Parsing characteristic parameters from *.log file
m = re.match(
'([0-9:]+) [A-Z0-9]+ +(\d+)MHz SN: *([\d.-]+) Bias: *([\d.-]+)'
+ ' Ct: *(\d+) IF: *([\d-]+) +([\d.-]+)',
line)
try:
datetime_sec = (datetime.strptime(
date_str + ' ' + m.group(1),
'%Y%m%d %H:%M:%S')
- datetime.utcfromtimestamp(0)).total_seconds()
except:
eprint('Found illegal line "%s". Aborted')
raise
freq_khz = mhz_to_freq_khz(int(m.group(2)))
max_sn = float(m.group(3))
best_pos_hz = int(m.group(4))
total_ct = int(m.group(5))
bg_pos_hz = int(m.group(6))
bg_sn = float(m.group(7))
# print datetime_sec, freq_khz, max_sn, best_pos_hz, total_ct
# print bg_pos_hz, bg_sn
# Originally, trying to calculate true time by comparing bad_slot and
# true slot.
# m = re.search(r'_([A-Z0-9]+)_', filename)
# callsign = m.group(1)
# bad_slot = get_slot(datetime_sec, band)
# true_slot = callsign_to_slot(callsign)
# diff = (bad_slot - true_slot) % 18
# if diff < 2 or diff > 3:
# # print bad_slot, callsign
# print diff
c = dbconn.cursor()
try:
c.execute('''INSERT INTO
received(datetime, offset_ms, freq_khz, bfo_offset_hz, recorder,
char1_max_sn, char1_best_pos_hz, char1_total_ct,
char1_bg_pos_hz, char1_bg_sn)
VALUES(?,?,?,?,?,?,?,?,?,?)''',
(
datetime_sec,
offset_ms,
freq_khz,
bfo_offset_hz,
recorder,
max_sn,
best_pos_hz,
total_ct,
bg_pos_hz,
bg_sn
))
except sqlite3.IntegrityError as err:
if not ignore_err:
raise
elif err[0] != 'UNIQUE constraint failed: biashist.datetime':
raise
dbconn.commit()
def output_signal(datetime_sec, samples, samplerate):
"""
Record (or 'convert' in the migration recorder case) one file from
the raw file specified by 'datestr' and 'line' in the file.
Note that the 'line' is true line number of the file. Comment line is also
counted. And return True.
Return false if signal file already existed.
"""
from lib.fileio import mkdir_if_required, getpath_signalfile
import os
import time
import wave
import numpy as np
import sys # XXX
# If length of samples are short, append zeros at the tail
expected_n_samples = samplerate * LEN_INPUT_SEC * 2 * 2 # 2 ch * S16_LE
if len(samples) < expected_n_samples:
samples.extend([0] * (expected_n_samples - len(samples)))
n_samples = len(samples) / 4
np.set_printoptions(edgeitems=1000000)
lrlag = config.getint('SignalRecorder', 'lrlag')
sig_iq = config.get('SignalRecorder', 'sig_iq')
filename = getpath_signalfile(
time.strftime('%Y%m%d/%H%M%S.wav', time.gmtime(datetime_sec)))
print filename
# filepath = getpath_signalfile(datestr + '/' + timestr + '.wav')
s = np.frombuffer(samples, dtype=np.dtype(np.int16))
s = s.reshape((n_samples, 2))
print len(s), s.shape
ch_L = s[:, 0]
ch_R = s[:, 1]
# Adjust lag if required
if lrlag > 0:
lag = lrlag
ch_R[0:n_samples - lag] = ch_R[lag:n_samples]
elif lrlag < 0:
lag = -lrlag
ch_L[0:n_samples - lag] = ch_L[lag:n_samples]
# XXX L/R from 12:33 JST Nov/20
# XXX R/L from 12:58 JST Nov/20 Lite9 good
# XXX L/R from 13:53 JST Nov/20 Lite9 bad
# XXX R/L from 14:56 JST Nov/20 with Ensemble III and back antenna: bad
# XXX R/L from 15:30 JST Nov/20 with Ensemble III and main antenna: good
# XXX R/L from 15:40 JST Nov/20 with Ensemble III and back antenna: bad
# XXX R/L from 16:18 JST Nov/20 with Ensemble III and main antenna:
# ch_I = ch_R # XXX L/R from 12:33 JST Nov/20
# ch_Q = ch_L # XXX
if sig_iq == 'L/R':
ch_I = ch_L
ch_Q = ch_R
elif sig_iq == 'R/L':
ch_I = ch_R
ch_Q = ch_L
else:
eprint('[SignalRecorder] sig_iq must be L/R or R/L')
raise Exception
out_samples = np.column_stack((ch_I, ch_Q)).flatten()
bytes = bytearray(out_samples)
mkdir_if_required(filename)
wavfile = wave.open(filename, 'wb')
wavfile.setnchannels(2)
wavfile.setsampwidth(2)
wavfile.setframerate(samplerate)
wavfile.writeframesraw(bytes)
wavfile.close()
return True
def charex(sigdata, samplerate, offset_ms, bfo_offset_hz, debug=False):
"""
Actually calculate characteristics of the signal record and return the
result.
"""
from scipy import signal
# np.set_printoptions(edgeitems=100)
if debug:
eprint(samplerate, offset_ms, bfo_offset_hz)
n_samples = samplerate * len_input_sec
# Generating an LPF
# Not sure if we can apply Nuttall window and also Hamming window which
# firwin() automatically applies. But as same as Beacon Monitor-1 code.
if 'lpf' not in dir(charex):
charex.lpf = signal.nuttall(n_filter_order + 1) \
* signal.firwin(n_filter_order + 1, lpf_cutoff)
# Generating an complex tone (f = samplerate / 4)
# XXX There are errors in latter elements but ignorable...
if 'tone_f_4' not in dir(charex):
charex.tone_f_4 = \
np.exp(1j * np.deg2rad(np.arange(0, 90 * n_samples, 90)))
if len(sigdata) != n_samples * n_channels * np.dtype(dtype).itemsize:
eprint('Length of sigdata (%d) is illegal' % (len(sigdata)))
return character1(-float('Inf'), 0, 0, 0, 0.0)
# Convert the sigdata (raw stream) to input complex vector
# It is okay that each I/Q value is 16-bit signed integer and as same as
# the original Beacon Monitor-1 libexec/proc.m (MATLAB implementation).
iq_matrix = np.frombuffer(sigdata, dtype=dtype).reshape((n_samples, 2))
input_vec = iq_matrix[..., 0] + 1j * iq_matrix[..., 1]
# input_vec is like this.
# [ 88.-30.j 87.-29.j 88.-27.j ..., -2. +4.j -2. +0.j -2. -1.j]
# print input_vec, len(input_vec)
# Apply LPF to narrow band width to half, and remove preceding samples
# as same as Monitor-1
sig = signal.lfilter(charex.lpf, 1,
np.append(input_vec, np.zeros(n_filter_order / 2)))
sig = sig[n_filter_order / 2 : ]
# Applying tone (f = samplerate / 4) to shift signal upward on freq. domain
sig *= charex.tone_f_4
# Drop imaginary parts as same as Monitor-1
sig = np.real(sig)
# print sig, len(sig)
# Background noise estimation
bg, bg_smooth = bg_est(sig, samplerate, offset_ms)
# Now, start analysis in signal parts of time domain
max_sn = -np.inf
best_pos = 0
ct_pos = np.zeros(wid_freq_detect, dtype=np.int16)
for n in range(sec_sg_from, sec_sg_until):
start = n * samplerate - offset_ms / 1000 * samplerate
lvl, pos, sn = \
sg_est(sig, bg_smooth, start, samplerate, offset_ms, canceling=True)
ct_pos[pos] += 1
if sn > max_sn:
max_sn = sn
best_pos = pos
# Calculating values in the no-signal part (beginning part)
bg_len = get_bg_len(offset_ms, samplerate)
bg_lvl, bg_pos, bg_sn = \
sg_est(sig, bg_smooth, 0, bg_len, offset_ms, canceling=False)
# print 'bg_pos', bg_pos
# print 'bg_len', bg_len
lower_pos, upper_pos, dummy = get_sig_bins(best_pos)
# print lower_pos, upper_pos, ct_pos
# print ct_pos[lower_pos : upper_pos + 1]
total_ct = sum(ct_pos[lower_pos : upper_pos + 1])
if debug:
print 'SN: %4.1f Bias: %4d Ct: %d IF: %4d %4.1f Z: -1 -1.0' % \
(max_sn, bin_to_freq(best_pos), total_ct, bin_to_freq(bg_pos),
bg_sn)
return character1(max_sn, bin_to_freq(best_pos), total_ct,
bin_to_freq(bg_pos), bg_sn)
def startrec(arg_from, ignore_err=False, check_limit=False, debug=False):
"""
Repeat signal conversion from 'arg_from'.
If arg_from is 'new', it start from new line of today's file.
Otherwise, treats arg_from as datestr (e.g. 20171028)
"""
from datetime import datetime
from lib.fileio import open_db_file
import math
import re
import time
if arg_from == 'new':
datestr = datetime.utcnow().strftime('%Y%m%d')
seek_to_tail = True
elif arg_from == 'today':
datestr = datetime.utcnow().strftime('%Y%m%d')
seek_to_tail = False
else:
datestr = arg_from
seek_to_tail = False
curfd = None
curline = 0 # will be initialized in the loop below in anyway
while True:
# For first iteration or curfd is closed, open a new file
if curfd is None:
curfd = open_db_file('ibprec_%s.txt' % (datestr), 'r')
curline = 0
# If seek_to_tail is True, it means starting from new line (next of the
# current last line, so skip already existing lines.
if seek_to_tail == True:
seek_to_tail = False # never seek again
if debug:
print "Skipping to tail of the file..."
while True:
line = curfd.readline()
if line == '':
break # no more lines
curline += 1
# Now, wait for a line from curfd
while True:
line = curfd.readline()
if line == '':
time.sleep(0.5) # sleep for a short period
else:
curline += 1
break
line = line.rstrip()
# Check if the line is a comment-only line
m = re.match(r' *#', line)
if m:
if debug:
print "COMMENT:", line
continue
# Check if the line is an end-of-file marker
m = re.search(r'MHz\s*$', line)
if not m:
if debug:
print "TERMINATED:", line
curfd.close()
curfd = None
curline = 0
datestr = nextday_datestr(datestr)
continue
# This is not required. %H:%M:%S is included in the line...
# m = re.match(r'\d+', line)
# utctime = datetime.utcfromtimestamp(
# math.floor((float(m.group(0)) + 6.0) / 10.0) * 10.0)
# Extract time time string from the line, and convert to %H%M%S.
m = re.search(r' (\d{2}):(\d{2}):(\d{2}) ', line)
timestr = m.group(1) + m.group(2) + m.group(3)
# Extract frequency (kHz) from the line
m = re.search(r' (\d+)MHz\s*$', line)
mhz = int(m.group(1))
if check_limit:
# Check if generated signal files are too much
if exceeded_sigfiles_limit():
eprint("Signal files limit (number of size) is exceeded.")
eprint("Waiting until not meeting the condition again.")
while exceeded_sigfiles_limit():
time.sleep(0.5)
# Finally process the line
status1 = record_one_file(datestr, timestr, curline, ignore_err, debug)
status2 = register_db(datestr, timestr, mhz, ignore_err, debug=debug)
# If some processes were required, show the progress
if status1 or status2:
print line