def write_wav_file(filename, data, to_signal_dir=False):
"""
Convert raw signal data 'data' to .wav format and write to 'filename'.
If to_signal_dir is True, written to the standard direcotry.
"""
from lib.config import BeaconConfigParser
from lib.fileio import mkdir_if_required, getpath_signalfile
import os
import wave
# First, we need to take care of PCM2902 lag between L and R channels
iqlag = BeaconConfigParser().getint('Migration', 'iqlag')
if iqlag > 0:
i_ch_lag = iqlag
q_ch_lag = 0
else:
i_ch_lag = 0
q_ch_lag = -iqlag
n_samples = len(data) / 4 # I_low, I_high, Q_low, Q_high, ...
# Take account lag and split to I/Q
def limit(val, maxval):
if val > maxval:
val = maxval
return val
def substr(s, start, len):
return s[start:start + len]
i_ch = {}
q_ch = {}
for i in range(n_samples):
i_pos = limit(i + i_ch_lag, n_samples - 1)
q_pos = limit(i + q_ch_lag, n_samples - 1)
i_ch[i] = substr(data, ((i_pos * 2 + 0) * 2), 2) # I comes first
q_ch[i] = substr(data, ((q_pos * 2 + 1) * 2), 2) # I comes first
# Reconstruct stream from I/Q
data = ''
for i in range(n_samples):
data += i_ch[i]
data += q_ch[i]
if to_signal_dir:
filename = getpath_signalfile(filename)
mkdir_if_required(filename)
# Read parameter samplerate from config file
samplerate = BeaconConfigParser().getint('Signal', 'samplerate')
wavfile = wave.open(filename, 'wb')
wavfile.setnchannels(2)
wavfile.setsampwidth(2)
wavfile.setframerate(samplerate)
wavfile.writeframesraw(data)
wavfile.close()
def startrec_with_recover(check_limit=False, debug=False):
"""
Even startrec() failed, it will be relaunched
"""
global config
from time import sleep
import logging
from lib.config import BeaconConfigParser
config = BeaconConfigParser()
logging.basicConfig(filename='sigrec.log')
def datestr():
from datetime import datetime
return datetime.now().strftime('%Y-%m-%d %H:%M:%S')
while True:
try:
startrec(check_limit=check_limit, debug=debug)
eprint('startrec() exited at %s. Continued.' % (datestr()))
except KeyboardInterrupt:
eprint('Interrupted by user. Aborted.')
break
except:
eprint('startrec() raised an exception at %s. Continued.' % \
(datestr()))
logging.exception('startrec() at ' + datestr())
sleep(1)
def register_db(datestr, timestr, mhz, ignore_err=False, debug=False):
"""
Register record information to database and return True.
Return false if a duplicate record was found.
"""
from datetime import datetime
from lib.config import BeaconConfigParser
from lib.fileio import connect_database
from lib.ibp import mhz_to_freq_khz
from sqlite3 import IntegrityError
# Convert datestr and timestr to seconds from epoch
datetime_utc = datetime.strptime(datestr + ' ' + timestr, '%Y%m%d %H%M%S')
seconds_from_epoch = int(
(datetime_utc - datetime.utcfromtimestamp(0)).total_seconds())
if debug:
print "seconds_from_epoch:", seconds_from_epoch
if seconds_from_epoch % 10 != 0:
raise Exception('seconds_from_epoch is not multiple of 10 seconds')
# Obtain parameters from configuration
config = BeaconConfigParser()
conn = connect_database()
c = conn.cursor()
err_occurred = False
try:
c.execute(
'''INSERT INTO
received(datetime, offset_ms, freq_khz, bfo_offset_hz, recorder)
VALUES(?,?,?,?,?)''',
(seconds_from_epoch, config.getint('Migration', 'offset_ms'),
mhz_to_freq_khz(mhz), config.getint('Migration', 'bfo_offset_hz'),
config.get('Migration', 'recorder')))
conn.commit()
except IntegrityError as err:
if ignore_err and \
err[0] == 'UNIQUE constraint failed: received.datetime':
err_occurred = True
else:
raise
conn.close()
return not err_occurred
def task_keeper(debug=False):
"""
Run all tasks listed in the config file and monitor them
"""
from lib.config import BeaconConfigParser
from multiprocessing import Process
from time import sleep
import sys
config = BeaconConfigParser()
logging.basicConfig(filename=config.getpath('TaskKeeper', 'logfile'))
task_names = config.get('TaskKeeper', 'tasks').split(',')
proc = {}
for task in task_names:
if task[0] == '@':
exit_return = True
task = task[1:]
else:
exit_return = False
exec 'from ' + task + ' import task as f'
proc[task] = Process(target=keep_task,
args=(eval('f'), task + '.task()', exit_return))
proc[task].start()
try:
spinner = spinning_cursor()
while True:
sys.stdout.write(spinner.next())
sys.stdout.write('\b')
sys.stdout.flush()
sleep(0.25)
except KeyboardInterrupt:
eprint('Interrupted by user. Aborted.')
for task in task_names:
if task[0] == '@':
task = task[1:]
proc[task].join()
def charex_all(onepass=False, force=False, dryrun=False, debug=False):
"""
Retrieve any record in the database, which doesn't have calculated
characteristics by this charex.py yet, and pass them to charex()
"""
global config
from lib.fileio import connect_database
import time
from lib.config import BeaconConfigParser
config = BeaconConfigParser()
conn = connect_database()
while True:
c = conn.cursor()
# If specified 'force', even the record has characteristics parameters,
# fetch any records for update.
if force:
cond = ''
else:
cond = 'WHERE char1_max_sn IS NULL'
c.execute('''SELECT datetime, offset_ms, bfo_offset_hz
FROM received
%s
ORDER BY datetime''' % (cond))
for row in c.fetchall():
try:
sigdata, samplerate = read_sigdata(datetime_sec=row[0])
paramset = charex(
sigdata, samplerate, row[1], row[2], debug=debug)
if not dryrun:
paramset.updatedb(conn, row[0])
except IOError as err:
if err[1] == 'No such file or directory':
if debug:
pass
# eprint('Signal file not found. Skipped')
else:
raise
if onepass:
break
else:
# For continuous passes, 'force fetch' is NOT required
force = False
# To let rest database, wait for a short time period
time.sleep(0.5)
conn.close()
def exceeded_sigfiles_limit():
"""
Check if too many signal files are generated and return True if so.
"""
from lib.config import BeaconConfigParser
import re
prog = re.compile(r'\.wav$')
ct = 0
for root, dirs, files in \
os.walk(BeaconConfigParser().getpath('Signal', 'dir')):
for f in files:
if prog.search(f):
ct += 1
if ct >= BeaconConfigParser().getint('SignalRecorder',
'sigfiles_num_limit'):
return True
else:
return False
def biashist_mig_all(ignore_err=False, debug=False):
from lib.config import BeaconConfigParser
from lib.fileio import connect_database
from fnmatch import fnmatch
import os
dbdir = BeaconConfigParser().getpath('Migration', 'dbdir')
recorder = BeaconConfigParser().get('Migration', 'recorder')
offset_ms = BeaconConfigParser().getint('Migration', 'offset_ms')
bfo_offset_hz = \
BeaconConfigParser().getint('Migration', 'bfo_offset_hz')
conn = connect_database()
for file in sorted(os.listdir(dbdir)):
if fnmatch(file, 'ibprec_*.log'):
if debug:
print "Migrating", file
biashist_mig_band(conn, recorder, offset_ms, bfo_offset_hz,
os.path.join(dbdir, file), ignore_err=ignore_err)
conn.close()
def init_db(destroy='no', preserve=False, debug=False):
from lib.config import BeaconConfigParser
import os
if destroy != 'yes':
raise Exception('Not accepted by "yes"')
if not preserve:
try:
os.remove(BeaconConfigParser().getpath('Common', 'database'))
except OSError as err:
if err[1] != 'No such file or directory':
raise
conn = connect_database()
c = conn.cursor()
c.execute(SCHEMA_RECEIVED)
conn.commit()
conn.close()
eprint('Database is initialized and set up.')
def record_one_file(datestr, timestr, line, skip_if_exist=False, debug=False):
"""
Record (or 'convert' in the migration recorder case) one file from
the raw file specified by 'datestr' and 'line' in the file.
Note that the 'line' is true line number of the file. Comment line is also
counted. And return True.
Return false if signal file already existed.
"""
from lib.config import BeaconConfigParser
from lib.fileio import getpath_signalfile
from sigretr import retrieve_signal, write_wav_file, adjust_len
import os
import wave
if not hasattr(record_one_file, 'n_samples'):
record_one_file.n_samples = \
LEN_INPUT_SEC * BeaconConfigParser().getint('Signal', 'samplerate')
filename = datestr + '/' + timestr + '.wav'
filepath = getpath_signalfile(datestr + '/' + timestr + '.wav')
# If the signal file exists and can be ignored, skip file retrieval
try:
if skip_if_exist and \
wave.open(filepath, 'rb').getnframes() == \
record_one_file.n_samples:
return False
except IOError as err:
if err[1] == 'No such file or directory':
# File does not exist...
pass
else:
raise
except:
raise
# Read signal data from raw file, and write it as .wav file
sig = retrieve_signal(datestr, line, debug=False)
sig = adjust_len(sig)
write_wav_file(filename, sig, to_signal_dir=True)
return True
def getpath_signalfile(filename):
"""
Return the actual path name of signal file by given filename
"""
sigdir = BeaconConfigParser().getpath('Signal', 'dir')
return os.path.join(sigdir, filename)
def connect_database():
import sqlite3
database_file = BeaconConfigParser().getpath('Common', 'database')
mkdir_if_required(database_file)
return sqlite3.connect(database_file)
def open_db_file(name, mode=None):
dbdir = BeaconConfigParser().getpath('Migration', 'dbdir')
return open(os.path.join(dbdir, name), mode)
def __init__(self):
from lib.config import BeaconConfigParser
import yaml
yamlfile = BeaconConfigParser().get('Common', 'stations')
self.stations = yaml.load(open(yamlfile, 'r'))
def set_freq(freq_hz, dryrun=False, debug=False):
"""
Set SoftRock to the specified freq_hz
If attached SoftRock has old firmware, we need special cares.
"""
from lib.config import BeaconConfigParser
import math
bytes = [0, 0, 0, 0]
if debug:
print 'freq_hz = %d' % (freq_hz)
if get_version()[0] < 15:
# This came from the Monitor-1 code. I don't know why this calculation
# is required because firmware 14.0 documentation is missing.
CALIB = 2.0962539700083447 # 2013-09-16 27.9 deg
v = float(freq_hz)
v *= CALIB
v *= 4 # Firmware 14.0 requires this
ival = int(math.floor(v + 0.5))
else:
calib = eval(BeaconConfigParser().get('SignalRecorder',
'calib_softrock'))
if debug:
print 'calib = %g' % (calib)
# Referred http://www.n8mdp.com/sdrLinux.php and usbsoftrock source code
# on the site
MULT = 4
freq_hz *= (1.0 + calib) / 1e6 * (1 << 21) * MULT
ival = int(freq_hz + 0.5)
if debug:
print 'ival = %d' % (ival)
bytes[0] = ival & 0xff
bytes[1] = (ival >> 8) & 0xff
bytes[2] = (ival >> 16) & 0xff
bytes[3] = (ival >> 24) & 0xff
if debug:
print 'bytes =', bytes
if not dryrun:
softrock_handle().controlWrite(
usb1.TYPE_VENDOR | usb1.RECIPIENT_DEVICE | usb1.ENDPOINT_OUT, 0x32,
0x700 + I2C_ADDR, 0, bytes, TIMEOUT)
if get_version()[0] < 15:
# This version may not have BPF automatic setting.
if debug:
print 'This SoftRock may not have BPF automatic setting.'
print 'Using a manual way.'
if freq_hz < 4000000:
bpf = 0
elif freq_hz < 8000000:
bpf = 1
elif freq_hz < 16000000:
bpf = 2
else:
bpf = 3
if debug:
print 'bpf = %d' % (bpf)
# This came from the Monitor-1 code.
bytes = [0] * 8
if not dryrun:
softrock_handle().controlWrite(
usb1.TYPE_VENDOR | usb1.RECIPIENT_DEVICE | usb1.ENDPOINT_IN, 4,
(bpf & 0x3) << 4, 0, bytes, 1000)
def cleanup(debug=False):
"""
Search signal files and remove it if it's old
"""
from datetime import datetime
from lib.config import BeaconConfigParser
from lib.fileio import connect_database
import os
import re
config = BeaconConfigParser()
files_path = config.getpath('Signal', 'dir')
timelimit_sec = eval(config.get('Cleaner', 'timelimit_sec'))
if debug:
print 'timelimit_sec = %d' % (timelimit_sec)
conn = connect_database()
c = conn.cursor()
for date_dir in os.listdir(files_path):
if not re.match(r'[0-9]{8}$', date_dir):
continue
# Now found a date directory
date_dir_path = os.path.join(files_path, date_dir)
for file in os.listdir(date_dir_path):
m = re.match(r'([0-9]{6})\.wav$', file)
if not m:
continue
time_str = m.group(1)
datetime_sec = int((
datetime.strptime(date_dir + ' ' + time_str, '%Y%m%d %H%M%S') -
datetime.utcfromtimestamp(0)).total_seconds())
# print date_dir, time_str, datetime_sec
c.execute(
'''SELECT datetime
FROM received
WHERE datetime == ? AND char1_max_sn IS NOT NULL''',
(datetime_sec, ))
row = c.fetchone()
# If the signal files hasn't have characteristics in database,
# it must be skipped.
if row is None:
continue
# If the file is too old, now we can remove the signal file
# print row
sec_diff = int(
(datetime.utcnow() -
datetime.utcfromtimestamp(0)).total_seconds()) - datetime_sec
# print sec_diff
if sec_diff > timelimit_sec:
rm_filename = os.path.join(files_path, date_dir, file)
if debug:
print 'Removing file %s' % (rm_filename)
os.remove(rm_filename)
# If the date directory is empty, remove the directory
if os.listdir(date_dir_path) == []:
if debug:
print 'Removing directory %s' % (date_dir_path)
os.rmdir(date_dir_path)
return
def graphserv(debug=False):
from lib.config import BeaconConfigParser
port = BeaconConfigParser().getint('GraphServ', 'port')
app.run(host='0.0.0.0', port=port)