def setUp(self):
super().setUp()
self.validator = Validator()
self.validator.flush_names()
logging.getLogger().setLevel(logging.CRITICAL)
FileUtil.safe_delete_dir('in')
FileUtil.safe_delete_dir('out')
FileUtil.safe_create_dir('in')
FileUtil.safe_create_dir('out')
cols = dict()
cols['name'] = 'National 2m'
cols['medium_name'] = 'Natl 2m'
cols['short_name'] = 'NATL 2M'
cols['zone_id'] = ''
cols['rx_freq'] = '146.52'
cols['rx_ctcss'] = ''
cols['rx_dcs'] = ''
cols['rx_dcs_invert'] = ''
cols['tx_power'] = 'High'
cols['tx_offset'] = ''
cols['tx_ctcss'] = ''
cols['tx_dcs'] = ''
cols['tx_dcs_invert'] = ''
cols['digital_timeslot'] = ''
cols['digital_color'] = ''
cols['digital_contact_id'] = ''
cols['latitude'] = ''
cols['longitude'] = ''
self.radio_cols = cols
def __init__(
self,
radio_list,
):
self.radio_list = radio_list
self._validator = Validator()
self._migrations = MigrationManager()
class RadioImportChirp:
def __init__(self):
self.validator = Validator()
self.validator.flush_names()
def run_import(self, cols, import_path):
radio_channel_cols = RadioChannel.generate_empty_dict()
radio_channel_cols['name'] = cols['Name']
radio_channel_cols['medium_name'] = cols['Name']
radio_channel_cols['short_name'] = cols['Name']
radio_channel_cols['zone_id'] = ''
radio_channel_cols['rx_freq'] = cols['Frequency']
radio_channel_cols['rx_ctcss'] = ''
if cols['Tone'] == 'Tone' or cols['Tone'] == 'TSQL':
radio_channel_cols['tx_ctcss'] = cols['rToneFreq']
if cols['Tone'] == 'TSQL':
radio_channel_cols['rx_ctcss'] = cols['cToneFreq']
if cols['Tone'] == 'DTCS':
radio_channel_cols['rx_dcs'] = cols['DtcsCode']
radio_channel_cols['tx_dcs'] = cols['DtcsCode']
if cols['DtcsPolarity'][0] == 'R':
radio_channel_cols['rx_dcs_invert'] = 'True'
if cols['DtcsPolarity'][1] == 'R':
radio_channel_cols['tx_dcs_invert'] = 'True'
radio_channel_cols['tx_power'] = 'High'
if float(cols['Offset']) != 0:
neg = 1
if cols['Duplex'] == '-':
neg = -1
radio_channel_cols['tx_offset'] = float(cols['Offset']) * neg
line_num = int(cols['Location']) + 1
errors = self.validator.validate_radio_channel(radio_channel_cols,
line_num, import_path,
None, None)
for err in errors:
logging.error(
f'\t\tline:{err.line_num} validation error: {err.message}')
raise ValidationError("Import line failed to parse.", line_num,
import_path)
channel = RadioChannelChirp(radio_channel_cols, None, None)
return channel
def test_validate_files_exist(self): files = ['input.csv', 'digital_contacts.csv', 'dmr_id.csv', 'zones.csv', 'user.csv'] for filename in files: f = PathManager.open_input_file(filename, 'w+') f.close() errors = Validator.validate_files_exist() self.assertEqual(0, len(errors))
def test_only_some_files_exist(self):
f = PathManager.open_input_file('input.csv', 'w+')
f.close()
errors = Validator.validate_files_exist()
self.assertEqual(4, len(errors))
def test_validate_no_files_exist(self): errors = Validator.validate_files_exist() self.assertEqual(5, len(errors))
class ValidatorTest(BaseTestSetup):
def setUp(self):
super().setUp()
self.validator = Validator()
self.validator.flush_names()
logging.getLogger().setLevel(logging.CRITICAL)
FileUtil.safe_delete_dir('in')
FileUtil.safe_delete_dir('out')
FileUtil.safe_create_dir('in')
FileUtil.safe_create_dir('out')
cols = dict()
cols['name'] = 'National 2m'
cols['medium_name'] = 'Natl 2m'
cols['short_name'] = 'NATL 2M'
cols['zone_id'] = ''
cols['rx_freq'] = '146.52'
cols['rx_ctcss'] = ''
cols['rx_dcs'] = ''
cols['rx_dcs_invert'] = ''
cols['tx_power'] = 'High'
cols['tx_offset'] = ''
cols['tx_ctcss'] = ''
cols['tx_dcs'] = ''
cols['tx_dcs_invert'] = ''
cols['digital_timeslot'] = ''
cols['digital_color'] = ''
cols['digital_contact_id'] = ''
cols['latitude'] = ''
cols['longitude'] = ''
self.radio_cols = cols
def test_validate_no_files_exist(self):
errors = Validator.validate_files_exist()
self.assertEqual(5, len(errors))
def test_validate_files_exist(self):
files = ['input.csv', 'digital_contacts.csv', 'dmr_id.csv', 'zones.csv', 'user.csv']
for filename in files:
f = PathManager.open_input_file(filename, 'w+')
f.close()
errors = Validator.validate_files_exist()
self.assertEqual(0, len(errors))
def test_only_some_files_exist(self):
f = PathManager.open_input_file('input.csv', 'w+')
f.close()
errors = Validator.validate_files_exist()
self.assertEqual(4, len(errors))
def test_validate_radio_channel_name_dupe(self):
errors = self.validator.validate_radio_channel(self.radio_cols, 1, 'FILE_NO_EXIST_UNITTEST', {}, {})
self.assertEqual(len(errors), 0)
errors = self.validator.validate_radio_channel(self.radio_cols, 2, 'FILE_NO_EXIST_UNITTEST', {}, {})
self.assertEqual(len(errors), 3)
short_found = False
medium_found = False
long_found = False
for err in errors:
short_found = err.args[0].find('Collision in short_name') or short_found
medium_found = err.args[0].find('Collision in medium_name') or medium_found
long_found = err.args[0].find('Collision in name') or long_found
self.assertTrue(short_found)
self.assertTrue(medium_found)
self.assertTrue(long_found)
def test_validate_missing_contact(self):
radio_cols = dict()
radio_cols['number'] = '1'
radio_cols['name'] = 'Test channel'
radio_cols['medium_name'] = 'TestChan'
radio_cols['short_name'] = 'TestChn'
radio_cols['zone_id'] = ''
radio_cols['rx_freq'] = '146.52'
radio_cols['rx_ctcss'] = ''
radio_cols['rx_dcs'] = ''
radio_cols['rx_dcs_invert'] = ''
radio_cols['tx_power'] = 'High'
radio_cols['tx_offset'] = '0.6'
radio_cols['tx_ctcss'] = ''
radio_cols['tx_dcs'] = ''
radio_cols['tx_dcs_invert'] = ''
radio_cols['digital_timeslot'] = '1'
radio_cols['digital_color'] = '2'
radio_cols['digital_contact_id'] = '314'
radio_cols['latitude'] = ''
radio_cols['longitude'] = ''
digital_contact_cols = dict()
digital_contact_cols['number'] = '1'
digital_contact_cols['digital_id'] = '314'
digital_contact_cols['name'] = 'Digi Contact'
digital_contact_cols['call_type'] = 'Group'
digital_contacts = {
314: DmrContact(digital_contact_cols),
}
errors = self.validator.validate_radio_channel(radio_cols, 1, 'FILE_NO_EXIST_UNITTEST', digital_contacts, {})
self.assertEqual(len(errors), 0)
self.validator.flush_names()
errors = self.validator.validate_radio_channel(radio_cols, 1, 'FILE_NO_EXIST_UNITTEST', {}, {})
self.assertEqual(len(errors), 1)
found = errors[0].args[0].find('Cannot find digital contact')
self.assertEqual(found, 0)
def test_ignore_extra_column(self):
self.radio_cols['foo'] = '1'
errors = self.validator.validate_radio_channel(self.radio_cols, 1, 'FILE_NO_EXIST_UNITTEST', {}, {})
self.assertEqual(len(errors), 0)
def test_validate_tx_power(self):
self.radio_cols['tx_power'] = 'mega'
errors = self.validator.validate_radio_channel(self.radio_cols, 1, 'FILE_NO_EXIST_UNITTEST', {}, {})
self.assertEqual(len(errors), 1)
found = errors[0].args[0].find('Transmit power (`tx_power`) invalid')
self.assertEqual(found, 0)
def test_validate_tx_power_not_present(self):
self.radio_cols['tx_power'] = ''
errors = self.validator.validate_radio_channel(self.radio_cols, 1, 'FILE_NO_EXIST_UNITTEST', {}, {})
self.assertEqual(len(errors), 1)
found = errors[0].args[0].find('Transmit power (`tx_power`) invalid')
self.assertEqual(found, 0)
def test_validate_zone_not_present(self):
self.radio_cols['zone_id'] = '1'
errors = self.validator.validate_radio_channel(self.radio_cols, 1, 'FILE_NO_EXIST_UNITTEST', {}, {})
self.assertEqual(len(errors), 1)
found = errors[0].args[0].find('Zone ID not found:')
self.assertEqual(found, 0)
def test_validate_zone_present(self):
self.radio_cols['zone_id'] = '1'
zone = RadioZone({
'number': 1,
'name': 'Zone 1',
})
zones = {1: zone}
errors = self.validator.validate_radio_channel(self.radio_cols, 1, 'FILE_NO_EXIST_UNITTEST', {}, zones)
self.assertEqual(len(errors), 0)
def test_validate_lat_long_present(self):
self.radio_cols['latitude'] = '0.0'
self.radio_cols['longitude'] = '0.0'
errors = self.validator.validate_radio_channel(self.radio_cols, 1, 'LAT_LONG_UNITTEST', {}, {})
self.assertEqual(len(errors), 0)
def test_validate_lat_long_not_present(self):
self.radio_cols['latitude'] = ''
self.radio_cols['longitude'] = ''
errors = self.validator.validate_radio_channel(self.radio_cols, 1, 'LAT_LONG_UNITTEST', {}, {})
self.assertEqual(len(errors), 0)
def test_validate_lat_only_present(self):
self.radio_cols['latitude'] = '0.0'
self.radio_cols['longitude'] = ''
errors = self.validator.validate_radio_channel(self.radio_cols, 1, 'LAT_LONG_UNITTEST', {}, {})
self.assertEqual(len(errors), 1)
found = errors[0].args[0].find('Only one of latitude or longitude provided')
self.assertEqual(found, 0)
def test_validate_long_only_present(self):
self.radio_cols['latitude'] = ''
self.radio_cols['longitude'] = '0.0'
errors = self.validator.validate_radio_channel(self.radio_cols, 1, 'LAT_LONG_UNITTEST', {}, {})
self.assertEqual(len(errors), 1)
found = errors[0].args[0].find('Only one of latitude or longitude provided')
self.assertEqual(found, 0)
def test_validate_bad_lat(self):
self.radio_cols['latitude'] = '-91.0'
self.radio_cols['longitude'] = '0.0'
errors = self.validator.validate_radio_channel(self.radio_cols, 1, 'LAT_LONG_UNITTEST', {}, {})
self.assertEqual(len(errors), 1)
found = errors[0].args[0].find('Latitude must be between')
self.assertEqual(found, 0)
def test_validate_bad_long(self):
self.radio_cols['latitude'] = '0.0'
self.radio_cols['longitude'] = '-181.0'
errors = self.validator.validate_radio_channel(self.radio_cols, 1, 'LAT_LONG_UNITTEST', {}, {})
self.assertEqual(len(errors), 1)
found = errors[0].args[0].find('Longitude must be between')
self.assertEqual(found, 0)
class RadioGenerator:
def __init__(
self,
radio_list,
):
self.radio_list = radio_list
self._validator = Validator()
self._migrations = MigrationManager()
@classmethod
def info(cls, dangerous_ops_info):
logging.info(f'''
HAM RADIO SYNC GENERATOR v{radio_sync_version.version}
Homepage: https://github.com/n2qzshce/ham-radio-sync
Purpose: The intent of this program is to generate codeplug files to import into various radio applications by
using a master set of csv files that have all the relevant information.
How to use: Start by running the Wizard to generate the `in` directory and `out` directory.
For more information about these files, please see:
https://github.com/n2qzshce/ham-radio-sync/blob/master/INPUTS_OUTPUTS_SYNCING.md
{dangerous_ops_info}
Functions:
(Dangerous Operation) Cleanup - Will delete the contents of your `in` and `out` directory.
(Dangerous Operation) Wizard - Creates sample input files to help you get started.
(Dangerous Operation) Migrate - If you have recently updated ham radio sync, this can add any new columns that
may be needed. Migrations will rename your existing files by adding a `.bak` extension. You can run the
'migrations cleanup' to remove these files.
Create radio plugs - Will generate CSVs to import for the radios you have selected.
Debug logging - This will enable chattier logging. Generally only needed if you have been instructed to do so.
''')
def generate_all_declared(self):
file_errors = self._validator.validate_files_exist()
self._validator.flush_names()
if len(file_errors) > 0:
return False
results = self._migrations.check_migrations_needed()
if len(results.keys()) > 0:
logging.warning(
'You may be using an old version of the input files. Have you run migrations?'
)
logging.warning("Migrations check is under the 'File' menu.")
sleep(1)
for radio in radio_types.radio_choices():
radio_folder = PathManager.get_output_path(radio)
if not os.path.exists(radio_folder):
continue
logging.info(f'Deleting old output folder `{radio}`')
FileUtil.safe_delete_dir(radio_folder)
digital_contacts, digi_contact_errors = self._generate_digital_contact_data(
)
dmr_ids, dmr_id_errors = self._generate_dmr_id_data()
zones, zone_errors = self._generate_zone_data()
user, user_data_errors = self._generate_user_data()
preload_errors = digi_contact_errors + dmr_id_errors + zone_errors + user_data_errors
feed = PathManager.open_input_file('input.csv', 'r')
csv_reader = csv.DictReader(feed)
radio_channel_errors = []
radio_channels = []
line_num = 1
for line in csv_reader:
line_errors = self._validator.validate_radio_channel(
line, line_num, feed.name, digital_contacts, zones)
radio_channel_errors += line_errors
line_num += 1
if len(line_errors) > 0:
continue
radio_channel = RadioChannel(line, digital_contacts, dmr_ids)
radio_channels.append(radio_channel)
if radio_channel.zone_id.fmt_val(None) is not None:
zones[radio_channel.zone_id.fmt_val()].add_channel(
radio_channel)
feed.close()
all_errors = preload_errors + radio_channel_errors
if len(all_errors) > 0:
logging.error('--- VALIDATION ERRORS, CANNOT CONTINUE ---')
for err in all_errors:
logging.error(
f'\t\tfile: `{err.file_name}` line:{err.line_num} validation error: {err.message}'
)
return False
else:
logging.info(
'File validation complete, no obvious formatting errors found')
radio_files = dict()
headers_gen = RadioChannel.create_empty()
FileUtil.safe_create_dir('out')
channel_numbers = dict()
for radio in self.radio_list:
radio_casted = RadioChannelBuilder.casted(headers_gen, radio)
FileUtil.safe_create_dir(f'out/{radio}')
logging.info(f'Generating for radio type `{radio}`')
if radio_casted.skip_radio_csv():
logging.info(
f'`{radio}` uses special output style. Skipping channels csv'
)
continue
output = RadioWriter.output_writer(f'{radio}/{radio}_channels.csv',
'\r\n')
file_headers = radio_casted.headers()
output.writerow(file_headers)
radio_files[radio] = output
channel_numbers[radio] = 1
logging.info('Processing radio channels')
line = 1
for radio_channel in radio_channels:
logging.debug(f'Processing radio line {line}')
if line % file_util.RADIO_LINE_LOG_INTERVAL == 0:
logging.info(f'Processing radio line {line}')
line += 1
for radio in self.radio_list:
if radio not in radio_files.keys():
continue
if not radio_types.supports_dmr(
radio) and radio_channel.is_digital():
continue
casted_channel = RadioChannelBuilder.casted(
radio_channel, radio)
input_data = casted_channel.output(channel_numbers[radio])
radio_files[radio].writerow(input_data)
channel_numbers[radio] += 1
additional_data = RadioAdditional(radio_channels, dmr_ids,
digital_contacts, zones, user)
for radio in self.radio_list:
if radio in radio_files.keys():
radio_files[radio].close()
casted_additional_data = RadioAdditionalBuilder.casted(
additional_data, radio)
casted_additional_data.output()
logging.info(f'''Radio generator complete. Your output files are in
`{os.path.abspath('out')}`
The next step is to import these files into your radio programming application. (e.g. CHiRP)'''
)
return True
def _generate_digital_contact_data(self):
logging.info('Processing digital contacts')
feed = PathManager.open_input_file('digital_contacts.csv', 'r')
csv_feed = csv.DictReader(feed)
digital_contacts = dict()
errors = []
line_num = 1
for line in csv_feed:
logging.debug(f'Processing line {line_num}: `{line}`')
line_errors = self._validator.validate_digital_contact(
line, 1, feed.name)
errors += line_errors
line_num += 1
if len(line_errors) != 0:
continue
contact = DmrContact(line)
digital_contacts[contact.digital_id.fmt_val()] = contact
feed.close()
return digital_contacts, errors
def _generate_dmr_id_data(self):
logging.info('Processing dmr ids')
feed = PathManager.open_input_file('dmr_id.csv', 'r')
csv_feed = csv.DictReader(feed)
dmr_ids = dict()
errors = []
line_num = 0
for line in csv_feed:
logging.debug(f'Processing line {line_num}: `{line}`')
line_num += 1
line_errors = self._validator.validate_dmr_id(
line, line_num, feed.name)
errors += line_errors
if len(line_errors) != 0:
continue
dmr_id = DmrId(line)
dmr_ids[line_num] = dmr_id
feed.close()
return dmr_ids, errors
def _generate_zone_data(self):
logging.info('Processing zones')
feed = PathManager.open_input_file('zones.csv', 'r')
csv_feed = csv.DictReader(feed)
zones = dict()
errors = []
line_num = 1
for line in csv_feed:
logging.debug(f'Processing line {line_num}: `{line}`')
line_errors = self._validator.validate_radio_zone(
line, line_num, feed.name)
errors += line_errors
line_num += 1
if len(line_errors) != 0:
continue
zone = RadioZone(line)
zones[zone.number.fmt_val()] = zone
feed.close()
return zones, errors
def _generate_user_data(self):
logging.info('Processing dmr IDs. This step can take a while.')
feed = PathManager.open_input_file('user.csv', 'r')
csv_feed = csv.DictReader(feed)
users = dict()
errors = []
rows_processed = 0
for line in csv_feed:
line_errors = self._validator.validate_dmr_user(
line, rows_processed + 1, feed.name)
errors += line_errors
rows_processed += 1
if len(line_errors) != 0:
continue
zone = DmrUser(line)
users[zone.radio_id.fmt_val()] = zone
logging.debug(f'Writing user row {rows_processed}')
if rows_processed % file_util.USER_LINE_LOG_INTERVAL == 0:
logging.info(f'Processed {rows_processed} DMR users')
feed.close()
return users, errors
def __init__(self):
self.validator = Validator()
self.validator.flush_names()