def sigmf_recording(csv_file, data_file): """ Parameters: csv_file --- csv file path where are recorded: #Time,Robot_Number,X,Y,Angle,Start_trame,End_trame,Channel_frequency,Sample_rate data_file --- Sigmf-data file path """ # Initialize SigMF file sigmf_file = SigMFFile(data_file=data_file, global_info=global_info) # Define archive name for SigMF data and metadata files archive_name = csv_file.split('.')[0] with open(csv_file) as csvfile: reader_csv = csv.DictReader(csvfile) offset = 0 for row in reader_csv: sample_rate = int(row['Sample_rate']) len_packet = int(row['End_trame']) - int(row['Start_trame']) frequency = int(row['Channel_frequency']) robot_node = int(row['Robot_node']) latitude = int(row['X']) longitude = int(row['Y']) start_frame = offset end_frame = offset + len_packet add_capture(sigmf_file, start_frame, sample_rate, frequency) add_annotation(sigmf_file, start_frame, end_frame, latitude, longitude, robot_node) offset += len_packet # Dump contents to SigMF archive format archive_path = sigmf_file.archive(archive_name) return archive_path
def test_sigmffile(test_data_file): f = SigMFFile() f.set_global_field("core:datatype", "f32") f.add_annotation(start_index=0, length=len(TEST_FLOAT32_DATA)) f.add_capture(start_index=0) f.set_data_file(test_data_file.name) assert f._metadata == TEST_METADATA return f
def test_multichannel_seek(): '''assure that seeking is working correctly with multichannel files''' _, temp_path = tempfile.mkstemp() # write some dummy data and read back np.arange(18, dtype=np.uint16).tofile(temp_path) temp_signal = SigMFFile( data_file=temp_path, global_info={ SigMFFile.DATATYPE_KEY: 'cu16_le', SigMFFile.NUM_CHANNELS_KEY: 3, }, ) # read after the first sample temp_samples = temp_signal.read_samples(start_index=1, autoscale=False) # assure samples are in the order we expect assert np.all(temp_samples[:, 0] == np.array([6 + 7j, 12 + 13j]))
def test_multichannel_types(): '''check that real & complex for all types is reading multiple channels correctly''' lut = { 'i8': np.int8, 'u8': np.uint8, 'i16': np.int16, 'u16': np.uint16, 'u32': np.uint32, 'i32': np.int32, 'f32': np.float32, } raw_count = 16 _, temp_path = tempfile.mkstemp() for key, dtype in lut.items(): # for each type of storage np.arange(raw_count, dtype=dtype).tofile(temp_path) for num_channels in [1, 8]: # for single or 8 channel for complex_prefix in ['r', 'c']: # for real or complex check_count = raw_count * 1 # deepcopy temp_signal = SigMFFile( data_file=temp_path, global_info={ SigMFFile.DATATYPE_KEY: f'{complex_prefix}{key}_le', SigMFFile.NUM_CHANNELS_KEY: num_channels, }, ) temp_samples = temp_signal.read_samples() if complex_prefix == 'c': # complex data will be half as long check_count //= 2 assert np.all(np.iscomplex(temp_samples)) if num_channels != 1: assert temp_samples.ndim == 2 check_count //= num_channels assert check_count == temp_signal._count_samples()
def test_add_multiple_captures_and_annotations(): sigf = SigMFFile() for idx in range(3): simulate_capture(sigf, idx, 1024)
def test_add_multiple_captures_and_annotations(): f = SigMFFile() for i in range(3): simulate_capture(f, i, 1024)
def build_sigmf_md( self, task_id, measurement_params, data, schedule_entry, recording_id, start_time, end_time, ): frequency = self.sdr.radio.frequency sample_rate = self.sdr.radio.sample_rate # Build global metadata sigmf_md = SigMFFile() sigmf_md.set_global_info( GLOBAL_INFO.copy() ) # prevent GLOBAL_INFO from being modified by sigmf sigmf_md.set_global_field( "core:datatype", "cf32_le" ) # 2x 32-bit float, Little Endian sigmf_md.set_global_field("core:sample_rate", sample_rate) measurement_object = { "time_start": start_time, "time_stop": end_time, "domain": "Time", "measurement_type": "survey" if self.is_multirecording else "single-frequency", "frequency_tuned_low": frequency, "frequency_tuned_high": frequency, } sigmf_md.set_global_field("ntia-core:measurement", measurement_object) sensor = capabilities["sensor"] sensor["id"] = settings.FQDN get_sensor_location_sigmf(sensor) sigmf_md.set_global_field("ntia-sensor:sensor", sensor) from status.views import get_last_calibration_time sigmf_md.set_global_field( "ntia-sensor:calibration_datetime", get_last_calibration_time() ) action_def = { "name": self.name, "description": self.description, "summary": self.description.splitlines()[0], } sigmf_md.set_global_field("ntia-scos:action", action_def) if self.is_multirecording: sigmf_md.set_global_field("ntia-scos:recording", recording_id) sigmf_md.set_global_field("ntia-scos:task", task_id) from schedule.serializers import ScheduleEntrySerializer serializer = ScheduleEntrySerializer( schedule_entry, context={"request": schedule_entry.request} ) schedule_entry_json = serializer.to_sigmf_json() schedule_entry_json["id"] = schedule_entry.name sigmf_md.set_global_field("ntia-scos:schedule", schedule_entry_json) sigmf_md.set_global_field( "ntia-location:coordinate_system", get_coordinate_system_sigmf() ) num_samples = measurement_params.get_num_samples() capture_md = { "core:frequency": frequency, "core:datetime": self.sdr.radio.capture_time, } sigmf_md.add_capture(start_index=0, metadata=capture_md) calibration_annotation_md = self.sdr.radio.create_calibration_annotation() sigmf_md.add_annotation( start_index=0, length=num_samples, metadata=calibration_annotation_md ) time_domain_detection_md = { "ntia-core:annotation_type": "TimeDomainDetection", "ntia-algorithm:detector": "sample_iq", "ntia-algorithm:number_of_samples": num_samples, "ntia-algorithm:units": "volts", "ntia-algorithm:reference": "preselector input", } sigmf_md.add_annotation( start_index=0, length=num_samples, metadata=time_domain_detection_md ) # Recover the sigan overload flag sigan_overload = self.sdr.radio.sigan_overload # Check time domain average power versus calibrated compression time_domain_avg_power = 10 * np.log10(np.mean(np.abs(data) ** 2)) time_domain_avg_power += ( 10 * np.log10(1 / (2 * 50)) + 30 ) # Convert log(V^2) to dBm sensor_overload = False # explicitly check is not None since 1db compression could be 0 if self.sdr.radio.sensor_calibration_data["1db_compression_sensor"] is not None: sensor_overload = ( time_domain_avg_power > self.sdr.radio.sensor_calibration_data["1db_compression_sensor"] ) # Create SensorAnnotation and add gain setting and overload indicators sensor_annotation_md = { "ntia-core:annotation_type": "SensorAnnotation", "ntia-sensor:overload": sensor_overload or sigan_overload, "ntia-sensor:gain_setting_sigan": measurement_params.gain, } sigmf_md.add_annotation( start_index=0, length=num_samples, metadata=sensor_annotation_md ) return sigmf_md
def test_add_capture(): f = SigMFFile() f.add_capture(start_index=0, metadata={})
def test_add_annotation(): f = SigMFFile() f.add_capture(start_index=0) m = {"latitude": 40.0, "longitude": -105.0} f.add_annotation(start_index=0, length=128, metadata=m)
def test_invalid_capture_seq(): assert not SigMFFile(MD_INVALID_SEQUENCE_CAP).validate() assert not SigMFFile(MD_INVALID_SEQUENCE_ANN).validate()
def run(self, suites_to_run, pause=False, write_output=True): for test_suite in self.test_suites: # Skip test suites that we don't want to run if suites_to_run != [] and (not test_suite in suites_to_run): continue print("[+] Testing suite: '%s'" % test_suite) summary = TestSummary(suite=test_suite, pause=pause) # Get all metadata files associated with the suite get_mtime = lambda f: os.stat(os.path.join(self.test_suites_directory, test_suite, f)).st_mtime metadata_files = [os.path.join(self.test_suites_directory, test_suite, x) for x in sorted(os.listdir(os.path.join(self.test_suites_directory, test_suite)), key=get_mtime) if x.endswith('.sigmf-meta')] # Parse metadata files for metadata_file in metadata_files: print("[+] %s" % metadata_file) data_file = os.path.splitext(metadata_file)[0] + '.sigmf-data' # Load sigmf data TODO abstract f = open(metadata_file, 'r') sigmf = SigMFFile(metadata=f.read()) if not sigmf.validate(): raise Exception("Invalid SigMF format") global_meta = sigmf.get_global_info() capture_meta = sigmf.get_capture_info(0) f.close() # Initialize test parameters sample_rate = global_meta["core:sample_rate"] # Get LoRa configuration capture_freq = capture_meta["core:frequency"] if "lora:frequency_offset" in capture_meta: frequency_offset = capture_meta["lora:frequency_offset"] else: frequency_offset = 0 transmit_freq = capture_meta["lora:frequency"] sf = capture_meta["lora:sf"] cr = capture_meta["lora:cr"] bw = int(capture_meta["lora:bw"]) prlen = capture_meta["lora:prlen"] crc = capture_meta["lora:crc"] implicit = capture_meta["lora:implicit"] lora_config = LoRaConfig(transmit_freq, sf, cr, bw, prlen, crc, implicit) # Get test case configuration payload = capture_meta["test:expected"] times = capture_meta["test:times"] test = Test(payload, times) # Build flowgraph tb = gr.top_block() file_source = blocks.file_source(gr.sizeof_gr_complex, data_file, False) lora_receiver = lora.lora_receiver(sample_rate, capture_freq, [868100000], bw, sf, False, 4, True, reduced_rate=False, decimation=1) throttle = blocks.throttle(gr.sizeof_gr_complex, sample_rate, True) message_socket_sink = lora.message_socket_sink("127.0.0.1", 40868, 2) freq_xlating_fir_filter = filter.freq_xlating_fir_filter_ccc(1, (firdes.low_pass(1, sample_rate, 200000, 100000, firdes.WIN_HAMMING, 6.67)), frequency_offset, sample_rate) # Make connections tb.connect((file_source, 0), (throttle, 0)) tb.connect((throttle, 0), (freq_xlating_fir_filter, 0)) tb.connect((freq_xlating_fir_filter, 0), (lora_receiver, 0)) tb.msg_connect((lora_receiver, 'frames'), (message_socket_sink, 'in')) tb.start() tb.wait() decoded_data = self.server.get_payloads(times) # Output from the flowgraph summary.add(TestResult(decoded_data=decoded_data, lora_config=lora_config, test=test), print_intermediate=True) # Finally, export the result for the suite summary.export_summary(path=self.reports_directory, write_output=write_output)
def build_sigmf_md(self): logger.debug("Building SigMF metadata file") sigmf_md = SigMFFile() sigmf_md.set_global_info(GLOBAL_INFO) sigmf_md.set_global_field("core:sample_rate", self.sample_rate) sigmf_md.set_global_field("core:description", self.description) sensor_def = capabilities['sensor_definition'] sigmf_md.set_global_field("ntia:sensor_definition", sensor_def) sigmf_md.set_global_field("ntia:sensor_id", settings.FQDN) sigmf_md.set_global_field("scos:version", SCOS_TRANSFER_SPEC_VER) capture_md = { "core:frequency": self.frequency, "core:time": utils.get_datetime_str_now() } sigmf_md.add_capture(start_index=0, metadata=capture_md) for i, detector in enumerate(M4sDetector): single_frequency_fft_md = { "number_of_samples_in_fft": self.fft_size, "window": "blackman", "equivalent_noise_bandwidth": self.enbw, "detector": detector.name + "_power", "number_of_ffts": self.nffts, "units": "dBm", "reference": "not referenced" } annotation_md = { "scos:measurement_type": { "single_frequency_fft_detection": single_frequency_fft_md, } } sigmf_md.add_annotation(start_index=(i * self.fft_size), length=self.fft_size, metadata=annotation_md) return sigmf_md
def acquire_data(self, fc, parent_entry, task_id): tuning_parameters = self.tuning_parameters[fc] self.configure_usrp(fc, **tuning_parameters) # Use the radio's actual reported sample rate instead of requested rate sample_rate = self.usrp.radio.sample_rate # Build global metadata sigmf_md = SigMFFile() sigmf_md.set_global_info(GLOBAL_INFO) sigmf_md.set_global_field("core:sample_rate", sample_rate) sigmf_md.set_global_field("core:description", self.description) sensor_def = capabilities['sensor_definition'] sigmf_md.set_global_field("ntia:sensor_definition", sensor_def) sigmf_md.set_global_field("ntia:sensor_id", settings.FQDN) sigmf_md.set_global_field("scos:version", SCOS_TRANSFER_SPEC_VER) # Acquire data and build per-capture metadata data = np.array([], dtype=np.complex64) nsamps = int(sample_rate * tuning_parameters['duration_ms'] * 1e-3) dt = utils.get_datetime_str_now() acq = self.usrp.radio.acquire_samples(nsamps).astype(np.complex64) data = np.append(data, acq) capture_md = {"core:frequency": fc, "core:datetime": dt} sigmf_md.add_capture(start_index=0, metadata=capture_md) annotation_md = {"applied_scale_factor": self.usrp.radio.scale_factor} sigmf_md.add_annotation(start_index=0, length=nsamps, metadata=annotation_md) return data, sigmf_md
def test_ordered_metadata(): '''check to make sure the metadata is sorted as expected''' sigf = SigMFFile() top_sort_order = ['global', 'captures', 'annotations'] for kdx, key in enumerate(sigf.ordered_metadata()): assert kdx == top_sort_order.index(key)
def acquire_data(self, parent_entry, task_id): # Build global metadata sigmf_md = SigMFFile() sigmf_md.set_global_info(GLOBAL_INFO) sigmf_md.set_global_field("core:sample_rate", self.sample_rate) sigmf_md.set_global_field("core:description", self.description) try: sensor_def_obj = SensorDefinition.objects.get() sensor_def = SensorDefinitionSerializer(sensor_def_obj).data sigmf_md.set_global_field("scos:sensor_definition", sensor_def) except SensorDefinition.DoesNotExist: pass try: fqdn = settings.ALLOWED_HOSTS[1] except IndexError: fqdn = 'not.set' sigmf_md.set_global_field("scos:sensor_id", fqdn) sigmf_md.set_global_field("scos:version", SCOS_TRANSFER_SPEC_VER) # Acquire data and build per-capture metadata data = np.array([], dtype=np.complex64) nsamps = self.nsamples for idx, fc in enumerate(self.fcs): self.usrp.radio.tune_frequency(fc) dt = utils.get_datetime_str_now() acq = self.usrp.radio.acquire_samples(nsamps).astype(np.complex64) data = np.append(data, acq) start_idx = idx * nsamps capture_md = {"core:frequency": fc, "core:datetime": dt} sigmf_md.add_capture(start_index=start_idx, metadata=capture_md) annotation_md = { "applied_scale_factor": self.usrp.radio.scale_factor } sigmf_md.add_annotation(start_index=start_idx, length=nsamps, metadata=annotation_md) return data, sigmf_md
def build_sigmf_md(self, task_id, data): logger.debug("Building SigMF metadata file") # Use the radio's actual reported sample rate instead of requested rate sample_rate = self.sdr.radio.sample_rate sigmf_md = SigMFFile() sigmf_md.set_global_info(GLOBAL_INFO) sigmf_md.set_global_field("core:sample_rate", sample_rate) sensor_def = capabilities["sensor_definition"] sensor_def["id"] = settings.FQDN sigmf_md.set_global_field("ntia-sensor:sensor", sensor_def) action_def = { "name": self.name, "description": self.description, "type": ["FrequencyDomain"], } sigmf_md.set_global_field("ntia-scos:action", action_def) sigmf_md.set_global_field("ntia-scos:task_id", task_id) capture_md = { "core:frequency": self.sdr.radio.frequency, "core:datetime": utils.get_datetime_str_now(), } sigmf_md.add_capture(start_index=0, metadata=capture_md) for i, detector in enumerate(M4sDetector): frequency_domain_detection_md = { "ntia-core:annotation_type": "FrequencyDomainDetection", "ntia-algorithm:number_of_samples_in_fft": self.measurement_params.fft_size, "ntia-algorithm:window": "flattop", "ntia-algorithm:equivalent_noise_bandwidth": self.enbw, "ntia-algorithm:detector": detector.name + "_power", "ntia-algorithm:number_of_ffts": self.measurement_params.num_ffts, "ntia-algorithm:units": "dBm", "ntia-algorithm:reference": "not referenced", "nita-algorithm:detection_domain": "frequency", } sigmf_md.add_annotation( start_index=(i * self.measurement_params.fft_size), length=self.measurement_params.fft_size, metadata=frequency_domain_detection_md, ) calibration_annotation_md = self.sdr.radio.create_calibration_annotation( ) sigmf_md.add_annotation( start_index=0, length=self.measurement_params.fft_size * len(M4sDetector), metadata=calibration_annotation_md, ) # Recover the sigan overload flag sigan_overload = self.sdr.radio.sigan_overload # Check time domain average power versus calibrated compression flattened_data = data.flatten() time_domain_avg_power = 10 * np.log10( np.mean(np.abs(flattened_data)**2)) time_domain_avg_power += (10 * np.log10(1 / (2 * 50)) + 30 ) # Convert log(V^2) to dBm sensor_overload = ( time_domain_avg_power > self.sdr.radio.sensor_calibration_data["1db_compression_sensor"]) # Create SensorAnnotation and add gain setting and overload indicators sensor_annotation_md = { "ntia-core:annotation_type": "SensorAnnotation", "ntia-sensor:overload_sensor": sensor_overload, "ntia-sensor:overload_sigan": sigan_overload, "ntia-sensor:gain_setting_sigan": self.measurement_params.gain, } location = get_location() if location: sensor_annotation_md["core:latitude"] = (location.latitude, ) sensor_annotation_md["core:longitude"] = location.longitude sigmf_md.add_annotation( start_index=0, length=self.measurement_params.fft_size * len(M4sDetector), metadata=sensor_annotation_md, ) return sigmf_md
def _run_test(self, config, test): test_name = "{:s}-{:s}-{:n}".format(self.hw, config.file_repr(), self.test_count) test_data_path = os.path.join(self.path, test_name + '.sigmf-data') test_meta_path = os.path.join(self.path, test_name + '.sigmf-meta') self.test_count += 1 capture_meta = { "core:sample_start": 0, "core:frequency": self.capture_freq, "core:datetime": str(datetime.utcnow()), "lora:frequency": config.freq, "lora:frequency_offset": self.frequency_offset, "lora:sf": config.sf, "lora:cr": config.cr, "lora:bw": config.bw, "lora:prlen": config.prlen, "lora:crc": config.crc, "lora:implicit": config.implicit, "test:expected": test.payload, "test:times": test.times, } # Configure transmitter try: #self.lc.set_freq(config.freq) self.lc.set_sf(config.sf) self.lc.set_cr(config.cr) self.lc.set_bw(config.bw / 1e3) self.lc.set_prlen(str(config.prlen)) self.lc.set_crc("on" if config.crc else "off") #self.lc.set_implicit("on" if config.implicit else "off") self.lc.set_pwr(1) except Exception as e: print(e) exit(1) # Build GNU Radio flowgraph tb = gr.top_block() osmosdr_source = osmosdr.source(args="numchan=" + str(1) + " " + '' ) osmosdr_source.set_sample_rate(self.sample_rate) osmosdr_source.set_center_freq(self.capture_freq, 0) osmosdr_source.set_freq_corr(0, 0) osmosdr_source.set_dc_offset_mode(0, 0) osmosdr_source.set_iq_balance_mode(0, 0) osmosdr_source.set_gain_mode(False, 0) osmosdr_source.set_gain(10, 0) osmosdr_source.set_if_gain(20, 0) osmosdr_source.set_bb_gain(20, 0) osmosdr_source.set_antenna('', 0) osmosdr_source.set_bandwidth(0, 0) file_sink = blocks.file_sink(gr.sizeof_gr_complex, test_data_path, False) # Connect blocks tb.connect((osmosdr_source, 0), (file_sink, 0)) # Run print("Running %s" % test_name) tb.start() self.transmit_data(test) tb.stop() tb.wait() # Save metadata file with open(test_meta_path, 'w') as f: test_sigmf = SigMFFile(data_file=test_data_path, global_info=copy.deepcopy(self.global_meta)) test_sigmf.add_capture(0, metadata=capture_meta) test_sigmf.dump(f, pretty=True)
def build_sigmf_md(self, task_id, data, schedule_entry, start_time, end_time): logger.debug("Building SigMF metadata file") # Use the radio's actual reported sample rate instead of requested rate sample_rate = self.sdr.radio.sample_rate frequency = self.sdr.radio.frequency sigmf_md = SigMFFile() sigmf_md.set_global_info(GLOBAL_INFO.copy( )) # prevent GLOBAL_INFO from being modified by sigmf sigmf_md.set_global_field("core:datatype", "rf32_le") # 32-bit float, Little Endian sigmf_md.set_global_field("core:sample_rate", sample_rate) measurement_object = { "time_start": start_time, "time_stop": end_time, "domain": "Frequency", "measurement_type": "single-frequency", "frequency_tuned_low": frequency, "frequency_tuned_high": frequency, } sigmf_md.set_global_field("ntia-core:measurement", measurement_object) sensor = capabilities["sensor"] sensor["id"] = settings.FQDN get_sensor_location_sigmf(sensor) sigmf_md.set_global_field("ntia-sensor:sensor", sensor) from status.views import get_last_calibration_time sigmf_md.set_global_field("ntia-sensor:calibration_datetime", get_last_calibration_time()) sigmf_md.set_global_field("ntia-scos:task", task_id) action_def = { "name": self.name, "description": self.description, "summary": self.description.splitlines()[0], } sigmf_md.set_global_field("ntia-scos:action", action_def) from schedule.serializers import ScheduleEntrySerializer serializer = ScheduleEntrySerializer( schedule_entry, context={"request": schedule_entry.request}) schedule_entry_json = serializer.to_sigmf_json() schedule_entry_json["id"] = schedule_entry.name sigmf_md.set_global_field("ntia-scos:schedule", schedule_entry_json) sigmf_md.set_global_field("ntia-location:coordinate_system", get_coordinate_system_sigmf()) capture_md = { "core:frequency": frequency, "core:datetime": self.sdr.radio.capture_time, } sigmf_md.add_capture(start_index=0, metadata=capture_md) frequencies = get_fft_frequencies(self.measurement_params.fft_size, sample_rate, frequency).tolist() for i, detector in enumerate(M4sDetector): frequency_domain_detection_md = { "ntia-core:annotation_type": "FrequencyDomainDetection", "ntia-algorithm:number_of_samples_in_fft": self.measurement_params.fft_size, "ntia-algorithm:window": "flattop", "ntia-algorithm:equivalent_noise_bandwidth": self.enbw, "ntia-algorithm:detector": "fft_" + detector.name + "_power", "ntia-algorithm:number_of_ffts": self.measurement_params.num_ffts, "ntia-algorithm:units": "dBm", "ntia-algorithm:reference": "preselector input", "ntia-algorithm:frequency_start": frequencies[0], "ntia-algorithm:frequency_stop": frequencies[-1], "ntia-algorithm:frequency_step": frequencies[1] - frequencies[0], } sigmf_md.add_annotation( start_index=(i * self.measurement_params.fft_size), length=self.measurement_params.fft_size, metadata=frequency_domain_detection_md, ) calibration_annotation_md = self.sdr.radio.create_calibration_annotation( ) sigmf_md.add_annotation( start_index=0, length=self.measurement_params.fft_size * len(M4sDetector), metadata=calibration_annotation_md, ) # Recover the sigan overload flag sigan_overload = self.sdr.radio.sigan_overload # Check time domain average power versus calibrated compression flattened_data = data.flatten() time_domain_avg_power = 10 * np.log10( np.mean(np.abs(flattened_data)**2)) time_domain_avg_power += (10 * np.log10(1 / (2 * 50)) + 30 ) # Convert log(V^2) to dBm sensor_overload = False # explicitly check is not None since 1db compression could be 0 if self.sdr.radio.sensor_calibration_data[ "1db_compression_sensor"] is not None: sensor_overload = ( time_domain_avg_power > self.sdr.radio. sensor_calibration_data["1db_compression_sensor"]) # Create SensorAnnotation and add gain setting and overload indicators sensor_annotation_md = { "ntia-core:annotation_type": "SensorAnnotation", "ntia-sensor:overload": sensor_overload or sigan_overload, "ntia-sensor:gain_setting_sigan": self.measurement_params.gain, } sigmf_md.add_annotation( start_index=0, length=self.measurement_params.fft_size * len(M4sDetector), metadata=sensor_annotation_md, ) return sigmf_md
def _run_test(self, config, test): test_name = "{:s}-{:s}-{:n}".format(self.hw, config.file_repr(), self.test_count) test_data_path = os.path.join(self.path, test_name + '.sigmf-data') test_meta_path = os.path.join(self.path, test_name + '.sigmf-meta') self.test_count += 1 capture_meta = { "core:sample_start": 0, "core:frequency": self.capture_freq, "core:datetime": str(datetime.utcnow()), "lora:frequency": config.freq, "lora:frequency_offset": self.frequency_offset, "lora:sf": config.sf, "lora:cr": config.cr, "lora:bw": config.bw, "lora:prlen": config.prlen, "lora:crc": config.crc, "lora:implicit": config.implicit, "test:expected": test.payload, "test:times": test.times, } # Configure transmitter try: #self.lc.set_freq(config.freq) self.lc.set_sf(config.sf) self.lc.set_cr(config.cr) self.lc.set_bw(config.bw / 1e3) self.lc.set_prlen(str(config.prlen)) self.lc.set_crc("on" if config.crc else "off") #self.lc.set_implicit("on" if config.implicit else "off") self.lc.set_pwr(1) except Exception as e: print(e) exit(1) # Build GNU Radio flowgraph gr.enable_realtime_scheduling() tb = gr.top_block() osmosdr_source = osmosdr.source(args="numchan=" + str(1) + " " + '' ) osmosdr_source.set_sample_rate(self.sample_rate) osmosdr_source.set_center_freq(self.capture_freq, 0) osmosdr_source.set_freq_corr(0, 0) osmosdr_source.set_dc_offset_mode(0, 0) osmosdr_source.set_iq_balance_mode(0, 0) osmosdr_source.set_gain_mode(False, 0) osmosdr_source.set_gain(10, 0) osmosdr_source.set_if_gain(20, 0) osmosdr_source.set_bb_gain(20, 0) osmosdr_source.set_antenna('', 0) osmosdr_source.set_bandwidth(0, 0) file_sink = blocks.file_sink(gr.sizeof_gr_complex, test_data_path, False) # Connect blocks tb.connect((osmosdr_source, 0), (file_sink, 0)) # Run print("Running %s" % test_name) tb.start() self.transmit_data(test) tb.stop() tb.wait() # Save metadata file with open(test_meta_path, 'w') as f: test_sigmf = SigMFFile(data_file=test_data_path, global_info=copy.deepcopy(self.global_meta)) test_sigmf.add_capture(0, metadata=capture_meta) test_sigmf.dump(f, pretty=True)
def create_sigmf_metafile(self, x_len, dest_data_filename, _file): sigmf_md = SigMFFile(data_file=dest_data_filename) sigmf_md.set_global_field("core:datatype", self.datatype) sigmf_md.set_global_field("core:sample_rate", self.sample_rate) sigmf_md.set_global_field("core:author", self.author) sigmf_md.set_global_field("core:description", self.description) sha = sigmf_md.calculate_hash() print sha start_index = 0 capture_len = x_len capture_md = { "core:time": utils.get_sigmf_iso8601_datetime_now(), "frequency": self.frequency } sigmf_md.add_capture(start_index=start_index, metadata=capture_md) annotation_md = { "genesys:transmitter": { "antenna": { "model": "Ettus VERT2450", "type": "Vertical", "gain": 3, "high_frequency": 2480000000, "low_frequency": 2400000000 }, "model": "Ettus USRP X310 with UBX-160 (10 MHz-6 GHz, 160 MHz BW) Daughterboard" }, "genesys:reciever": { "antenna": { "model": "Ettus VERT2450", "type": "Vertical", "gain": 3, "high_frequency": 2480000000, "low_frequency": 2400000000 }, "model": "Ettus USRP B210" } } sigmf_md.add_annotation(start_index=start_index, length=capture_len, metadata=annotation_md) return sigmf_md
def build_sigmf_md(self, task_id, measurement_params, data): # Build global metadata sigmf_md = SigMFFile() sigmf_md.set_global_info(GLOBAL_INFO) sample_rate = self.sdr.radio.sample_rate sigmf_md.set_global_field("core:sample_rate", sample_rate) sensor_def = capabilities["sensor_definition"] sensor_def["id"] = settings.FQDN sigmf_md.set_global_field("ntia-sensor:sensor", sensor_def) action_def = { "name": self.name, "description": self.description, "type": ["TimeDomain"], } sigmf_md.set_global_field("ntia-scos:action", action_def) sigmf_md.set_global_field("ntia-scos:task_id", task_id) dt = utils.get_datetime_str_now() num_samples = measurement_params.get_num_samples() capture_md = { "core:frequency": self.sdr.radio.frequency, "core:datetime": dt } sigmf_md.add_capture(start_index=0, metadata=capture_md) calibration_annotation_md = self.sdr.radio.create_calibration_annotation( ) sigmf_md.add_annotation(start_index=0, length=num_samples, metadata=calibration_annotation_md) time_domain_detection_md = { "ntia-core:annotation_type": "TimeDomainDetection", "ntia-algorithm:detector": "sample_iq", "ntia-algorithm:detection_domain": "time", "ntia-algorithm:number_of_samples": num_samples, "ntia-algorithm:units": "volts", "ntia-algorithm:reference": "not referenced", } sigmf_md.add_annotation(start_index=0, length=num_samples, metadata=time_domain_detection_md) # Recover the sigan overload flag sigan_overload = self.sdr.radio.sigan_overload # Check time domain average power versus calibrated compression time_domain_avg_power = 10 * np.log10(np.mean(np.abs(data)**2)) time_domain_avg_power += (10 * np.log10(1 / (2 * 50)) + 30 ) # Convert log(V^2) to dBm sensor_overload = ( time_domain_avg_power > self.sdr.radio.sensor_calibration_data["1db_compression_sensor"]) # Create SensorAnnotation and add gain setting and overload indicators sensor_annotation_md = { "ntia-core:annotation_type": "SensorAnnotation", "ntia-sensor:overload_sensor": sensor_overload, "ntia-sensor:overload_sigan": sigan_overload, "ntia-sensor:gain_setting_sigan": measurement_params.gain, } location = get_location() if location: sensor_annotation_md["core:latitude"] = (location.latitude, ) sensor_annotation_md["core:longitude"] = location.longitude sigmf_md.add_annotation(start_index=0, length=num_samples, metadata=sensor_annotation_md) return sigmf_md
def create_sigmf_metafile(self, x_len, dest_data_filename, _file): sigmf_md = SigMFFile(data_file=dest_data_filename) sigmf_md.set_global_field("core:datatype", self.datatype) sigmf_md.set_global_field("core:sample_rate", self.sample_rate) sigmf_md.set_global_field("core:author", self.author) sigmf_md.set_global_field("core:version", self.version) pattern = '(\d+)ft' distance = re.findall(pattern, _file) print 'distance', distance #--description "SigMF IQ samples recording of demodulated data derived from over-the-cable WiFi transmissions collected by a fixed USRP B210 as a receiver. The transmitter emitted IEEE 802.11a standards compliant frames generated via a MATLAB WLAN System toolbox. Using UHD software, a controlled level of IQ imbalance is introduced at the runtime such that the demodulated symbols acquire unique characteristics." self.description = "SigMF IQ samples recording of over-the-air WiFi transmissions collected by a fixed USRP B210 as a receiver. The data is collected in indoor environmnet of Kostas Research Institute (KRI), at Northeastern University, with a transmitter-receiver separation distance of " + distance[ 0] + "ft. The transmitter emitted IEEE 802.11a standards compliant frames generated via a MATLAB WLAN System toolbox." sigmf_md.set_global_field("core:description", self.description) sha = sigmf_md.calculate_hash() print sha start_index = 0 capture_len = x_len capture_md = { "core:time": utils.get_sigmf_iso8601_datetime_now(), "frequency": self.frequency } sigmf_md.add_capture(start_index=start_index, metadata=capture_md) # annotation_md = { # "core:latitude": 40.0 + 0.0001 * 0, # "core:longitude": -105.0 + 0.0001 * 0, # } print sigmf_md annotation_md = { "genesys:transmitter": { "antenna": { "model": "Ettus VERT2450", "type": "Vertical", "gain": 3, "high_frequency": 2480000000, "low_frequency": 2400000000 }, "model": "Ettus USRP X310 with UBX-160 (10 MHz-6 GHz, 160 MHz BW) Daughterboard" }, "genesys:reciever": { "antenna": { "model": "Ettus VERT2450", "type": "Vertical", "gain": 3, "high_frequency": 2480000000, "low_frequency": 2400000000 }, "model": "Ettus USRP B210" } } # annotation_md = { # "genesys:transmitter_identification": 'hello', # "genesys:receiver_identification": 'hello', # } sigmf_md.add_annotation(start_index=start_index, length=capture_len, metadata=annotation_md) #print "Hello" #sigmf_md.set_annotations("genesys:transmitter_identification","Hello") return sigmf_md
def build_sigmf_md(self): logger.debug("Building SigMF metadata file") sigmf_md = SigMFFile() sigmf_md.set_global_field("core:datatype", "rf32_le") sigmf_md.set_global_field("core:sample_rate", self.sample_rate) sigmf_md.set_global_field("core:description", self.description) sensor_def_obj = SensorDefinition.objects.get() sensor_def_json = SensorDefinitionSerializer(sensor_def_obj).data sigmf_md.set_global_field("scos:sensor_definition", sensor_def_json) try: fqdn = settings.ALLOWED_HOSTS[1] except IndexError: fqdn = 'not.set' sigmf_md.set_global_field("scos:sensor_id", fqdn) sigmf_md.set_global_field("scos:version", SCOS_TRANSFER_SPEC_VER) capture_md = { "core:frequency": self.frequency, "core:time": utils.get_datetime_str_now() } sigmf_md.add_capture(start_index=0, metadata=capture_md) for i, detector in enumerate(M4sDetector): single_frequency_fft_md = { "number_of_samples_in_fft": self.fft_size, "window": "blackman", "equivalent_noise_bandwidth": self.enbw, "detector": detector.name + "_power", "number_of_ffts": self.nffts, "units": "dBm", "reference": "not referenced" } annotation_md = { "scos:measurement_type": { "single_frequency_fft_detection": single_frequency_fft_md, } } sigmf_md.add_annotation(start_index=(i * self.fft_size), length=self.fft_size, metadata=annotation_md) return sigmf_md
def test_default_constructor(): SigMFFile()
def test_valid_data(): assert SigMFFile(MD_VALID).validate()
def test_set_non_required_global_field(): f = SigMFFile() f.set_global_field('this_is:not_in_the_schema', None)
def run(self, suites_to_run, pause=False, write_output=True): for test_suite in self.test_suites: # Skip test suites that we don't want to run if suites_to_run != [] and (not test_suite in suites_to_run): continue print("[+] Testing suite: '%s'" % test_suite) summary = TestSummary(suite=test_suite, pause=pause) # Get all metadata files associated with the suite get_mtime = lambda f: os.stat( os.path.join(self.test_suites_directory, test_suite, f) ).st_mtime metadata_files = [ os.path.join(self.test_suites_directory, test_suite, x) for x in sorted(os.listdir( os.path.join(self.test_suites_directory, test_suite)), key=get_mtime) if x.endswith('.sigmf-meta') ] # Parse metadata files for metadata_file in metadata_files: print("[+] %s" % metadata_file) data_file = os.path.splitext(metadata_file)[0] + '.sigmf-data' # Load sigmf data TODO abstract f = open(metadata_file, 'r') sigmf = SigMFFile(metadata=f.read()) if not sigmf.validate(): raise Exception("Invalid SigMF format") global_meta = sigmf.get_global_info() capture_meta = sigmf.get_capture_info(0) f.close() # Initialize test parameters sample_rate = global_meta["core:sample_rate"] # Get LoRa configuration capture_freq = capture_meta["core:frequency"] if "lora:frequency_offset" in capture_meta: frequency_offset = capture_meta["lora:frequency_offset"] else: frequency_offset = 0 transmit_freq = capture_meta["lora:frequency"] sf = capture_meta["lora:sf"] cr = capture_meta["lora:cr"] bw = int(capture_meta["lora:bw"]) prlen = capture_meta["lora:prlen"] crc = capture_meta["lora:crc"] implicit = capture_meta["lora:implicit"] lora_config = LoRaConfig(transmit_freq, sf, cr, bw, prlen, crc, implicit) # Get test case configuration payload = capture_meta["test:expected"] times = capture_meta["test:times"] test = Test(payload, times) # Build flowgraph tb = gr.top_block() file_source = blocks.file_source(gr.sizeof_gr_complex, data_file, False) lora_receiver = lora.lora_receiver(sample_rate, capture_freq, [868100000], bw, sf, False, 4, True, reduced_rate=False, decimation=1) throttle = blocks.throttle(gr.sizeof_gr_complex, sample_rate, True) message_socket_sink = lora.message_socket_sink( "127.0.0.1", 40868, 2) freq_xlating_fir_filter = filter.freq_xlating_fir_filter_ccc( 1, (firdes.low_pass(1, sample_rate, 200000, 100000, firdes.WIN_HAMMING, 6.67)), frequency_offset, sample_rate) # Make connections tb.connect((file_source, 0), (throttle, 0)) tb.connect((throttle, 0), (freq_xlating_fir_filter, 0)) tb.connect((freq_xlating_fir_filter, 0), (lora_receiver, 0)) tb.msg_connect((lora_receiver, 'frames'), (message_socket_sink, 'in')) tb.start() tb.wait() decoded_data = self.server.get_payloads( times) # Output from the flowgraph summary.add(TestResult(decoded_data=decoded_data, lora_config=lora_config, test=test), print_intermediate=True) # Finally, export the result for the suite summary.export_summary(path=self.reports_directory, write_output=write_output)
def run_gui(): window_input = WindowInput() capture_data_input = CaptureData() capture_text_blocks = dict() window_text_blocks = dict() f = SigMFFile() capture_selector_dict = dict() layout = [ [ Text('This is the APL SIGMF tool to archive RF datasets', size=(80, 1)) ], [ Text('Enter your data and signal captures below. You must include', auto_size_text=True), Text('required', text_color='red', font=DEFAULT_FONT + ('italic', ), auto_size_text=True), Text('fields.', size=(50, 1), auto_size_text=True) ], [Text('_' * 150, auto_size_text=True)] ] layout.append([Text('Global Data', font=('Arial', 12, 'bold'))]) num_components = 0 line = [] for el in window_input.iter_core(): size = (30, 1) if len(line) == 0 else (None, None) auto_size = True if len(line) == 0 else (10, 1) line.extend([ Text(el, justification='right', size=size, text_color='red' if el in window_input.req_tags else None, auto_size_text=auto_size) ]) if el in window_input.el_multiline: window_text_blocks.update({ el: Multiline(window_input.el_text.get(el, ''), key=el, tooltip=window_input.el_tooltips.get(el, None), size=(30, 2)) }) elif el in window_input.el_selector: window_text_blocks.update({ el: Combo(values=window_input.el_selector[el], key=el, size=window_input.el_size.get(el, (None, None))) }) elif el in window_input.el_checkbox: window_text_blocks.update({ el: Checkbox(window_input.el_text.get(el, ''), key=el, size=window_input.el_size.get(el, (None, None))) }) else: window_text_blocks.update({ el: InputText(window_input.el_text.get(el, ''), key=el, tooltip=window_input.el_tooltips.get(el, None)) }) line.append(window_text_blocks[el]) if el in window_input.el_units: line.append(Text(window_input.el_units[el])) num_components += 1 if num_components < window_input.first_line_size: continue layout.append(line) line = [] for el1, el2 in window_input.iter_x_secondary(2): line = [] for el, size in zip([el1, el2], [30, 22]): if el is None: continue color = 'red' if el in window_input.req_tags else None window_text_blocks.update({ el: InputText(window_input.el_text.get(el, ''), key=el, tooltip=window_input.el_tooltips.get(el, None)) }) line.extend([ Text(el, justification='right', size=(size, 1), text_color=color), window_text_blocks[el] ]) if el in window_input.el_units: line.append(Text(window_input.el_units[el], size=(5, 1))) else: line.append(Text('', size=(5, 1))) layout.append(line) layout.extend( [[Text('_' * 150, auto_size_text=True)], [Text('Individual Capture Data', font=('Arial', 12, 'bold'))], [ Text('Capture Selector', auto_size_text=True), combo_button, Text('', size=(10, 1)), Button('Add Capture', enable_events=True), Button('Remove Capture', enable_events=True, size=(15, 1)), Button('Clear Capture', enable_events=True, size=(15, 1)) ]]) for el1, el2, el3 in capture_data_input.iter_x(3): line = [] for el in [el1, el2, el3]: if el is None: continue capture_text_blocks.update({ el: InputText(key=el, tooltip=capture_data_input.el_tooltips.get(el, None)) }) color = 'red' if el in capture_data_input.req_tags else None line.extend([ Text(el, justification='right', size=(20, 1), text_color=color), capture_text_blocks[el] ]) if el in capture_data_input.el_units: line.append(Text(capture_data_input.el_units[el], size=(5, 1))) else: line.append(Text('', size=(5, 1))) layout.append(line) window_text_blocks.update( {WindowInput.DATA_FILE: InputText('', key=WindowInput.DATA_FILE)}) layout.extend([[Text('_' * 150, auto_size_text=True)], [Text('Data Location', font=('Arial', 12, 'bold'))], [ Text(WindowInput.DATA_FILE, size=(30, 1), justification='right', text_color='red'), window_text_blocks[WindowInput.DATA_FILE], FileBrowse() ], [ Text(WindowInput.OUTPUT_FOLDER, size=(30, 1), justification='right'), InputText('', key=WindowInput.OUTPUT_FOLDER), FolderBrowse(), submit_button ], [ Text(WindowInput.LOAD_PATH, size=(30, 1), justification='right'), InputText('', key=WindowInput.LOAD_PATH), FileBrowse(file_types=(("Archive Files", "*.sigmf"), )), load_button ], [validate_button, Button('View Data')]]) window = Window('APL SigMF Archive Creator', auto_size_buttons=False, default_element_size=(20, 1), auto_size_text=False, default_button_element_size=(10, 1)).Layout(layout) while True: validate_button.Update(text='Update') load_button.Update(text='Load') submit_button.Update(text='Save Archive') window.Refresh() event, values = window.Read() print(event, values) if event == 'Load Archive': load_path = values[WindowInput.LOAD_PATH] if load_path == '': show_error('No archive file provided') continue load_button.Update(text='Loading...') window.Refresh() print('reading from ', values[WindowInput.LOAD_PATH]) f = fromarchive(values[WindowInput.LOAD_PATH]) update_global_screen(window_input, window_text_blocks, f.get_global_info(), f) capture_selector_dict = {} for capture in f.get_captures(): add_capture(capture_data_input, capture, capture_selector_dict, f, from_archive=True) elif event == 'Data Type Help': PopupOK( 'Format: <TypeCharacters><ElementBitSize>_<Endianness>\n\n' '\tTypeCharacters:\n' '\t\tUnsigned data: \"u\"\n' '\t\tComplex data: \"c\"\n' '\t\tFixedpoint data: \"f\"\n' '\tElementBitSize:\n' '\t\t32 bits, 16 bits, or 8 bits\n' '\tEndianness:\n' '\t\tl: Little Endian\n' '\t\tb: Big Endian\n\n\n\n' 'Example: \"uc32_l\"\n' 'Unsigned complex data where each element is 32 bits, or 64 bits total, formatted in little endian.', title='Data Type Help') elif event == 'Update': validate_button.Update(text='Validating...') window.Refresh() window_data_type_dict = {} added = True for el in window_input.iter(): req_field = True if el in window_input.req_tags else False el_type = window_input.el_types.get(el, None) el_unit = window_input.el_units.get(el, None) if el in window_input.partial_component_list: added = added and add_sigmf_field(update_dictionary, values, el, window_data_type_dict, window_input.get_tag(el), required=req_field, type=el_type, unit=el_unit) else: added = added and add_sigmf_field( SigMFFile.set_global_field, values, el, f, window_input.get_tag(el), required=req_field, type=el_type, unit=el_unit) data_type_str = '' data_type_str += 'c' if bool( window_data_type_dict[WindowInput.DATA_TYPE_COMPLEX]) else '' data_type_str += 'f' if not bool(window_data_type_dict[ WindowInput.DATA_TYPE_FIXEDPOINT]) else '' data_type_str += 'u' if bool( window_data_type_dict[WindowInput.DATA_TYPE_UNSIGNED]) else '' data_type_str += str( window_data_type_dict[WindowInput.DATA_SAMPLE_SIZE]) + '_' data_type_str += 'l' if window_data_type_dict[ WindowInput.DATA_BYTE_ORDER] == 'little endian' else 'b' data_type_dict = {SigMFFile.DATATYPE_KEY: data_type_str} added = added and add_sigmf_field(SigMFFile.set_global_field, data_type_dict, SigMFFile.DATATYPE_KEY, f, SigMFFile.DATATYPE_KEY, required=True) print('HERE: ', window_data_type_dict) added = added and add_sigmf_field(SigMFFile.set_data_file, values, WindowInput.DATA_FILE, f, required=True) and added if not added: # requirement not given continue if validate_data(f): submit_button.Update(disabled=False, button_color=DEFAULT_BUTTON_COLOR) elif event == 'Capture Combo': capture_dict = capture_selector_dict[values['Capture Combo']] update_capture_screen(capture_data_input, capture_text_blocks, capture_dict) elif event == 'Add Capture': add_capture(capture_data_input, values, capture_selector_dict, f) elif event == 'Remove Capture': capture_dict = dict() added = add_sigmf_field(update_dictionary, values, CaptureData.START_INDEX, capture_dict, SigMFFile.START_INDEX_KEY, required=True, type=int) if not added: # requirement not given continue captures = [] for capture in f._metadata[SigMFFile.CAPTURE_KEY]: if capture[SigMFFile.START_INDEX_KEY] != capture_dict[ SigMFFile.START_INDEX_KEY]: captures.append(capture) f._metadata[SigMFFile.CAPTURE_KEY] = captures annotations = [] for annotation in f._metadata[SigMFFile.ANNOTATION_KEY]: if annotation[SigMFFile.START_INDEX_KEY] != capture_dict[ SigMFFile.START_INDEX_KEY]: annotations.append(annotation) f._metadata[SigMFFile.ANNOTATION_KEY] = annotations new_values = list(combo_button.Values) rm_val = 'Capture {}'.format( capture_dict[SigMFFile.START_INDEX_KEY]) if rm_val in capture_selector_dict: capture_selector_dict.pop(rm_val) if rm_val in new_values: new_values.remove(rm_val) combo_button.Update(values=new_values, set_to_index=0) capture_dict = capture_selector_dict.get( combo_button.DefaultValue, None) update_capture_screen(capture_data_input, capture_text_blocks, capture_dict) elif event == 'Clear Capture': update_capture_screen(capture_data_input, capture_text_blocks, None) elif event == 'View Data': PopupOK('Current data:\n', f.dumps(pretty=True), title='') elif event == 'Save Archive': output_folder = values[WindowInput.OUTPUT_FOLDER] if output_folder == '': show_error('No output folder provided') continue elif len(capture_selector_dict.keys()) == 0: show_error('No capture data specified') submit_button.Update(text='Saving...') window.Refresh() archive_file = output_folder + '/' + os.path.basename( f.data_file).split('.')[0] + SIGMF_ARCHIVE_EXT f.archive(archive_file) PopupOK('Saved archive as \n', archive_file, title='') elif event in ['Cancel', None, 'Exit']: window.Close() break window.Close()