def test_1268(self): version_two = dft.read('test-data/version2.dtk') simulation = version_two.simulation self.assertTrue('nodes' not in simulation) version_three = dft.read('test-data/version3.dtk') simulation = version_three.simulation self.assertTrue('nodes' not in simulation) version_four = dft.read('test-data/version4.dtk') simulation = version_four.simulation self.assertTrue('nodes' not in simulation) return
def make_uncompressed(): dtk = dft.read('test-data/baseline.dtk') dtk.compression = dft.NONE dft.write(dtk, 'test-data/uncompressed.dtk') return
def make_snappy(): dtk = dft.read('test-data/baseline.dtk') dtk.compression = dft.SNAPPY dft.write(dtk, 'test-data/snappy.dtk') return
def make_bad_chunk_snappy(): dtk = dft.read('test-data/snappy.dtk') # get the chunk, first node, not simulation chunk = dtk.chunks[1] # choose an index index = random.randint(0, len(chunk)) # convert string to array of chars chunk = [c for c in chunk] # perturb the bits old = chunk[index] new = chr(~ord(old) % 256) chunk[index] = new # convert array of chars back to string chunk = ''.join(chunk) dtk.chunks[1] = chunk print('Flipped bits of chunk #1 byte {0} ({1} -> {2})'.format( index, ord(old), ord(new))) dft.write(dtk, 'test-data/bad-chunk-snappy.dtk') return
def test_reading_file(self): dtk = dft.read('test-data/version4.dtk') self.assertEqual('IDM', dtk.author) self.assertEqual('DTK', dtk.tool) self.assertEqual(True, dtk.compressed) self.assertEqual(dft.LZ4, dtk.compression) self.assertEqual(625105, dtk.byte_count) self.assertEqual(5, dtk.chunk_count) self.assertEqual(384, dtk.chunk_sizes[0]) self.assertEqual(164210, dtk.chunk_sizes[1]) self.assertEqual(152439, dtk.chunk_sizes[2]) self.assertEqual(156570, dtk.chunk_sizes[3]) self.assertEqual(151502, dtk.chunk_sizes[4]) self.assertEqual(1, dtk.simulation.Run_Number) self.assertEqual(10001, dtk.simulation.individualHumanSuidGenerator.next_suid.id) self.assertEqual(0, dtk.simulation.sim_type) self.assertEqual(4, len(dtk.nodes)) node = dtk.nodes[0] self.assertEqual(1, node.externalId) self.assertEqual(2500, len(node.individualHumans)) human = node.individualHumans[0] self.assertEqual(1, human.cumulativeInfs) self.assertEqual(0, len(human.infections)) self.assertEqual(0, human.infectiousness) self.assertEqual(9598.48, human.m_age) self.assertEqual(0, human.m_gender) self.assertEqual(False, human.m_is_infected) return
def test_reading_file(self): dtk = dft.read('test-data/version2.dtk') self.assertEqual('', dtk.author) self.assertEqual('', dtk.tool) self.assertEqual(True, dtk.compressed) self.assertEqual(dft.LZ4, dtk.compression) self.assertEqual(168431, dtk.byte_count) self.assertEqual(2, dtk.chunk_count) self.assertEqual(1584, dtk.chunk_sizes[0]) self.assertEqual(166847, dtk.chunk_sizes[1]) self.assertEqual(28, dtk.simulation.Run_Number) self.assertEqual(1001, dtk.simulation.individualHumanSuidGenerator.next_suid.id) self.assertEqual(2, dtk.simulation.sim_type) self.assertEqual(1, len(dtk.nodes)) node = dtk.nodes[0] self.assertEqual(340461476, node.externalId) self.assertEqual(1000, len(node.individualHumans)) human = node.individualHumans[0] self.assertEqual(0, human.cumulativeInfs) self.assertEqual(0, len(human.infections)) self.assertEqual(0, human.infectiousness) self.assertEqual(7693.72, human.m_age) self.assertEqual(1, human.m_gender) self.assertEqual(False, human.m_is_infected) return
def test_bad_sim_chunk_snappy(self): with self.assertRaises(UserWarning): dtk_file = dft.read('test-data/bad-sim-snappy.dtk') # Accessing the simulation field raises the exception print( dtk_file.simulation.individualHumanSuidGenerator.next_suid.id) return
def test_reading_file(self): dtk = dft.read('test-data/version3.dtk') self.assertEqual('', dtk.author) self.assertEqual('', dtk.tool) self.assertEqual(True, dtk.compressed) self.assertEqual(dft.LZ4, dtk.compression) self.assertEqual(177151, dtk.byte_count) self.assertEqual(2, dtk.chunk_count) self.assertEqual(1574, dtk.chunk_sizes[0]) self.assertEqual(175577, dtk.chunk_sizes[1]) self.assertEqual(27, dtk.simulation.Run_Number) self.assertEqual( 1032, dtk.simulation.individualHumanSuidGenerator.next_suid.id) self.assertEqual(1, dtk.simulation.sim_type) self.assertEqual(1, len(dtk.nodes)) node = dtk.nodes[0] self.assertEqual(340461476, node.externalId) self.assertEqual(1023, len(node.individualHumans)) human = node.individualHumans[0] self.assertEqual(8, human.cumulativeInfs) self.assertEqual(5, len(human.infections)) self.assertEqual(1, human.infectiousness) self.assertEqual(2457.41, human.m_age) self.assertEqual(1, human.m_gender) self.assertEqual(True, human.m_is_infected) return
def test_reading_compressed_file(self): dtk = dft.read('test-data/version1.dtk') self.assertEqual('', dtk.author) self.assertEqual('', dtk.tool) self.assertEqual(True, dtk.compressed) self.assertEqual(dft.SNAPPY, dtk.compression) self.assertEqual(1438033, dtk.byte_count) self.assertEqual(1, dtk.chunk_count) self.assertEqual(1438033, dtk.chunk_sizes[0]) self.assertEqual(1, dtk.simulation.Run_Number) self.assertEqual( 10001, dtk.simulation.individualHumanSuidGenerator.next_suid.id) self.assertEqual(0, dtk.simulation.sim_type) self.assertEqual(4, len(dtk.nodes)) node = dtk.nodes[0] self.assertEqual(1, node.externalId) self.assertEqual(2500, len(node.individualHumans)) human = node.individualHumans[0] self.assertEqual(1, human.cumulativeInfs) self.assertEqual(0, len(human.infections)) self.assertEqual(0, human.infectiousness) self.assertEqual(9598.48, human.m_age) self.assertEqual(0, human.m_gender) self.assertEqual(False, human.m_is_infected) return
def test_round_trip(self): source = dft.DtkFileV1() source.author = 'dtkFileTests' source.compression = dft.SNAPPY # Choose something other than the default simulation = support.SerialObject({"simulation": {"enable_spatial_output": False, "Run_Number": 1, "enable_property_output": False, "infectionSuidGenerator": {"next_suid": {"id": 1}, "rank": 0, "numtasks": 1}, "__class__": "Simulation", "campaignFilename": "campaign.json", "nodes": [{"node": {"demographics_other": False, "birth_rate_sinusoidal_forcing_phase": 0, "sample_rate_20_plus": 0, "max_sampling_cell_pop": 0, "infectivity_boxcar_end_time": 0, "sample_rate_18mo_4yr": 0, "__class__": "Node", "infectivity_boxcar_start_time": 0, "demographics_gender": True, "individualHumans": [{"migration_is_destination_new_home": False, "home_node_id": {"id": 1}, "waiting_for_family_trip": False, "migration_will_return": True, "waypoints": [], "family_migration_type": 0, "Inf_Sample_Rate": 1, "migration_outbound": True, "m_mc_weight": 1, "migration_time_until_trip": 0, "StateChange": 0, "suid": {"id": 1}, "leave_on_family_trip": False, "interventions": {"drugVaccineReducedTransmit": 1, "drugVaccineReducedAcquire": 1, "__class__": "InterventionsContainer", "drugVaccineReducedMortality": 1, "interventions": []}, "migration_destination": {"id": 0}, "__class__": "IndividualHuman", "infectiousness": 0, "migration_mod": 1, "waypoints_trip_type": [], "m_age": 9588.48, "m_is_infected": False, "m_daily_mortality_rate": 0, "cumulativeInfs": 0, "family_migration_time_at_destination": 0, "family_migration_destination": {"id": 0}, "m_new_infection_state": 0, "migration_time_at_destination": 0, "family_migration_is_destination_new_home": False, "migration_type": 0, "max_waypoints": 0, "pregnancy_timer": 0, "family_migration_time_until_trip": 0, "is_pregnant": False, "susceptibility": {"trandecayoffset": 0, "age": 9588.48, "__class__": "Susceptibility", "acqdecayoffset": 0, "mod_mortality": 1, "mod_acquire": 1, "mod_transmit": 1, "mortdecayoffset": 0}, "is_on_family_trip": False, "infections": [], "above_poverty": 0, "Properties": [], "m_gender": 0}], "serializationMask": 3, "infectivity_sinusoidal_forcing_amplitude": 1, "birth_rate_boxcar_start_time": 0, "x_birth": 1, "sample_rate_immune": 0, "population_density_infectivity_correction": 0, "sample_rate_0_18mo": 0, "population_scaling": 0, "demographics_birth": False, "suid": {"id": 1}, "birth_rate_boxcar_forcing_amplitude": 1, "population_scaling_factor": 1, "birth_rate_boxcar_end_time": 0, "ind_sampling_type": 0, "vital_birth_time_dependence": 0, "externalId": 1, "age_initialization_distribution_type": 1, "infectivity_sinusoidal_forcing_phase": 0, "immune_threshold_for_downsampling": 0, "sample_rate_5_9": 0, "home_individual_ids": [{"key": 1, "value": {"id": 1}}], "population_density_c50": 0, "sample_rate_birth": 0, "birth_rate_sinusoidal_forcing_amplitude": 1, "infectivity_boxcar_forcing_amplitude": 1, "vital_birth": False, "maternal_transmission": False, "vital_birth_dependence": 1, "infectivity_scaling": 0, "animal_reservoir_type": 0, "zoonosis_rate": 0, "sample_rate_10_14": 0, "sample_rate_15_19": 0}, "suid": {"id": 1}}], "loadbalance_filename": "", "serializationMask": 3, "individualHumanSuidGenerator": {"next_suid": {"id": 1001}, "rank": 0, "numtasks": 1}, "sim_type": 0, "enable_default_report": True, "enable_event_report": False, "Ind_Sample_Rate": 1, "demographic_tracking": False}}) source.objects.append(simulation) handle, filename = tempfile.mkstemp() os.close(handle) dft.write(source, filename) dest = dft.read(filename) self.assertEqual('dtkFileTests', dest.author) self.assertEqual(dft.SNAPPY, dest.compression) simulation = dest.simulation self.assertEqual(False, simulation.enable_spatial_output) self.assertEqual(1, simulation.Run_Number) self.assertEqual(1001, simulation.individualHumanSuidGenerator.next_suid.id) self.assertEqual(0, simulation.sim_type) self.assertEqual(True, simulation.enable_default_report) node = simulation.nodes[0].node # Simulation nodes is a map<suid,node> self.assertEqual(1, node.externalId) self.assertEqual(False, node.demographics_other) self.assertEqual(True, node.demographics_gender) self.assertEqual(1, node.x_birth) individual = node.individualHumans[0] self.assertEqual(1, individual.m_mc_weight) self.assertEqual(1, individual.suid.id) self.assertEqual(9588.48, individual.m_age) self.assertEqual(0, individual.m_gender) os.remove(filename) return
def test_reading_file(self): dtk = dft.read('test-data/version4.dtk') self.assertEqual('IDM', dtk.author) self.assertEqual('DTK', dtk.tool) self.assertEqual(True, dtk.compressed) self.assertEqual(dft.LZ4, dtk.compression) self.assertEqual(625105, dtk.byte_count) self.assertEqual(5, dtk.chunk_count) self.assertEqual(384, dtk.chunk_sizes[0]) self.assertEqual(164210, dtk.chunk_sizes[1]) self.assertEqual(152439, dtk.chunk_sizes[2]) self.assertEqual(156570, dtk.chunk_sizes[3]) self.assertEqual(151502, dtk.chunk_sizes[4]) self.assertEqual(1, dtk.simulation.Run_Number) self.assertEqual( 10001, dtk.simulation.individualHumanSuidGenerator.next_suid.id) self.assertEqual(0, dtk.simulation.sim_type) self.assertEqual(4, len(dtk.nodes)) node = dtk.nodes[0] self.assertEqual(1, node.externalId) self.assertEqual(2500, len(node.individualHumans)) human = node.individualHumans[0] self.assertEqual(1, human.cumulativeInfs) self.assertEqual(0, len(human.infections)) self.assertEqual(0, human.infectiousness) self.assertEqual(9598.48, human.m_age) self.assertEqual(0, human.m_gender) self.assertEqual(False, human.m_is_infected) return
def make_none_snappy(): dtk = dft.read('test-data/snappy.dtk') dtk.header['compressed'] = False dtk.header['engine'] = dft.NONE dft.write(dtk, 'test-data/none-snappy.dtk') return
def make_none_lz4(): dtk = dft.read('test-data/baseline.dtk') dtk.header['compressed'] = False dtk.header['engine'] = dft.NONE dft.write(dtk, 'test-data/none-lz4.dtk') return
def make_lz4_none(): dtk = dft.read('test-data/uncompressed.dtk') dtk.header['compressed'] = True dtk.header['engine'] = dft.LZ4 dft.write(dtk, 'test-data/lz4-none.dtk') return
def make_snappy_none(): dtk = dft.read('test-data/uncompressed.dtk') dtk.header['compressed'] = True dtk.header['engine'] = dft.SNAPPY dft.write(dtk, 'test-data/snappy-none.dtk') return
def make_lz4_snappy(): dtk = dft.read('test-data/snappy.dtk') dtk.header['compressed'] = True dtk.header['engine'] = dft.LZ4 dft.write(dtk, 'test-data/lz4-snappy.dtk') return
def make_snappy_lz4(): dtk = dft.read('test-data/baseline.dtk') dtk.header['compressed'] = True dtk.header['engine'] = dft.SNAPPY dft.write(dtk, 'test-data/snappy-lz4.dtk') return
def test_engine_data_mismatch_f(self): with self.assertRaises(UserWarning): dtk_file = dft.read( 'test-data/snappy-lz4.dtk') # hdr (SNAPPY) vs. actual (LZ4) # Accessing the simulation field raises the exception print( dtk_file.simulation.individualHumanSuidGenerator.next_suid.id) return
def make_bad_sim_snappy(): dtk = dft.read('test-data/snappy.dtk') sim_text = dtk.contents[0] sim_text = sim_text.replace('"__class__":"Simulation"', '"__class__"*"Simulation"', 1) dtk.contents[0] = sim_text dft.write(dtk, 'test-data/bad-sim-snappy.dtk') return
def __do_read__(args): if args.output is not None: prefix = args.output else: root, _ = os.path.splitext(args.filename) prefix = root if args.raw: extension = 'bin' else: extension = 'json' dtk_file = dft.read(args.filename) if args.header: with open(args.header, 'w') as handle: json.dump(dtk_file.header, handle, indent=2, separators=(',', ':')) print('File header: {0}'.format(dtk_file.header)) for index in range(len(dtk_file.chunks)): if args.raw: # Write raw chunks to disk output = dtk_file.chunks[index] else: if args.unformatted: # Expand compressed contents, but don't serialize and format output = dtk_file.contents[index] else: # Expand compressed contents, serialize, write out formatted obj = dtk_file.objects[index] print('Formatting chunk {0} of {1}... '.format(index+1, len(dtk_file.chunks)), end='') output = json.dumps(obj, indent=2, separators=(',', ':')) if index == 0: output_filename = '.'.join([prefix, 'simulation', extension]) else: output_filename = '.'.join([prefix, 'node-{:0>5}'.format(index), extension]) print("Writing file '{0}'".format(output_filename)) with open(output_filename, 'w') as handle: handle.write(output) return
def __do_read__(args): if args.output is not None: prefix = args.output else: root, _ = os.path.splitext(args.filename) prefix = root if args.raw: extension = 'bin' else: extension = 'json' dtk_file = dft.read(args.filename) if args.header: with open(args.header, 'wb') as handle: json.dump(dtk_file.header, handle, indent=2, separators=(',', ':')) print('File header: {0}'.format(dtk_file.header)) for index in range(len(dtk_file.chunks)): if args.raw: # Write raw chunks to disk output = dtk_file.chunks[index] else: if args.unformatted: # Expand compressed contents, but don't serialize and format output = dtk_file.contents[index] else: # Expand compressed contents, serialize, write out formatted obj = dtk_file.objects[index] output = json.dumps(obj, indent=2, separators=(',', ':')) if index == 0: output_filename = '.'.join([prefix, 'simulation', extension]) else: output_filename = '.'.join([prefix, 'node-{:0>5}'.format(index), extension]) with open(output_filename, 'wb') as handle: handle.write(output) return
def test_reading_uncompressed_file(self): jason_text = '{"simulation":{"__class__":"SimulationPython","serializationMask":0,"nodes":[]}}' dtk = dft.read('test-data/simple.dtk') # noinspection SpellCheckingInspection self.assertEqual('clorton', dtk.author) self.assertEqual('notepad', dtk.tool) self.assertEqual(False, dtk.compressed) self.assertEqual(dft.NONE, dtk.compression) self.assertEqual(80, dtk.byte_count) self.assertEqual(1, dtk.chunk_count) self.assertEqual(80, dtk.chunk_sizes[0]) self.assertEqual(jason_text, str(dtk.chunks[0], 'utf-8')) self.assertEqual(jason_text, str(dtk.contents[0], 'utf-8')) self.assertEqual('SimulationPython', dtk.simulation['__class__']) self.assertEqual(0, dtk.simulation.serializationMask) self.assertEqual(0, len(dtk.nodes)) return
def test_reading_uncompressed_file(self): jason_text = '{"simulation":{"__class__":"SimulationPython","serializationMask":0,"nodes":[]}}' dtk = dft.read('test-data/simple.dtk') # noinspection SpellCheckingInspection self.assertEqual('clorton', dtk.author) self.assertEqual('notepad', dtk.tool) self.assertEqual(False, dtk.compressed) self.assertEqual(dft.NONE, dtk.compression) self.assertEqual(80, dtk.byte_count) self.assertEqual(1, dtk.chunk_count) self.assertEqual(80, dtk.chunk_sizes[0]) self.assertEqual(jason_text, dtk.chunks[0]) self.assertEqual(jason_text, dtk.contents[0]) self.assertEqual('SimulationPython', dtk.simulation['__class__']) self.assertEqual(0, dtk.simulation.serializationMask) self.assertEqual(0, len(dtk.nodes)) return
def test_reading_compressed_file(self): dtk = dft.read('test-data/version1.dtk') self.assertEqual('', dtk.author) self.assertEqual('', dtk.tool) self.assertEqual(True, dtk.compressed) self.assertEqual(dft.SNAPPY, dtk.compression) self.assertEqual(1438033, dtk.byte_count) self.assertEqual(1, dtk.chunk_count) self.assertEqual(1438033, dtk.chunk_sizes[0]) self.assertEqual(1, dtk.simulation.Run_Number) self.assertEqual(10001, dtk.simulation.individualHumanSuidGenerator.next_suid.id) self.assertEqual(0, dtk.simulation.sim_type) self.assertEqual(4, len(dtk.nodes)) node = dtk.nodes[0] self.assertEqual(1, node.externalId) self.assertEqual(2500, len(node.individualHumans)) human = node.individualHumans[0] self.assertEqual(1, human.cumulativeInfs) self.assertEqual(0, len(human.infections)) self.assertEqual(0, human.infectiousness) self.assertEqual(9598.48, human.m_age) self.assertEqual(0, human.m_gender) self.assertEqual(False, human.m_is_infected) return
def make_bad_chunk_snappy(): dtk = dft.read('test-data/snappy.dtk') # get the chunk, first node, not simulation chunk = dtk.chunks[1] # choose an index index = random.randint(0, len(chunk)) # convert string to array of chars chunk = [c for c in chunk] # perturb the bits old = chunk[index] new = chr(~ord(old) % 256) chunk[index] = new # convert array of chars back to string chunk = ''.join(chunk) dtk.chunks[1] = chunk print('Flipped bits of chunk #1 byte {0} ({1} -> {2})'.format(index, ord(old), ord(new))) dft.write(dtk, 'test-data/bad-chunk-snappy.dtk') return
def test_reading_zero_chunk_size(self): with self.assertRaises(UserWarning): dft.read('test-data/zero-chunk-size.dtk') return
def test_reading_negative_chunk_size(self): with self.assertRaises(UserWarning): dft.read('test-data/neg-chunk-size.dtk') return
def test_engine_data_mismatch_f(self): with self.assertRaises(UserWarning): dtk_file = dft.read('test-data/snappy-lz4.dtk') # hdr (SNAPPY) vs. actual (LZ4) # Accessing the simulation field raises the exception print(dtk_file.simulation.individualHumanSuidGenerator.next_suid.id) return
def test_reading_truncated_file(self): with self.assertRaises(UserWarning): dft.read('test-data/truncated.dtk') # simulation and one node (truncated) return
def test_reading_wrong_magic_number(self): with self.assertRaises(UserWarning): dft.read('test-data/bad-magic.dtk') return
def test_reading_invalid_header(self): with self.assertRaises(UserWarning): dft.read('test-data/bad-header.dtk') return
def test_reading_negative_version(self): with self.assertRaises(UserWarning): dft.read('test-data/neg-version.dtk') return
def test_bad_chunk_snappy(self): with self.assertRaises(UserWarning): dtk_file = dft.read('test-data/bad-chunk-snappy.dtk') # Accessing the node raises the exception print(dtk_file.nodes[0].externalId) return
def test_reading_zero_version(self): with self.assertRaises(UserWarning): dft.read('test-data/zero-version.dtk') return
def test_reading_unknown_version(self): with self.assertRaises(UserWarning): dft.read('test-data/future-version.dtk') return
def test_reading_truncated_file(self): with self.assertRaises(UserWarning): dft.read('test-data/truncated.dtk' ) # simulation and one node (truncated) return
def test_bad_sim_chunk_snappy(self): with self.assertRaises(UserWarning): dtk_file = dft.read('test-data/bad-sim-snappy.dtk') # Accessing the simulation field raises the exception print(dtk_file.simulation.individualHumanSuidGenerator.next_suid.id) return
def __init__(self, file): self.nextInfectionSuid_suid = None self.nextInfectionSuid_initialized = False self.dtk = dft.read(file) self._nodes = [n for n in self.dtk.nodes]