def __do_write__(args): print("Writing file '{0}'".format(args.filename), file=sys.stderr) print("Reading simulation data from '{0}'".format(args.simulation), file=sys.stderr) print("Reading node data from {0}".format(args.nodes), file=sys.stderr) print("Author = {0}".format(args.author), file=sys.stderr) print("Tool = {0}".format(args.tool), file=sys.stderr) print("{0} contents".format( "Compressing" if args.compress else "Not compressing"), file=sys.stderr) print( "{0} contents".format("Verifying" if args.verify else "Not verifying"), file=sys.stderr) print("Using compression engine '{0}'".format(args.engine), file=sys.stderr) dtk_file = dft.DtkFileV3() dtk_file.author = args.author dtk_file.tool = args.tool dtk_file.compression = args.engine _prepare_simulation_data(args.simulation, dtk_file) _prepare_node_data(args.nodes, dtk_file) dft.write(dtk_file, args.filename) return
def make_snappy(): dtk = dft.read('test-data/baseline.dtk') dtk.compression = dft.SNAPPY dft.write(dtk, 'test-data/snappy.dtk') return
def test_round_trip(self): source = dft.DtkFileV1() source.author = 'dtkFileTests' source.compression = dft.SNAPPY # Choose something other than the default simulation = support.SerialObject({"simulation": {"enable_spatial_output": False, "Run_Number": 1, "enable_property_output": False, "infectionSuidGenerator": {"next_suid": {"id": 1}, "rank": 0, "numtasks": 1}, "__class__": "Simulation", "campaignFilename": "campaign.json", "nodes": [{"node": {"demographics_other": False, "birth_rate_sinusoidal_forcing_phase": 0, "sample_rate_20_plus": 0, "max_sampling_cell_pop": 0, "infectivity_boxcar_end_time": 0, "sample_rate_18mo_4yr": 0, "__class__": "Node", "infectivity_boxcar_start_time": 0, "demographics_gender": True, "individualHumans": [{"migration_is_destination_new_home": False, "home_node_id": {"id": 1}, "waiting_for_family_trip": False, "migration_will_return": True, "waypoints": [], "family_migration_type": 0, "Inf_Sample_Rate": 1, "migration_outbound": True, "m_mc_weight": 1, "migration_time_until_trip": 0, "StateChange": 0, "suid": {"id": 1}, "leave_on_family_trip": False, "interventions": {"drugVaccineReducedTransmit": 1, "drugVaccineReducedAcquire": 1, "__class__": "InterventionsContainer", "drugVaccineReducedMortality": 1, "interventions": []}, "migration_destination": {"id": 0}, "__class__": "IndividualHuman", "infectiousness": 0, "migration_mod": 1, "waypoints_trip_type": [], "m_age": 9588.48, "m_is_infected": False, "m_daily_mortality_rate": 0, "cumulativeInfs": 0, "family_migration_time_at_destination": 0, "family_migration_destination": {"id": 0}, "m_new_infection_state": 0, "migration_time_at_destination": 0, "family_migration_is_destination_new_home": False, "migration_type": 0, "max_waypoints": 0, "pregnancy_timer": 0, "family_migration_time_until_trip": 0, "is_pregnant": False, "susceptibility": {"trandecayoffset": 0, "age": 9588.48, "__class__": "Susceptibility", "acqdecayoffset": 0, "mod_mortality": 1, "mod_acquire": 1, "mod_transmit": 1, "mortdecayoffset": 0}, "is_on_family_trip": False, "infections": [], "above_poverty": 0, "Properties": [], "m_gender": 0}], "serializationMask": 3, "infectivity_sinusoidal_forcing_amplitude": 1, "birth_rate_boxcar_start_time": 0, "x_birth": 1, "sample_rate_immune": 0, "population_density_infectivity_correction": 0, "sample_rate_0_18mo": 0, "population_scaling": 0, "demographics_birth": False, "suid": {"id": 1}, "birth_rate_boxcar_forcing_amplitude": 1, "population_scaling_factor": 1, "birth_rate_boxcar_end_time": 0, "ind_sampling_type": 0, "vital_birth_time_dependence": 0, "externalId": 1, "age_initialization_distribution_type": 1, "infectivity_sinusoidal_forcing_phase": 0, "immune_threshold_for_downsampling": 0, "sample_rate_5_9": 0, "home_individual_ids": [{"key": 1, "value": {"id": 1}}], "population_density_c50": 0, "sample_rate_birth": 0, "birth_rate_sinusoidal_forcing_amplitude": 1, "infectivity_boxcar_forcing_amplitude": 1, "vital_birth": False, "maternal_transmission": False, "vital_birth_dependence": 1, "infectivity_scaling": 0, "animal_reservoir_type": 0, "zoonosis_rate": 0, "sample_rate_10_14": 0, "sample_rate_15_19": 0}, "suid": {"id": 1}}], "loadbalance_filename": "", "serializationMask": 3, "individualHumanSuidGenerator": {"next_suid": {"id": 1001}, "rank": 0, "numtasks": 1}, "sim_type": 0, "enable_default_report": True, "enable_event_report": False, "Ind_Sample_Rate": 1, "demographic_tracking": False}}) source.objects.append(simulation) handle, filename = tempfile.mkstemp() os.close(handle) dft.write(source, filename) dest = dft.read(filename) self.assertEqual('dtkFileTests', dest.author) self.assertEqual(dft.SNAPPY, dest.compression) simulation = dest.simulation self.assertEqual(False, simulation.enable_spatial_output) self.assertEqual(1, simulation.Run_Number) self.assertEqual(1001, simulation.individualHumanSuidGenerator.next_suid.id) self.assertEqual(0, simulation.sim_type) self.assertEqual(True, simulation.enable_default_report) node = simulation.nodes[0].node # Simulation nodes is a map<suid,node> self.assertEqual(1, node.externalId) self.assertEqual(False, node.demographics_other) self.assertEqual(True, node.demographics_gender) self.assertEqual(1, node.x_birth) individual = node.individualHumans[0] self.assertEqual(1, individual.m_mc_weight) self.assertEqual(1, individual.suid.id) self.assertEqual(9588.48, individual.m_age) self.assertEqual(0, individual.m_gender) os.remove(filename) return
def make_uncompressed(): dtk = dft.read('test-data/baseline.dtk') dtk.compression = dft.NONE dft.write(dtk, 'test-data/uncompressed.dtk') return
def make_bad_chunk_snappy(): dtk = dft.read('test-data/snappy.dtk') # get the chunk, first node, not simulation chunk = dtk.chunks[1] # choose an index index = random.randint(0, len(chunk)) # convert string to array of chars chunk = [c for c in chunk] # perturb the bits old = chunk[index] new = chr(~ord(old) % 256) chunk[index] = new # convert array of chars back to string chunk = ''.join(chunk) dtk.chunks[1] = chunk print('Flipped bits of chunk #1 byte {0} ({1} -> {2})'.format( index, ord(old), ord(new))) dft.write(dtk, 'test-data/bad-chunk-snappy.dtk') return
def make_snappy_lz4(): dtk = dft.read('test-data/baseline.dtk') dtk.header['compressed'] = True dtk.header['engine'] = dft.SNAPPY dft.write(dtk, 'test-data/snappy-lz4.dtk') return
def make_none_snappy(): dtk = dft.read('test-data/snappy.dtk') dtk.header['compressed'] = False dtk.header['engine'] = dft.NONE dft.write(dtk, 'test-data/none-snappy.dtk') return
def make_none_lz4(): dtk = dft.read('test-data/baseline.dtk') dtk.header['compressed'] = False dtk.header['engine'] = dft.NONE dft.write(dtk, 'test-data/none-lz4.dtk') return
def make_lz4_snappy(): dtk = dft.read('test-data/snappy.dtk') dtk.header['compressed'] = True dtk.header['engine'] = dft.LZ4 dft.write(dtk, 'test-data/lz4-snappy.dtk') return
def make_lz4_none(): dtk = dft.read('test-data/uncompressed.dtk') dtk.header['compressed'] = True dtk.header['engine'] = dft.LZ4 dft.write(dtk, 'test-data/lz4-none.dtk') return
def make_snappy_none(): dtk = dft.read('test-data/uncompressed.dtk') dtk.header['compressed'] = True dtk.header['engine'] = dft.SNAPPY dft.write(dtk, 'test-data/snappy-none.dtk') return
def write(self): sim = self.dtk.simulation sim["infectionSuidGenerator"]['next_suid'][ 'id'] = self.getNextInfectionSuid() self.dtk.simulation = sim self.dtk.compression = dft.NONE # self.dtk.compressed = False dft.write(self.dtk, "my_dtk_file.dtk")
def make_bad_sim_snappy(): dtk = dft.read('test-data/snappy.dtk') sim_text = dtk.contents[0] sim_text = sim_text.replace('"__class__":"Simulation"', '"__class__"*"Simulation"', 1) dtk.contents[0] = sim_text dft.write(dtk, 'test-data/bad-sim-snappy.dtk') return
def write(self, output_file="my_sp_file.dtk"): self.close() sim = self.dtk.simulation sim["infectionSuidGenerator"]['next_suid'] = self.getNextInfectionSuid( ) self.dtk.simulation = sim self.dtk.compression = dft.LZ4 #dft.NONE gives an error and dtkFileTools __write_chunks__ # self.dtk.compressed = False dft.write(self.dtk, output_file)
def __do_write__(args): print("Writing file '{0}'".format(args.filename), file=sys.stderr) print("Reading simulation data from '{0}'".format(args.simulation), file=sys.stderr) print("Reading node data from {0}".format(args.nodes), file=sys.stderr) print("Author = {0}".format(args.author), file=sys.stderr) print("Tool = {0}".format(args.tool), file=sys.stderr) print("{0} contents".format("Compressing" if args.compress else "Not compressing"), file=sys.stderr) print("{0} contents".format("Verifying" if args.verify else "Not verifying"), file=sys.stderr) print("Using compression engine '{0}'".format(args.engine), file=sys.stderr) dtk_file = dft.DtkFileV3() dtk_file.author = args.author dtk_file.tool = args.tool dtk_file.compression = args.engine _prepare_simulation_data(args.simulation, dtk_file) _prepare_node_data(args.nodes, dtk_file) dft.write(dtk_file, args.filename) return
def make_bad_chunk_snappy(): dtk = dft.read('test-data/snappy.dtk') # get the chunk, first node, not simulation chunk = dtk.chunks[1] # choose an index index = random.randint(0, len(chunk)) # convert string to array of chars chunk = [c for c in chunk] # perturb the bits old = chunk[index] new = chr(~ord(old) % 256) chunk[index] = new # convert array of chars back to string chunk = ''.join(chunk) dtk.chunks[1] = chunk print('Flipped bits of chunk #1 byte {0} ({1} -> {2})'.format(index, ord(old), ord(new))) dft.write(dtk, 'test-data/bad-chunk-snappy.dtk') return