def __do_write__(args):
print("Writing file '{0}'".format(args.filename), file=sys.stderr)
print("Reading simulation data from '{0}'".format(args.simulation),
file=sys.stderr)
print("Reading node data from {0}".format(args.nodes), file=sys.stderr)
print("Author = {0}".format(args.author), file=sys.stderr)
print("Tool = {0}".format(args.tool), file=sys.stderr)
print("{0} contents".format(
"Compressing" if args.compress else "Not compressing"),
file=sys.stderr)
print(
"{0} contents".format("Verifying" if args.verify else "Not verifying"),
file=sys.stderr)
print("Using compression engine '{0}'".format(args.engine),
file=sys.stderr)
dtk_file = dft.DtkFileV3()
dtk_file.author = args.author
dtk_file.tool = args.tool
dtk_file.compression = args.engine
_prepare_simulation_data(args.simulation, dtk_file)
_prepare_node_data(args.nodes, dtk_file)
dft.write(dtk_file, args.filename)
return
def make_snappy():
dtk = dft.read('test-data/baseline.dtk')
dtk.compression = dft.SNAPPY
dft.write(dtk, 'test-data/snappy.dtk')
return
def make_snappy():
dtk = dft.read('test-data/baseline.dtk')
dtk.compression = dft.SNAPPY
dft.write(dtk, 'test-data/snappy.dtk')
return
def test_round_trip(self):
source = dft.DtkFileV1()
source.author = 'dtkFileTests'
source.compression = dft.SNAPPY # Choose something other than the default
simulation = support.SerialObject({"simulation": {"enable_spatial_output": False, "Run_Number": 1, "enable_property_output": False, "infectionSuidGenerator": {"next_suid": {"id": 1}, "rank": 0, "numtasks": 1}, "__class__": "Simulation", "campaignFilename": "campaign.json", "nodes": [{"node": {"demographics_other": False, "birth_rate_sinusoidal_forcing_phase": 0, "sample_rate_20_plus": 0, "max_sampling_cell_pop": 0, "infectivity_boxcar_end_time": 0, "sample_rate_18mo_4yr": 0, "__class__": "Node", "infectivity_boxcar_start_time": 0, "demographics_gender": True, "individualHumans": [{"migration_is_destination_new_home": False, "home_node_id": {"id": 1}, "waiting_for_family_trip": False, "migration_will_return": True, "waypoints": [], "family_migration_type": 0, "Inf_Sample_Rate": 1, "migration_outbound": True, "m_mc_weight": 1, "migration_time_until_trip": 0, "StateChange": 0, "suid": {"id": 1}, "leave_on_family_trip": False, "interventions": {"drugVaccineReducedTransmit": 1, "drugVaccineReducedAcquire": 1, "__class__": "InterventionsContainer", "drugVaccineReducedMortality": 1, "interventions": []}, "migration_destination": {"id": 0}, "__class__": "IndividualHuman", "infectiousness": 0, "migration_mod": 1, "waypoints_trip_type": [], "m_age": 9588.48, "m_is_infected": False, "m_daily_mortality_rate": 0, "cumulativeInfs": 0, "family_migration_time_at_destination": 0, "family_migration_destination": {"id": 0}, "m_new_infection_state": 0, "migration_time_at_destination": 0, "family_migration_is_destination_new_home": False, "migration_type": 0, "max_waypoints": 0, "pregnancy_timer": 0, "family_migration_time_until_trip": 0, "is_pregnant": False, "susceptibility": {"trandecayoffset": 0, "age": 9588.48, "__class__": "Susceptibility", "acqdecayoffset": 0, "mod_mortality": 1, "mod_acquire": 1, "mod_transmit": 1, "mortdecayoffset": 0}, "is_on_family_trip": False, "infections": [], "above_poverty": 0, "Properties": [], "m_gender": 0}], "serializationMask": 3, "infectivity_sinusoidal_forcing_amplitude": 1, "birth_rate_boxcar_start_time": 0, "x_birth": 1, "sample_rate_immune": 0, "population_density_infectivity_correction": 0, "sample_rate_0_18mo": 0, "population_scaling": 0, "demographics_birth": False, "suid": {"id": 1}, "birth_rate_boxcar_forcing_amplitude": 1, "population_scaling_factor": 1, "birth_rate_boxcar_end_time": 0, "ind_sampling_type": 0, "vital_birth_time_dependence": 0, "externalId": 1, "age_initialization_distribution_type": 1, "infectivity_sinusoidal_forcing_phase": 0, "immune_threshold_for_downsampling": 0, "sample_rate_5_9": 0, "home_individual_ids": [{"key": 1, "value": {"id": 1}}], "population_density_c50": 0, "sample_rate_birth": 0, "birth_rate_sinusoidal_forcing_amplitude": 1, "infectivity_boxcar_forcing_amplitude": 1, "vital_birth": False, "maternal_transmission": False, "vital_birth_dependence": 1, "infectivity_scaling": 0, "animal_reservoir_type": 0, "zoonosis_rate": 0, "sample_rate_10_14": 0, "sample_rate_15_19": 0}, "suid": {"id": 1}}], "loadbalance_filename": "", "serializationMask": 3, "individualHumanSuidGenerator": {"next_suid": {"id": 1001}, "rank": 0, "numtasks": 1}, "sim_type": 0, "enable_default_report": True, "enable_event_report": False, "Ind_Sample_Rate": 1, "demographic_tracking": False}})
source.objects.append(simulation)
handle, filename = tempfile.mkstemp()
os.close(handle)
dft.write(source, filename)
dest = dft.read(filename)
self.assertEqual('dtkFileTests', dest.author)
self.assertEqual(dft.SNAPPY, dest.compression)
simulation = dest.simulation
self.assertEqual(False, simulation.enable_spatial_output)
self.assertEqual(1, simulation.Run_Number)
self.assertEqual(1001, simulation.individualHumanSuidGenerator.next_suid.id)
self.assertEqual(0, simulation.sim_type)
self.assertEqual(True, simulation.enable_default_report)
node = simulation.nodes[0].node # Simulation nodes is a map<suid,node>
self.assertEqual(1, node.externalId)
self.assertEqual(False, node.demographics_other)
self.assertEqual(True, node.demographics_gender)
self.assertEqual(1, node.x_birth)
individual = node.individualHumans[0]
self.assertEqual(1, individual.m_mc_weight)
self.assertEqual(1, individual.suid.id)
self.assertEqual(9588.48, individual.m_age)
self.assertEqual(0, individual.m_gender)
os.remove(filename)
return
def make_uncompressed():
dtk = dft.read('test-data/baseline.dtk')
dtk.compression = dft.NONE
dft.write(dtk, 'test-data/uncompressed.dtk')
return
def make_bad_chunk_snappy():
dtk = dft.read('test-data/snappy.dtk')
# get the chunk, first node, not simulation
chunk = dtk.chunks[1]
# choose an index
index = random.randint(0, len(chunk))
# convert string to array of chars
chunk = [c for c in chunk]
# perturb the bits
old = chunk[index]
new = chr(~ord(old) % 256)
chunk[index] = new
# convert array of chars back to string
chunk = ''.join(chunk)
dtk.chunks[1] = chunk
print('Flipped bits of chunk #1 byte {0} ({1} -> {2})'.format(
index, ord(old), ord(new)))
dft.write(dtk, 'test-data/bad-chunk-snappy.dtk')
return
def make_uncompressed():
dtk = dft.read('test-data/baseline.dtk')
dtk.compression = dft.NONE
dft.write(dtk, 'test-data/uncompressed.dtk')
return
def make_snappy_lz4():
dtk = dft.read('test-data/baseline.dtk')
dtk.header['compressed'] = True
dtk.header['engine'] = dft.SNAPPY
dft.write(dtk, 'test-data/snappy-lz4.dtk')
return
def make_none_snappy():
dtk = dft.read('test-data/snappy.dtk')
dtk.header['compressed'] = False
dtk.header['engine'] = dft.NONE
dft.write(dtk, 'test-data/none-snappy.dtk')
return
def make_none_lz4():
dtk = dft.read('test-data/baseline.dtk')
dtk.header['compressed'] = False
dtk.header['engine'] = dft.NONE
dft.write(dtk, 'test-data/none-lz4.dtk')
return
def make_lz4_snappy():
dtk = dft.read('test-data/snappy.dtk')
dtk.header['compressed'] = True
dtk.header['engine'] = dft.LZ4
dft.write(dtk, 'test-data/lz4-snappy.dtk')
return
def make_lz4_none():
dtk = dft.read('test-data/uncompressed.dtk')
dtk.header['compressed'] = True
dtk.header['engine'] = dft.LZ4
dft.write(dtk, 'test-data/lz4-none.dtk')
return
def make_lz4_none():
dtk = dft.read('test-data/uncompressed.dtk')
dtk.header['compressed'] = True
dtk.header['engine'] = dft.LZ4
dft.write(dtk, 'test-data/lz4-none.dtk')
return
def make_snappy_none():
dtk = dft.read('test-data/uncompressed.dtk')
dtk.header['compressed'] = True
dtk.header['engine'] = dft.SNAPPY
dft.write(dtk, 'test-data/snappy-none.dtk')
return
def make_snappy_none():
dtk = dft.read('test-data/uncompressed.dtk')
dtk.header['compressed'] = True
dtk.header['engine'] = dft.SNAPPY
dft.write(dtk, 'test-data/snappy-none.dtk')
return
def make_snappy_lz4():
dtk = dft.read('test-data/baseline.dtk')
dtk.header['compressed'] = True
dtk.header['engine'] = dft.SNAPPY
dft.write(dtk, 'test-data/snappy-lz4.dtk')
return
def make_none_snappy():
dtk = dft.read('test-data/snappy.dtk')
dtk.header['compressed'] = False
dtk.header['engine'] = dft.NONE
dft.write(dtk, 'test-data/none-snappy.dtk')
return
def make_none_lz4():
dtk = dft.read('test-data/baseline.dtk')
dtk.header['compressed'] = False
dtk.header['engine'] = dft.NONE
dft.write(dtk, 'test-data/none-lz4.dtk')
return
def make_lz4_snappy():
dtk = dft.read('test-data/snappy.dtk')
dtk.header['compressed'] = True
dtk.header['engine'] = dft.LZ4
dft.write(dtk, 'test-data/lz4-snappy.dtk')
return
def write(self):
sim = self.dtk.simulation
sim["infectionSuidGenerator"]['next_suid'][
'id'] = self.getNextInfectionSuid()
self.dtk.simulation = sim
self.dtk.compression = dft.NONE
# self.dtk.compressed = False
dft.write(self.dtk, "my_dtk_file.dtk")
def make_bad_sim_snappy():
dtk = dft.read('test-data/snappy.dtk')
sim_text = dtk.contents[0]
sim_text = sim_text.replace('"__class__":"Simulation"', '"__class__"*"Simulation"', 1)
dtk.contents[0] = sim_text
dft.write(dtk, 'test-data/bad-sim-snappy.dtk')
return
def make_bad_sim_snappy():
dtk = dft.read('test-data/snappy.dtk')
sim_text = dtk.contents[0]
sim_text = sim_text.replace('"__class__":"Simulation"',
'"__class__"*"Simulation"', 1)
dtk.contents[0] = sim_text
dft.write(dtk, 'test-data/bad-sim-snappy.dtk')
return
def write(self, output_file="my_sp_file.dtk"):
self.close()
sim = self.dtk.simulation
sim["infectionSuidGenerator"]['next_suid'] = self.getNextInfectionSuid(
)
self.dtk.simulation = sim
self.dtk.compression = dft.LZ4 #dft.NONE gives an error and dtkFileTools __write_chunks__
# self.dtk.compressed = False
dft.write(self.dtk, output_file)
def __do_write__(args):
print("Writing file '{0}'".format(args.filename), file=sys.stderr)
print("Reading simulation data from '{0}'".format(args.simulation), file=sys.stderr)
print("Reading node data from {0}".format(args.nodes), file=sys.stderr)
print("Author = {0}".format(args.author), file=sys.stderr)
print("Tool = {0}".format(args.tool), file=sys.stderr)
print("{0} contents".format("Compressing" if args.compress else "Not compressing"), file=sys.stderr)
print("{0} contents".format("Verifying" if args.verify else "Not verifying"), file=sys.stderr)
print("Using compression engine '{0}'".format(args.engine), file=sys.stderr)
dtk_file = dft.DtkFileV3()
dtk_file.author = args.author
dtk_file.tool = args.tool
dtk_file.compression = args.engine
_prepare_simulation_data(args.simulation, dtk_file)
_prepare_node_data(args.nodes, dtk_file)
dft.write(dtk_file, args.filename)
return
def make_bad_chunk_snappy():
dtk = dft.read('test-data/snappy.dtk')
# get the chunk, first node, not simulation
chunk = dtk.chunks[1]
# choose an index
index = random.randint(0, len(chunk))
# convert string to array of chars
chunk = [c for c in chunk]
# perturb the bits
old = chunk[index]
new = chr(~ord(old) % 256)
chunk[index] = new
# convert array of chars back to string
chunk = ''.join(chunk)
dtk.chunks[1] = chunk
print('Flipped bits of chunk #1 byte {0} ({1} -> {2})'.format(index, ord(old), ord(new)))
dft.write(dtk, 'test-data/bad-chunk-snappy.dtk')
return
