def test_get_descriptors(self):
g = gadget.Gadget(device_desc, fs_config_desc, hs_config_desc)
self.assertEquals(g.GetDeviceDescriptor(), device_desc)
self.assertEquals(g.GetFullSpeedConfigurationDescriptor(), fs_config_desc)
self.assertEquals(g.GetHighSpeedConfigurationDescriptor(), hs_config_desc)
with self.assertRaisesRegexp(RuntimeError, 'not connected'):
g.GetConfigurationDescriptor()
def test_set_interface(self):
g = gadget.Gadget(device_desc, fs_config_desc, hs_config_desc)
chip = mock.Mock()
g.Connected(chip, usb_constants.Speed.HIGH)
self.assertTrue(g.ControlWrite(0, 9, 1, 0, 0))
chip.reset_mock()
self.assertTrue(g.ControlWrite(1, 11, 1, 0, 0))
chip.StopEndpoint.assert_has_calls([
mock.call(0x01),
mock.call(0x81)
])
chip.StartEndpoint.assert_has_calls([
mock.call(hs_interrupt_in_endpoint_desc),
mock.call(hs_interrupt_out_endpoint_desc)
])
chip.reset_mock()
self.assertTrue(g.ControlWrite(1, 11, 0, 0, 0))
chip.StopEndpoint.assert_has_calls([
mock.call(0x01),
mock.call(0x81)
])
chip.StartEndpoint.assert_has_calls([
mock.call(hs_bulk_in_endpoint_desc),
mock.call(hs_bulk_out_endpoint_desc)
])
def test_connect_high_speed(self): g = gadget.Gadget(device_desc, fs_config_desc, hs_config_desc) g.Connected(mock.Mock(), usb_constants.Speed.HIGH) self.assertTrue(g.IsConnected()) self.assertEquals(g.GetSpeed(), usb_constants.Speed.HIGH) self.assertEquals(g.GetConfigurationDescriptor(), hs_config_desc) g.Disconnected() self.assertFalse(g.IsConnected())
def test_set_configuration_zero(self):
g = gadget.Gadget(device_desc, fs_config_desc, hs_config_desc)
chip = mock.Mock()
g.Connected(chip, usb_constants.Speed.HIGH)
g.ControlWrite(0, 9, 1, 0, 0)
chip.StartEndpoint.reset_mock()
g.ControlWrite(0, 9, 0, 0, 0)
chip.StopEndpoint.assert_has_calls([mock.call(0x01), mock.call(0x81)])
def test_set_configuration(self):
g = gadget.Gadget(device_desc, fs_config_desc, hs_config_desc)
chip = mock.Mock()
g.Connected(chip, usb_constants.Speed.HIGH)
g.ControlWrite(0, 9, 1, 0, 0)
chip.StartEndpoint.assert_has_calls([
mock.call(hs_bulk_in_endpoint_desc),
mock.call(hs_bulk_out_endpoint_desc)
])
def test_send_packet_disconnected(self):
g = gadget.Gadget(device_desc, fs_config_desc, hs_config_desc)
with self.assertRaisesRegexp(RuntimeError, 'not connected'):
g.SendPacket(0x81, 'Hello world!')
g.Connected(mock.Mock(), usb_constants.Speed.HIGH)
g.SendPacket(0x81, 'Hello world!')
g.Disconnected()
with self.assertRaisesRegexp(RuntimeError, 'not connected'):
g.SendPacket(0x81, 'Hello world!')
def test_get_microsoft_os_string_descriptor(self):
g = gadget.Gadget(device_desc, fs_config_desc, hs_config_desc)
g.EnableMicrosoftOSDescriptorsV1(vendor_code=0x42)
os_string_descriptor = g.ControlRead(0x80,
usb_constants.Request.GET_DESCRIPTOR,
0x03EE,
0x0000,
0x12)
self.assertEqual(os_string_descriptor,
"\x12\x03M\x00S\x00F\x00T\x001\x000\x000\x00\x42\x00")
def test_get_bos_descriptor(self):
g = gadget.Gadget(device_desc, fs_config_desc, hs_config_desc)
self.assertIsNone(g.ControlRead(0x80, 0x06, 0x0F00, 0x0000, 5))
container_id = uuid.uuid4()
g.AddDeviceCapabilityDescriptor(usb_descriptors.ContainerIdDescriptor(
ContainerID=container_id.bytes_le))
bos_descriptor_header = g.ControlRead(0x80, 0x06, 0x0F00, 0x0000, 5)
self.assertEquals('\x05\x0F\x19\x00\x01', bos_descriptor_header)
bos_descriptor = g.ControlRead(0x80, 0x06, 0x0F00, 0x0000, 25)
self.assertEquals(
'\x05\x0F\x19\x00\x01\x14\x10\x04\x00' + container_id.bytes_le,
bos_descriptor)
def test_get_microsoft_os_compat_id_descriptor(self):
g = gadget.Gadget(device_desc, fs_config_desc, hs_config_desc)
g.EnableMicrosoftOSDescriptorsV1(vendor_code=0x42)
g.SetMicrosoftCompatId(0, 'WINUSB')
chip = mock.Mock()
g.Connected(chip, usb_constants.Speed.HIGH)
expected_compatid_header = \
"\x28\x00\x00\x00\x00\x01\x04\x00\x01\0\0\0\0\0\0\0"
compatid_header = g.ControlRead(0xC0, 0x42, 0x0000, 0x0004, 0x0010)
self.assertEqual(compatid_header, expected_compatid_header)
compatid_descriptor = g.ControlRead(0xC0, 0x42, 0x0000, 0x0004, 0x0028)
self.assertEqual(compatid_descriptor,
expected_compatid_header +
"\x00\x01WINUSB\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0")
def test_get_microsoft_os_20_descriptor_set(self):
g = gadget.Gadget(device_desc, fs_config_desc, hs_config_desc)
g.EnableMicrosoftOSDescriptorsV2(vendor_code=0x42)
g.SetMicrosoftCompatId(0, 'WINUSB')
chip = mock.Mock()
g.Connected(chip, usb_constants.Speed.HIGH)
bos_descriptor = g.ControlRead(0x80, 0x06, 0x0F00, 0x0000, 33)
self.assertEquals(
'\x05\x0F\x21\x00\x01' +
'\x1C\x10\x05\x00' +
uuid.UUID('{D8DD60DF-4589-4CC7-9CD2-659D9E648A9F}').bytes_le +
'\x00\x00\x03\x06\x2E\x00\x42\x00',
bos_descriptor)
descriptor_set = g.ControlRead(0xC0, 0x42, 0x0000, 0x0007, 48)
self.assertEquals(
'\x0A\x00\x00\x00\x00\x00\x03\x06\x2E\x00' +
'\x08\x00\x01\x00\x00\x00\x24\x00' +
'\x08\x00\x02\x00\x00\x00\x1C\x00' +
'\x14\x00\x03\x00WINUSB\0\0\0\0\0\0\0\0\0\0',
descriptor_set)
def RunTest():
md = capstone.Cs(capstone.CS_ARCH_X86, capstone.CS_MODE_64)
md.detail = True
for data in dataList:
instList = list(md.disasm(data, 0))
inst = instList[0]
for inst in instList:
print(inst.bytes.hex().rjust(16), "\t", inst.mnemonic, inst.op_str)
gdgt = gadget.Gadget(archx64, instList)
print("")
print("Cost:", gdgt.cost())
print(gdgt.gadgetMatrix.printDep())
print("")
print("\n----------------------------------------------")
def test_set_bad_interface(self): g = gadget.Gadget(device_desc, fs_config_desc, hs_config_desc) g.Connected(mock.Mock(), usb_constants.Speed.HIGH) self.assertTrue(g.ControlWrite(0, 9, 1, 0, 0)) self.assertIsNone(g.ControlWrite(1, 11, 0, 1, 0))
def test_get_languages(self): g = gadget.Gadget(device_desc, fs_config_desc, hs_config_desc) g.AddStringDescriptor(1, 'Hello world!') desc = g.ControlRead(0x80, 6, 0x0300, 0, 255) self.assertEquals(desc, '\x04\x03\x09\x04')
def test_language_id_out_of_range(self):
g = gadget.Gadget(device_desc, fs_config_desc, hs_config_desc)
with self.assertRaisesRegexp(ValueError, 'language code out of range'):
g.AddStringDescriptor(1, 'Hello world!', lang=-1)
def test_halt_endpoint(self): g = gadget.Gadget(device_desc, fs_config_desc, hs_config_desc) chip = mock.Mock() g.Connected(chip, usb_constants.Speed.HIGH) g.HaltEndpoint(0x01) chip.HaltEndpoint.assert_called_once_with(0x01)
def test_get_string_descriptor(self): g = gadget.Gadget(device_desc, fs_config_desc, hs_config_desc) g.AddStringDescriptor(1, 'Hello world!') desc = g.ControlRead(0x80, 6, 0x0301, 0x0409, 255) self.assertEquals(desc, '\x1A\x03H\0e\0l\0l\0o\0 \0w\0o\0r\0l\0d\0!\0')
def test_send_invalid_endpoint(self):
g = gadget.Gadget(device_desc, fs_config_desc, hs_config_desc)
chip = mock.Mock()
g.Connected(chip, usb_constants.Speed.HIGH)
with self.assertRaisesRegexp(ValueError, 'non-input endpoint'):
g.SendPacket(0x01, 'Hello world!')
def test_receive_packet(self): g = gadget.Gadget(device_desc, fs_config_desc, hs_config_desc) self.assertIsNone(g.ReceivePacket(0x01, 'Hello world!'))
def test_send_packet(self): g = gadget.Gadget(device_desc, fs_config_desc, hs_config_desc) chip = mock.Mock() g.Connected(chip, usb_constants.Speed.HIGH) g.SendPacket(0x81, 'Hello world!') chip.SendPacket.assert_called_once_with(0x81, 'Hello world!')
def test_get_missing_string_language(self): g = gadget.Gadget(device_desc, fs_config_desc, hs_config_desc) g.AddStringDescriptor(1, 'Hello world!') desc = g.ControlRead(0x80, 6, 0x0301, 0x040C, 255) self.assertEquals(desc, None)
def test_string_index_out_of_range(self):
g = gadget.Gadget(device_desc, fs_config_desc, hs_config_desc)
with self.assertRaisesRegexp(ValueError, 'index out of range'):
g.AddStringDescriptor(0, 'Hello world!')
def make_bahamas_clusters_fof(data_dir,
save_dir,
number_of_clusters=200,
starting_cluster=0):
"""
Creates in the save_dir a hdf5 file for a number of clusters in the snapshot located in data_dir.
The hdf5 files contain the particle data of their corresponding cluster. The particles are selected based on their
FoF group number, which we treat as clusters.
Args:
data_dir: Directory containing the bahamas data (e.g. 'AGN_TUNED_nu0_L100N256_WMAP9')
save_dir: Directory to save the cluster hdf5 files in
number_of_clusters: Number of clusters for which to make hdf5 files
starting_cluster: The *number_of_clusters* largest clusters are selected, with index 0 the largest cluster.
"""
if not os.path.isdir(save_dir):
os.makedirs(save_dir, exist_ok=True)
# The original file contains all the particles in the simulation
# Eventually we want to make a new hdf5 file that contains only the particles from a certain cluster
path = os.path.join(
data_dir, 'data/particledata_032/eagle_subfind_particles_032.0.hdf5')
original_file = h5py.File(path, 'r')
groups = list(original_file.keys())
# Determine which groups contain particles and which don't.
particle_groups = []
non_particle_groups = []
for group in groups:
if 'PartType' in group:
particle_groups.append(group)
else:
non_particle_groups.append(group)
# Determine particle sub- and subsubgroups.
# For every group that contains particles,
# a selection needs to be taken that only contains the particles belonging to a particular cluster
particle_subgroups = {
group: list(original_file[group].keys())
for group in particle_groups
}
particle_subsubgroups = {
group: {
subgroup: list(original_file[group][subgroup].keys())
for subgroup in ['ElementAbundance', 'SmoothedElementAbundance']
}
for group in ['PartType0', 'PartType4']
}
print('Particle groups made')
# Load the particle data
snapnum = 32
# snapdata = g.Gadget(data_dir, "snap", snapnum, sim="BAHAMAS")
partdata = g.Gadget(data_dir, "particles", snapnum, sim="BAHAMAS")
print('Data loaded')
# Find the FoF group number for every particle for every particle type
fof_group_number_per_particle = {
ptype: partdata.read_var(ptype + '/GroupNumber', verbose=False)
for ptype in particle_groups
}
# Find the different FoF groups for every particle type by taking the set of the previous variable,
# taking the length of the set gives an estimate of the number of 'clusters' per particle type
fof_sets = {
ptype: list(set(fof_group_number_per_particle[ptype]))
for ptype in particle_groups
}
print('Data prepared')
for cluster in range(starting_cluster,
starting_cluster + number_of_clusters):
cluster_dir = os.path.join(save_dir, f'cluster_{cluster:03}')
if not os.path.isdir(cluster_dir):
os.makedirs(cluster_dir, exist_ok=True)
# Create a new file in which the cluster particles will be saved
with h5py.File(os.path.join(cluster_dir, f'cluster_{cluster:03}.hdf5'),
'w') as f2:
# Non-particle groups can just be copied
for group in non_particle_groups:
original_file.copy(group, f2)
# Take a subset of all particles which are present in the cluster
# and add their properties to the new hdf5 file
for group in particle_groups:
# Indices of the particles in the cluster subset for a certain particle type
inds = np.where(fof_group_number_per_particle[group] ==
fof_sets[group][cluster])[0]
# Make sure there are particles of the current type in the cluster subset
if len(inds) != 0:
for subgroup in particle_subgroups[group]:
# These subgroups are actual groups instead of datasets, so we need to go one layer deeper
if subgroup in [
'ElementAbundance', 'SmoothedElementAbundance'
]:
for subsubgroup in particle_subsubgroups[group][
subgroup]:
field = group + '/' + subgroup + '/' + subsubgroup
# Create a new dataset with the subset of the particles
f2.create_dataset(field,
data=partdata.read_var(
field,
verbose=False)[inds])
# Also add the attributes
for attr in list(
original_file[field].attrs.items()):
f2[field].attrs.create(attr[0], attr[1])
else:
# These 'subgroups' are datasets and can be added directly
field = group + '/' + subgroup
f2.create_dataset(field,
data=partdata.read_var(
field, verbose=False)[inds])
# Again also add the attributes
for attr in list(
original_file[field].attrs.items()):
f2[field].attrs.create(attr[0], attr[1])
print(f'cluster {cluster} particles are done')
def test_set_bad_configuration(self): g = gadget.Gadget(device_desc, fs_config_desc, hs_config_desc) g.Connected(mock.Mock(), usb_constants.Speed.HIGH) self.assertIsNone(g.ControlWrite(0, 9, 2, 0, 0))
def make_bahamas_clusters_box(data_dir,
save_dir,
number_of_clusters=200,
starting_cluster=0):
"""
Creates in the save_dir a npy file for a number of clusters in the snapshot located in data_dir.
The npy files contain the particle data of their corresponding cluster. The particles are selected on the basis
of being inside a fixed (physical) size box centered on the cluster. The size is determined by the field of view
of the Chandra Acis-I instrument, and the fixed redshift at which the clusters will be viewed.
Args:
data_dir: Directory containing the bahamas data (e.g. 'AGN_TUNED_nu0_L100N256_WMAP9')
save_dir: Directory to save the cluster hdf5 files in
number_of_clusters: Number of clusters for which to make hdf5 files
starting_cluster: The *number_of_clusters* largest clusters are selected, with index 0 the largest cluster.
"""
if os.path.basename(data_dir) == 'AGN_TUNED_nu0_L400N1024_WMAP9':
simulation_box_size = 400 # in simulation distance units
elif os.path.basename(data_dir) == 'AGN_TUNED_nu0_L100N256_WMAP9':
simulation_box_size = 100
else:
raise Exception(f'Snapshot {os.path.basename(data_dir)} is unknown!')
if not os.path.isdir(save_dir):
os.makedirs(save_dir, exist_ok=True)
gdata = g.Gadget(data_dir, 'subh', snapnum=32, sim='BAHAMAS')
subhalo_ids = [
int(idx) for idx in gdata.read_var('FOF/FirstSubhaloID', verbose=False)
]
centers = gdata.read_var('Subhalo/CentreOfPotential', verbose=False)
centers = centers[subhalo_ids[:-1]]
partdata = g.Gadget(data_dir, "particles", snapnum=32, sim="BAHAMAS")
coordinates = partdata.read_var('PartType0' + '/Coordinates',
verbose=False)
internal_energy = partdata.read_var('PartType0' + '/InternalEnergy',
verbose=False)
rho = partdata.read_var('PartType0' + '/Density', verbose=False)
properties = np.concatenate(
(coordinates, rho[..., None], internal_energy[..., None]), axis=1)
h = 0.7
# The factor sqrt(2) is to ensure that any projection of the simulation box still fills the fov
cutout_box_size = 4.7048754823854417e+24 * np.sqrt(2)
cluster_idx = starting_cluster
clusters_saved = 0
while clusters_saved < number_of_clusters:
lower_lim = centers[cluster_idx] - 0.5 * cutout_box_size * np.array(
[1, 1, 1])
upper_lim = centers[cluster_idx] + 0.5 * cutout_box_size * np.array(
[1, 1, 1])
indices = np.where((coordinates < lower_lim)
| (coordinates > upper_lim))[0]
cluster_properties = np.delete(properties, indices, axis=0)
print(cluster_idx, cluster_properties.shape)
# Currently only use clusters that are not on a periodic boundary
if any(lower_lim / gdata.cm_per_mpc * h < 0) or any(
upper_lim / gdata.cm_per_mpc * h > simulation_box_size):
print('split cluster')
print(lower_lim / gdata.cm_per_mpc * h,
upper_lim / gdata.cm_per_mpc * h)
else:
cluster_dir = os.path.join(save_dir, f'cluster_{cluster_idx:03}')
if not os.path.isdir(cluster_dir):
os.makedirs(cluster_dir, exist_ok=True)
np.save(os.path.join(cluster_dir, f'cluster_{cluster_idx:03}.npy'),
cluster_properties)
clusters_saved += 1
cluster_idx += 1
def test_class_and_vendor_transfers(self): g = gadget.Gadget(device_desc, fs_config_desc, hs_config_desc) self.assertIsNone(g.ControlRead(0xA0, 0, 0, 0, 0)) self.assertIsNone(g.ControlRead(0xC0, 0, 0, 0, 0)) self.assertIsNone(g.ControlWrite(0x20, 0, 0, 0, '')) self.assertIsNone(g.ControlWrite(0x40, 0, 0, 0, ''))
