def setUp(self):
self.seed = 1234567890
self.prng = random.RandomState(self.seed)
self.state = self.prng.get_state()
def setUp(self):
super(TestCase, self).setUp()
self._rng = npr.RandomState(zlib.adler32(
self._testMethodName.encode()))
def rand_nonzero(): post = lambda x: onp.where(x == 0, 1, x) randn = npr.RandomState(0).randn return partial(_rand_dtype, randn, scale=3, post=post)
def rand_small():
randn = npr.RandomState(0).randn
return partial(_rand_dtype, randn, scale=1e-3)
def rand_small_positive():
rand = npr.RandomState(0).rand
return partial(_rand_dtype, rand, scale=2e-5)
def rand_bool():
rng = npr.RandomState(0)
def generator(shape, dtype):
return _cast_to_shape(rng.rand(*_dims_of_shape(shape)) < 0.5, shape, dtype)
return generator
class CarlaDataProvider(object): # pylint: disable=too-many-public-methods
"""
This class provides access to various data of all registered actors
It buffers the data and updates it on every CARLA tick
Currently available data:
- Absolute velocity
- Location
- Transform
Potential additions:
- Acceleration
In addition it provides access to the map and the transform of all traffic lights
"""
_actor_velocity_map = dict()
_actor_location_map = dict()
_actor_transform_map = dict()
_traffic_light_map = dict()
_carla_actor_pool = dict()
_client = None
_world = None
_map = None
_sync_flag = False
_spawn_points = None
_spawn_index = 0
_blueprint_library = None
_ego_vehicle_route = None
_rng = random.RandomState(2000)
@staticmethod
def register_actor(actor):
"""
Add new actor to dictionaries
If actor already exists, throw an exception
"""
if actor in CarlaDataProvider._actor_velocity_map:
raise KeyError(
"Vehicle '{}' already registered. Cannot register twice!".
format(actor.id))
else:
CarlaDataProvider._actor_velocity_map[actor] = 0.0
if actor in CarlaDataProvider._actor_location_map:
raise KeyError(
"Vehicle '{}' already registered. Cannot register twice!".
format(actor.id))
else:
CarlaDataProvider._actor_location_map[actor] = None
if actor in CarlaDataProvider._actor_transform_map:
raise KeyError(
"Vehicle '{}' already registered. Cannot register twice!".
format(actor.id))
else:
CarlaDataProvider._actor_transform_map[actor] = None
@staticmethod
def register_actors(actors):
"""
Add new set of actors to dictionaries
"""
for actor in actors:
CarlaDataProvider.register_actor(actor)
@staticmethod
def on_carla_tick():
"""
Callback from CARLA
"""
for actor in CarlaDataProvider._actor_velocity_map:
if actor is not None and actor.is_alive:
CarlaDataProvider._actor_velocity_map[
actor] = calculate_velocity(actor)
for actor in CarlaDataProvider._actor_location_map:
if actor is not None and actor.is_alive:
CarlaDataProvider._actor_location_map[
actor] = actor.get_location()
for actor in CarlaDataProvider._actor_transform_map:
if actor is not None and actor.is_alive:
CarlaDataProvider._actor_transform_map[
actor] = actor.get_transform()
world = CarlaDataProvider._world
if world is None:
print("WARNING: CarlaDataProvider couldn't find the world")
@staticmethod
def get_velocity(actor):
"""
returns the absolute velocity for the given actor
"""
for key in CarlaDataProvider._actor_velocity_map:
if key.id == actor.id:
return CarlaDataProvider._actor_velocity_map[key]
# We are intentionally not throwing here
# This may cause exception loops in py_trees
print('{}.get_velocity: {} not found!'.format(__name__, actor))
return 0.0
@staticmethod
def get_location(actor):
"""
returns the location for the given actor
"""
for key in CarlaDataProvider._actor_location_map:
if key.id == actor.id:
return CarlaDataProvider._actor_location_map[key]
# We are intentionally not throwing here
# This may cause exception loops in py_trees
print('{}.get_location: {} not found!'.format(__name__, actor))
return None
@staticmethod
def get_transform(actor):
"""
returns the transform for the given actor
"""
for key in CarlaDataProvider._actor_transform_map:
if key.id == actor.id:
return CarlaDataProvider._actor_transform_map[key]
# We are intentionally not throwing here
# This may cause exception loops in py_trees
print('{}.get_transform: {} not found!'.format(__name__, actor))
return None
@staticmethod
def set_client(client):
"""
Set the CARLA client
"""
CarlaDataProvider._client = client
@staticmethod
def get_client():
"""
Get the CARLA client
"""
return CarlaDataProvider._client
@staticmethod
def set_world(world):
"""
Set the world and world settings
"""
CarlaDataProvider._world = world
CarlaDataProvider._sync_flag = world.get_settings().synchronous_mode
CarlaDataProvider._map = world.get_map()
CarlaDataProvider._blueprint_library = world.get_blueprint_library()
CarlaDataProvider.generate_spawn_points()
CarlaDataProvider.prepare_map()
@staticmethod
def get_world():
"""
Return world
"""
return CarlaDataProvider._world
@staticmethod
def get_map(world=None):
"""
Get the current map
"""
if CarlaDataProvider._map is None:
if world is None:
if CarlaDataProvider._world is None:
raise ValueError(
"class member \'world'\' not initialized yet")
else:
CarlaDataProvider._map = CarlaDataProvider._world.get_map()
else:
CarlaDataProvider._map = world.get_map()
return CarlaDataProvider._map
@staticmethod
def is_sync_mode():
"""
@return true if syncronuous mode is used
"""
return CarlaDataProvider._sync_flag
@staticmethod
def find_weather_presets():
"""
Get weather presets from CARLA
"""
rgx = re.compile(
'.+?(?:(?<=[a-z])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])|$)')
name = lambda x: ' '.join(m.group(0) for m in rgx.finditer(x))
presets = [
x for x in dir(carla.WeatherParameters) if re.match('[A-Z].+', x)
]
return [(getattr(carla.WeatherParameters, x), name(x))
for x in presets]
@staticmethod
def prepare_map():
"""
This function set the current map and loads all traffic lights for this map to
_traffic_light_map
"""
if CarlaDataProvider._map is None:
CarlaDataProvider._map = CarlaDataProvider._world.get_map()
# Parse all traffic lights
CarlaDataProvider._traffic_light_map.clear()
for traffic_light in CarlaDataProvider._world.get_actors().filter(
'*traffic_light*'):
if traffic_light not in CarlaDataProvider._traffic_light_map.keys(
):
CarlaDataProvider._traffic_light_map[
traffic_light] = traffic_light.get_transform()
else:
raise KeyError(
"Traffic light '{}' already registered. Cannot register twice!"
.format(traffic_light.id))
@staticmethod
def annotate_trafficlight_in_group(traffic_light):
"""
Get dictionary with traffic light group info for a given traffic light
"""
dict_annotations = {'ref': [], 'opposite': [], 'left': [], 'right': []}
# Get the waypoints
ref_location = CarlaDataProvider.get_trafficlight_trigger_location(
traffic_light)
ref_waypoint = CarlaDataProvider.get_map().get_waypoint(ref_location)
ref_yaw = ref_waypoint.transform.rotation.yaw
group_tl = traffic_light.get_group_traffic_lights()
for target_tl in group_tl:
if traffic_light.id == target_tl.id:
dict_annotations['ref'].append(target_tl)
else:
# Get the angle between yaws
target_location = CarlaDataProvider.get_trafficlight_trigger_location(
target_tl)
target_waypoint = CarlaDataProvider.get_map().get_waypoint(
target_location)
target_yaw = target_waypoint.transform.rotation.yaw
diff = (target_yaw - ref_yaw) % 360
if diff > 330:
continue
elif diff > 225:
dict_annotations['right'].append(target_tl)
elif diff > 135.0:
dict_annotations['opposite'].append(target_tl)
elif diff > 30:
dict_annotations['left'].append(target_tl)
return dict_annotations
@staticmethod
def get_trafficlight_trigger_location(traffic_light): # pylint: disable=invalid-name
"""
Calculates the yaw of the waypoint that represents the trigger volume of the traffic light
"""
def rotate_point(point, angle):
"""
rotate a given point by a given angle
"""
x_ = math.cos(math.radians(angle)) * point.x - math.sin(
math.radians(angle)) * point.y
y_ = math.sin(math.radians(angle)) * point.x - math.cos(
math.radians(angle)) * point.y
return carla.Vector3D(x_, y_, point.z)
base_transform = traffic_light.get_transform()
base_rot = base_transform.rotation.yaw
area_loc = base_transform.transform(
traffic_light.trigger_volume.location)
area_ext = traffic_light.trigger_volume.extent
point = rotate_point(carla.Vector3D(0, 0, area_ext.z), base_rot)
point_location = area_loc + carla.Location(x=point.x, y=point.y)
return carla.Location(point_location.x, point_location.y,
point_location.z)
@staticmethod
def update_light_states(ego_light,
annotations,
states,
freeze=False,
timeout=1000000000):
"""
Update traffic light states
"""
reset_params = []
for state in states:
relevant_lights = []
if state == 'ego':
relevant_lights = [ego_light]
else:
relevant_lights = annotations[state]
for light in relevant_lights:
prev_state = light.get_state()
prev_green_time = light.get_green_time()
prev_red_time = light.get_red_time()
prev_yellow_time = light.get_yellow_time()
reset_params.append({
'light': light,
'state': prev_state,
'green_time': prev_green_time,
'red_time': prev_red_time,
'yellow_time': prev_yellow_time
})
light.set_state(states[state])
if freeze:
light.set_green_time(timeout)
light.set_red_time(timeout)
light.set_yellow_time(timeout)
return reset_params
@staticmethod
def reset_lights(reset_params):
"""
Reset traffic lights
"""
for param in reset_params:
param['light'].set_state(param['state'])
param['light'].set_green_time(param['green_time'])
param['light'].set_red_time(param['red_time'])
param['light'].set_yellow_time(param['yellow_time'])
@staticmethod
def get_next_traffic_light(actor, use_cached_location=True):
"""
returns the next relevant traffic light for the provided actor
"""
if not use_cached_location:
location = actor.get_transform().location
else:
location = CarlaDataProvider.get_location(actor)
waypoint = CarlaDataProvider.get_map().get_waypoint(location)
# Create list of all waypoints until next intersection
list_of_waypoints = []
while waypoint and not waypoint.is_intersection:
list_of_waypoints.append(waypoint)
waypoint = waypoint.next(2.0)[0]
# If the list is empty, the actor is in an intersection
if not list_of_waypoints:
return None
relevant_traffic_light = None
distance_to_relevant_traffic_light = float("inf")
for traffic_light in CarlaDataProvider._traffic_light_map:
if hasattr(traffic_light, 'trigger_volume'):
tl_t = CarlaDataProvider._traffic_light_map[traffic_light]
transformed_tv = tl_t.transform(
traffic_light.trigger_volume.location)
distance = carla.Location(transformed_tv).distance(
list_of_waypoints[-1].transform.location)
if distance < distance_to_relevant_traffic_light:
relevant_traffic_light = traffic_light
distance_to_relevant_traffic_light = distance
return relevant_traffic_light
@staticmethod
def set_ego_vehicle_route(route):
"""
Set the route of the ego vehicle
@todo extend ego_vehicle_route concept to support multi ego_vehicle scenarios
"""
CarlaDataProvider._ego_vehicle_route = route
@staticmethod
def get_ego_vehicle_route():
"""
returns the currently set route of the ego vehicle
Note: Can be None
"""
return CarlaDataProvider._ego_vehicle_route
@staticmethod
def generate_spawn_points():
"""
Generate spawn points for the current map
"""
spawn_points = list(
CarlaDataProvider.get_map(
CarlaDataProvider._world).get_spawn_points())
CarlaDataProvider._rng.shuffle(spawn_points)
CarlaDataProvider._spawn_points = spawn_points
CarlaDataProvider._spawn_index = 0
@staticmethod
def create_blueprint(model,
rolename='scenario',
color=None,
actor_category="car"):
"""
Function to setup the blueprint of an actor given its model and other relevant parameters
"""
_actor_blueprint_categories = {
'car': 'vehicle.tesla.model3',
'van': 'vehicle.volkswagen.t2',
'truck': 'vehicle.carlamotors.carlacola',
'trailer': '',
'semitrailer': '',
'bus': 'vehicle.volkswagen.t2',
'motorbike': 'vehicle.kawasaki.ninja',
'bicycle': 'vehicle.diamondback.century',
'train': '',
'tram': '',
'pedestrian': 'walker.pedestrian.0001',
}
# Set the model
try:
blueprint = CarlaDataProvider._rng.choice(
CarlaDataProvider._blueprint_library.filter(model))
except IndexError:
# The model is not part of the blueprint library. Let's take a default one for the given category
bp_filter = "vehicle.*"
new_model = _actor_blueprint_categories[actor_category]
if new_model != '':
bp_filter = new_model
print("WARNING: Actor model {} not available. Using instead {}".
format(model, new_model))
blueprint = CarlaDataProvider._rng.choice(
CarlaDataProvider._blueprint_library.filter(bp_filter))
# Set the color
if color:
if not blueprint.has_attribute('color'):
print(
"WARNING: Cannot set Color ({}) for actor {} due to missing blueprint attribute"
.format(color, blueprint.id))
else:
default_color_rgba = blueprint.get_attribute(
'color').as_color()
default_color = '({}, {}, {})'.format(default_color_rgba.r,
default_color_rgba.g,
default_color_rgba.b)
try:
blueprint.set_attribute('color', color)
except ValueError:
# Color can't be set for this vehicle
print(
"WARNING: Color ({}) cannot be set for actor {}. Using instead: ({})"
.format(color, blueprint.id, default_color))
blueprint.set_attribute('color', default_color)
else:
if blueprint.has_attribute('color') and rolename != 'hero':
color = CarlaDataProvider._rng.choice(
blueprint.get_attribute('color').recommended_values)
blueprint.set_attribute('color', color)
# Make pedestrians mortal
if blueprint.has_attribute('is_invincible'):
blueprint.set_attribute('is_invincible', 'false')
# Set the rolename
if blueprint.has_attribute('role_name'):
blueprint.set_attribute('role_name', rolename)
return blueprint
@staticmethod
def handle_actor_batch(batch):
"""
Forward a CARLA command batch to spawn actors to CARLA, and gather the responses.
Returns list of actors on success, none otherwise
"""
actors = []
sync_mode = CarlaDataProvider.is_sync_mode()
if CarlaDataProvider._client and batch is not None:
responses = CarlaDataProvider._client.apply_batch_sync(
batch, sync_mode)
else:
return None
# wait for the actors to be spawned properly before we do anything
if sync_mode:
CarlaDataProvider._world.tick()
else:
CarlaDataProvider._world.wait_for_tick()
actor_ids = []
if responses:
for response in responses:
if not response.error:
actor_ids.append(response.actor_id)
carla_actors = CarlaDataProvider._world.get_actors(actor_ids)
for actor in carla_actors:
actors.append(actor)
return actors
@staticmethod
def request_new_actor(model,
spawn_point,
rolename='scenario',
autopilot=False,
random_location=False,
color=None,
actor_category="car"):
"""
This method tries to create a new actor, returning it if successful (None otherwise).
"""
blueprint = CarlaDataProvider.create_blueprint(model, rolename, color,
actor_category)
if random_location:
actor = None
while not actor:
spawn_point = CarlaDataProvider._rng.choice(
CarlaDataProvider._spawn_points)
actor = CarlaDataProvider._world.try_spawn_actor(
blueprint, spawn_point)
else:
# slightly lift the actor to avoid collisions with ground when spawning the actor
# DO NOT USE spawn_point directly, as this will modify spawn_point permanently
_spawn_point = carla.Transform(carla.Location(),
spawn_point.rotation)
_spawn_point.location.x = spawn_point.location.x
_spawn_point.location.y = spawn_point.location.y
# 0.2 -> 0.1
_spawn_point.location.z = spawn_point.location.z + 0.2
actor = CarlaDataProvider._world.try_spawn_actor(
blueprint, _spawn_point)
if actor is None:
raise RuntimeError(
"Error: Unable to spawn vehicle {} at {}".format(
blueprint.id, spawn_point))
else:
# Let's deactivate the autopilot of the actor if it belongs to vehicle
if actor in CarlaDataProvider._blueprint_library.filter(
'vehicle.*'):
actor.set_autopilot(autopilot)
else:
pass
# wait for the actor to be spawned properly before we do anything
if CarlaDataProvider.is_sync_mode():
CarlaDataProvider._world.tick()
else:
CarlaDataProvider._world.wait_for_tick()
if actor is None:
return None
CarlaDataProvider._carla_actor_pool[actor.id] = actor
CarlaDataProvider.register_actor(actor)
return actor
@staticmethod
def request_new_actors(actor_list):
"""
This method tries to series of actor in batch. If this was successful,
the new actors are returned, None otherwise.
param:
- actor_list: list of ActorConfigurationData
"""
SpawnActor = carla.command.SpawnActor # pylint: disable=invalid-name
PhysicsCommand = carla.command.SetSimulatePhysics # pylint: disable=invalid-name
FutureActor = carla.command.FutureActor # pylint: disable=invalid-name
ApplyTransform = carla.command.ApplyTransform # pylint: disable=invalid-name
SetAutopilot = carla.command.SetAutopilot # pylint: disable=invalid-name
batch = []
actors = []
CarlaDataProvider.generate_spawn_points()
for actor in actor_list:
# Get the blueprint
blueprint = CarlaDataProvider.create_blueprint(
actor.model, actor.rolename, actor.color, actor.category)
# Get the spawn point
transform = actor.transform
if actor.random_location:
if CarlaDataProvider._spawn_index >= len(
CarlaDataProvider._spawn_points):
print("No more spawn points to use")
break
else:
_spawn_point = CarlaDataProvider._spawn_points[
CarlaDataProvider._spawn_index]
CarlaDataProvider._spawn_index += 1
else:
_spawn_point = carla.Transform()
_spawn_point.rotation = transform.rotation
_spawn_point.location.x = transform.location.x
_spawn_point.location.y = transform.location.y
_spawn_point.location.z = transform.location.z + 0.2
# Get the command
command = SpawnActor(blueprint, _spawn_point)
command.then(SetAutopilot(FutureActor, actor.autopilot))
if actor.category == 'misc':
command.then(PhysicsCommand(FutureActor, True))
elif actor.args is not None and 'physics' in actor.args and actor.args[
'physics'] == "off":
command.then(ApplyTransform(FutureActor, _spawn_point)).then(
PhysicsCommand(FutureActor, False))
batch.append(command)
actors = CarlaDataProvider.handle_actor_batch(batch)
if not actors:
return None
for actor in actors:
if actor is None:
continue
CarlaDataProvider._carla_actor_pool[actor.id] = actor
CarlaDataProvider.register_actor(actor)
return actors
@staticmethod
def request_new_batch_actors(model,
amount,
spawn_points,
autopilot=False,
random_location=False,
rolename='scenario'):
"""
Simplified version of "request_new_actors". This method also create several actors in batch.
Instead of needing a list of ActorConfigurationData, an "amount" parameter is used.
This makes actor spawning easier but reduces the amount of configurability.
Some parameters are the same for all actors (rolename, autopilot and random location)
while others are randomized (color)
"""
SpawnActor = carla.command.SpawnActor # pylint: disable=invalid-name
SetAutopilot = carla.command.SetAutopilot # pylint: disable=invalid-name
FutureActor = carla.command.FutureActor # pylint: disable=invalid-name
CarlaDataProvider.generate_spawn_points()
batch = []
for i in range(amount):
# Get vehicle by model
blueprint = CarlaDataProvider.create_blueprint(model, rolename)
if random_location:
if CarlaDataProvider._spawn_index >= len(
CarlaDataProvider._spawn_points):
print(
"No more spawn points to use. Spawned {} actors out of {}"
.format(i + 1, amount))
break
else:
spawn_point = CarlaDataProvider._spawn_points[
CarlaDataProvider._spawn_index]
CarlaDataProvider._spawn_index += 1
else:
try:
spawn_point = spawn_points[i]
except IndexError:
print(
"The amount of spawn points is lower than the amount of vehicles spawned"
)
break
if spawn_point:
batch.append(
SpawnActor(blueprint, spawn_point).then(
SetAutopilot(FutureActor, autopilot)))
actors = CarlaDataProvider.handle_actor_batch(batch)
if actors is None:
return None
###############################################
# code for walker ai controllers: currently not working due to seg fault
if 'walker' in model:
walkers_list = []
for actor in actors:
walkers_list.append({"id": actor.id})
batch = []
walker_controller_bp = CarlaDataProvider._world.get_blueprint_library(
).find('controller.ai.walker')
for i in range(len(walkers_list)):
batch.append(
SpawnActor(walker_controller_bp, carla.Transform(),
walkers_list[i]["id"]))
actors_controllers = CarlaDataProvider.handle_actor_batch(batch)
for j, actor in enumerate(actors_controllers):
walkers_list[j]['con'] = actor.id
# CarlaDataProvider._world.wait_for_tick()
print('start')
for i in range(len(actors_controllers)):
print(i, '/', len(actors_controllers))
# start walker
actors_controllers[i].start()
# set walk to random point
actors_controllers[i].go_to_location(
CarlaDataProvider._world.
get_random_location_from_navigation())
# random max speed
actors_controllers[i].set_max_speed(
1 + random.random()
) # max speed between 1 and 2 (default is 1.4 m/s)
print('finish')
###############################################
for actor in actors:
if actor is None:
continue
CarlaDataProvider._carla_actor_pool[actor.id] = actor
CarlaDataProvider.register_actor(actor)
return actors
@staticmethod
def get_actors():
"""
Return list of actors and their ids
Note: iteritems from six is used to allow compatibility with Python 2 and 3
"""
return iteritems(CarlaDataProvider._carla_actor_pool)
@staticmethod
def actor_id_exists(actor_id):
"""
Check if a certain id is still at the simulation
"""
if actor_id in CarlaDataProvider._carla_actor_pool:
return True
return False
@staticmethod
def get_hero_actor():
"""
Get the actor object of the hero actor if it exists, returns none otherwise.
"""
for actor_id in CarlaDataProvider._carla_actor_pool:
if CarlaDataProvider._carla_actor_pool[actor_id].attributes[
'role_name'] == 'hero':
return CarlaDataProvider._carla_actor_pool[actor_id]
return None
@staticmethod
def get_actor_by_id(actor_id):
"""
Get an actor from the pool by using its ID. If the actor
does not exist, None is returned.
"""
if actor_id in CarlaDataProvider._carla_actor_pool:
return CarlaDataProvider._carla_actor_pool[actor_id]
print("Non-existing actor id {}".format(actor_id))
return None
@staticmethod
def remove_actor_by_id(actor_id):
"""
Remove an actor from the pool using its ID
"""
if actor_id in CarlaDataProvider._carla_actor_pool:
CarlaDataProvider._carla_actor_pool[actor_id].destroy()
CarlaDataProvider._carla_actor_pool[actor_id] = None
CarlaDataProvider._carla_actor_pool.pop(actor_id)
else:
print(
"Trying to remove a non-existing actor id {}".format(actor_id))
@staticmethod
def remove_actors_in_surrounding(location, distance):
"""
Remove all actors from the pool that are closer than distance to the
provided location
"""
for actor_id in CarlaDataProvider._carla_actor_pool.copy():
if CarlaDataProvider._carla_actor_pool[actor_id].get_location(
).distance(location) < distance:
CarlaDataProvider._carla_actor_pool[actor_id].destroy()
CarlaDataProvider._carla_actor_pool.pop(actor_id)
# Remove all keys with None values
CarlaDataProvider._carla_actor_pool = dict({
k: v
for k, v in CarlaDataProvider._carla_actor_pool.items() if v
})
@staticmethod
def cleanup():
"""
Cleanup and remove all entries from all dictionaries
"""
DestroyActor = carla.command.DestroyActor # pylint: disable=invalid-name
batch = []
for actor_id in CarlaDataProvider._carla_actor_pool.copy():
batch.append(
DestroyActor(CarlaDataProvider._carla_actor_pool[actor_id]))
if CarlaDataProvider._client:
try:
CarlaDataProvider._client.apply_batch_sync(batch)
except RuntimeError as e:
if "time-out" in str(e):
pass
else:
raise e
CarlaDataProvider._actor_velocity_map.clear()
CarlaDataProvider._actor_location_map.clear()
CarlaDataProvider._actor_transform_map.clear()
CarlaDataProvider._traffic_light_map.clear()
CarlaDataProvider._map = None
CarlaDataProvider._world = None
CarlaDataProvider._sync_flag = False
CarlaDataProvider._ego_vehicle_route = None
CarlaDataProvider._carla_actor_pool = dict()
CarlaDataProvider._client = None
CarlaDataProvider._spawn_points = None
CarlaDataProvider._spawn_index = 0
def _get_random_stream(self):
""" Returns a new RNG stream. DO NOT ACCESS FROM OUTSIDE.
"""
return rand.RandomState(self._master_RNG())
def _conv_layer_params(self):
weights_shape = 2, 2, 3, 4
biases_shape = weights_shape[-1:]
weights = array_ops.constant(npr.RandomState(0).randn(*weights_shape))
biases = array_ops.constant(npr.RandomState(1).randn(*biases_shape))
return (weights, biases)
def init_random_params(scale, layer_sizes, rng=npr.RandomState(0)):
return [(scale * rng.randn(m, n), scale * rng.randn(n))
for m, n, in zip(layer_sizes[:-1], layer_sizes[1:])]
def seed(self, new_seed=None):
self._seed = new_seed
self._master_RNG_fct = rand.RandomState(new_seed)
self._master_RNG = (
lambda: self._master_RNG_fct.random_integers(2**16 - 1))
def pick_backend(backend: str = None, samples: int = None, noise: NoiseModel = None, device=None,
exclude_symbolic: bool = True) -> str:
"""
choose, or verify, a backend for the user.
Parameters
----------
backend: str, optional:
what backend to choose or verify. if None: choose for the user.
samples: int, optional:
if int and not None, choose (verify) a simulator which supports sampling.
noise: str or NoiseModel, optional:
if not None, choose (verify) a simulator supports the specified noise.
device: optional:
verify that a given backend supports the specified device. MUST specify backend, if not None.
if None: do not emulate or use real device.
exclude_symbolic: bool, optional:
whether or not to exclude the tequila debugging simulator from the available simulators, when choosing.
Returns
-------
str:
the name of the chosen (or verified) backend.
"""
if len(INSTALLED_SIMULATORS) == 0:
raise TequilaException("No simulators installed on your system")
if backend is None and device is not None:
raise TequilaException('device use requires backend specification!')
if backend is None:
if noise is None:
if samples is None:
for f in SUPPORTED_BACKENDS:
if f in INSTALLED_SIMULATORS:
return f
else:
for f in INSTALLED_SAMPLERS.keys():
return f
else:
if samples is None:
raise TequilaException(
"Noise requires sampling; please provide a positive, integer value for samples")
for f in SUPPORTED_NOISE_BACKENDS:
if noise == 'device':
raise TequilaException('device noise requires a device, which requires a named backend!')
else:
return f
raise TequilaException(
'Could not find any installed sampler!')
if hasattr(backend, "lower"):
backend = backend.lower()
if backend == "random":
if device is not None:
raise TequilaException('cannot ask for a random backend and a specific device!')
from numpy import random as random
import time
state = random.RandomState(int(str(time.process_time()).split('.')[-1]) % 2 ** 32)
if samples is None:
backend = state.choice(list(INSTALLED_SIMULATORS.keys()), 1)[0]
else:
backend = state.choice(list(INSTALLED_SAMPLERS.keys()), 1)[0]
if exclude_symbolic:
while (backend == "symbolic"):
backend = state.choice(list(INSTALLED_SIMULATORS.keys()), 1)[0]
return backend
if device is not None and samples is None:
raise TequilaException('Use of a device requires sampling!')
if noise == 'device' and device is None:
raise TequilaException('Use of device noise requires a device!')
if backend not in SUPPORTED_BACKENDS:
raise TequilaException("Backend {backend} not supported ".format(backend=backend))
elif noise is False and samples is None and backend not in INSTALLED_SIMULATORS.keys():
raise TequilaException("Backend {backend} not installed ".format(backend=backend))
elif noise is False and samples is not None and backend not in INSTALLED_SAMPLERS.keys():
raise TequilaException("Backend {backend} not installed or sampling not supported".format(backend=backend))
elif noise is not False and samples is not None and backend not in INSTALLED_NOISE_SAMPLERS.keys():
raise TequilaException(
"Backend {backend} not installed or else Noise has not been implemented".format(backend=backend))
return backend
def indexed_loss_fun(w, L2_vect, i_iter):
rs = npr.RandomState(npr.RandomState(global_seed + i_hyper).randint(1000))
seed = i_hyper * 10**6 + i_iter # Deterministic seed needed for backwards pass.
idxs = rs.randint(N_train, size=batch_size)
return loss_fun(w, train_data['X'][idxs], train_data['T'][idxs], L2_vect)
N_classes = 10
N_train = 1000
N_valid = 10**3
N_tests = 10**3
N_batches = N_train / batch_size
N_iters = N_epochs * N_batches
# ----- Initial values of learned hyper-parameters -----
init_log_L2_reg = 0.0
init_log_alphas = -2.0
init_invlogit_betas = inv_logit(0.9)
init_log_param_scale = -2.0
# ----- Superparameters -----
meta_alpha = 0.05
N_meta_iter = 100
global_seed = npr.RandomState(3).randint(1000)
def fill_parser(parser, items):
partial_vects = [np.full(parser[name].size, items[i])
for i, name in enumerate(parser.names)]
return np.concatenate(partial_vects, axis=0)
def run():
train_data, valid_data, tests_data = load_data_dicts(N_train, N_valid, N_tests)
parser, pred_fun, loss_fun, frac_err = make_nn_funs(layer_sizes)
N_weight_types = len(parser.names)
hyperparams = VectorParser()
hyperparams['log_L2_reg'] = np.full(N_weight_types, init_log_L2_reg)
hyperparams['log_param_scale'] = np.full(N_weight_types, init_log_param_scale)
hyperparams['log_alphas'] = np.full(N_iters, init_log_alphas)
def _make_tensor(self): x = array_ops.placeholder(dtypes.float64, (3, 1)) w = array_ops.constant(npr.RandomState(0).randn(3, 3)) y = math_ops.matmul(w, x) g = gradients_impl.gradients(y, x)[0] return g
def __init__(self,
traindata,
x,
y,
input_len,
kernel,
stride,
Epsilon,
sigma=1.0,
learning_rate=0.5,
decay_function=asymptotic_decay,
neighborhood_function='gaussian',
topology='rectangular',
activation_distance='euclidean',
random_seed=None):
"""Initializes a Self Organizing Maps.
A rule of thumb to set the size of the grid for a dimensionality
reduction task is that it should contain 5*sqrt(N) neurons
where N is the number of samples in the dataset to analyze.
E.g. if your dataset has 150 samples, 5*sqrt(150) = 61.23
hence a map 8-by-8 should perform well.
Parameters
----------
x : int
x dimension of the SOM.
y : int
y dimension of the SOM.
input_len : int
Number of the elements of the vectors in input.
sigma : float, optional (default=1.0)
Spread of the neighborhood function, needs to be adequate
to the dimensions of the map.
(at the iteration t we have sigma(t) = sigma / (1 + t/T)
where T is #num_iteration/2)
learning_rate : initial learning rate
(at the iteration t we have
learning_rate(t) = learning_rate / (1 + t/T)
where T is #num_iteration/2)
decay_function : function (default=None)
Function that reduces learning_rate and sigma at each iteration
the default function is:
learning_rate / (1+t/(max_iterarations/2))
A custom decay function will need to to take in input
three parameters in the following order:
1. learning rate
2. current iteration
3. maximum number of iterations allowed
Note that if a lambda function is used to define the decay
MiniSom will not be pickable anymore.
neighborhood_function : string, optional (default='gaussian')
Function that weights the neighborhood of a position in the map.
Possible values: 'gaussian', 'mexican_hat', 'bubble', 'triangle'
topology : string, optional (default='rectangular')
Topology of the map.
Possible values: 'rectangular', 'hexagonal'
activation_distance : string, optional (default='euclidean')
Distance used to activate the map.
Possible values: 'euclidean', 'cosine', 'manhattan', 'chebyshev'
random_seed : int, optional (default=None)
Random seed to use.
"""
if sigma >= x or sigma >= y:
warn('Warning: sigma is too high for the dimension of the map.')
self._random_generator = random.RandomState(
random_seed) #生成符合正态分布的随机数[0,1]
self._learning_rate = learning_rate #初始学习率
self._sigma = sigma #邻域函数初始值为1，代表100%覆盖，迭代的过程中，sigma的值会随迭代次数不断减小，sigma(t) = sigma / (1 + t/T)
self._input_len = input_len #输入向量的维度，也是权值的维度
# random initialization
self._weights = self._random_generator.rand(
x, y, input_len) * 2 - 1 #生成符合正态分布的权重矩阵
self._weights /= linalg.norm(
self._weights, axis=-1, keepdims=True
) #权值标准化，linalg.norm求范数，默认是二范数，axis=-1，按行向量求范数，keepdims保持二维特性
self._activation_map = zeros((x, y)) #生成和网络一样大小的零矩阵，用来记录
self._neigx = arange(x) #得到一个列表[0,1,...,x-1]
self._neigy = arange(y) # 用来求邻域函数矩阵的值
if topology not in ['hexagonal', 'rectangular']: #判断是蜂窝网络还是矩阵网络
msg = '%s not supported only hexagonal and rectangular available'
raise ValueError(msg % topology)
self.topology = topology #确定网络属性
self._xx, self._yy = meshgrid(self._neigx, self._neigy) #生成网络坐标矩阵
self._xx = self._xx.astype(float) #坐标类型转换成浮点型
self._yy = self._yy.astype(float)
if topology == 'hexagonal':
self._xx[::-2] -= 0.5
if neighborhood_function in ['triangle']:
warn('triangle neighborhood function does not ' +
'take in account hexagonal topology')
self._decay_function = decay_function #学习率递减函数，learning_rate / (1+t/(max_iter/2))
neig_functions = {
'gaussian': self._gaussian,
}
if neighborhood_function not in neig_functions:
msg = '%s not supported. Functions available: %s'
raise ValueError(
msg %
(neighborhood_function, ', '.join(neig_functions.keys())))
if neighborhood_function in [
'triangle', 'bubble'
] and (divmod(sigma, 1)[1] != 0 or sigma < 1):
warn('sigma should be an integer >=1 when triangle or bubble' +
'are used as neighborhood function')
self.neighborhood = neig_functions[neighborhood_function] #确定领域函数
distance_functions = {
'euclidean': self._euclidean_distance,
}
if activation_distance not in distance_functions:
msg = '%s not supported. Distances available: %s'
raise ValueError(
msg %
(activation_distance, ', '.join(distance_functions.keys())))
self._activation_distance = distance_functions[
activation_distance] #相似度计算函数，一般采用欧氏距离，这里如果采用模糊的话，要在这里更改，添加一个Fuzzy距离函数
self.kernel = kernel # 是一个滑动窗口的尺寸，如[3,3]
self.stride = stride # 滑动窗口的步长
self.Epsilon = Epsilon # 模糊训练截止的条件
self.w_distance = 100 #记录权值的变化大小，用于终止训练
self.spldata = [] # 里面是以图片为单位的data
self.allfeature = [] # 里面是所有图片的所有特征，用于初始化权重和获取训练误差
self.traindata = traindata
self.processdata(self.traindata)
self.x = x
self.y = y
def rand_int(low, high=None):
randint = npr.RandomState(0).randint
def fn(shape, dtype):
return randint(low, high=high, size=shape, dtype=dtype)
return fn
def main(opts):
USAGE = "usage: %prog [options] image_filename"
parser = optparse.OptionParser(
usage=USAGE,
description=
'''A stegnography algorithm that works by reading and writting 3bytes at a time
hence the name''')
parser.add_option('-w',
'--write',
action='store_true',
help='Write message data to image file.')
parser.add_option('-f',
'--file',
dest='FN',
help='File to read message data from.')
parser.add_option('-o',
'--output',
dest='out',
type=str,
default='./out.png',
help='Output filename to write new image in write mode.')
parser.add_option(
'-r',
'--read',
action='store_true',
help=
'Read message data from image file to stdout (redirect to file a if you want).'
)
parser.add_option('-v',
'--verbosity',
action='store_true',
help='Set verbosity.')
parser.add_option(
'-p',
'--paranoid',
action='store_true',
help=
'Performs a read operation after writting to check if message data was written correctly.'
)
parser.add_option('-s',
'--seed',
default=1937007975,
type=int,
help='The seed for the algo\'s PRNG default=1937007975.')
(options, args) = parser.parse_args()
if (len(args) == 0) or (options.write == None and options.read == None):
parser.print_help()
sys.exit(1)
if (options.write and options.read):
print("[!] Cannot have both read and write options passed at once.")
sys.exit(2)
if (options.read and options.paranoid):
print("[-] Option paranoia would have not effect on a read operation.")
prog_inst = ThreeFactorStegAlgo(args[0], options.out, options.seed,
options.verbosity)
if (options.write):
try:
with open(options.FN, 'r') as msg_file:
data = msg_file.read()
prog_inst.write(data)
if (options.paranoid):
prog_inst.wout(
"[+] Paranoia mode: Checking to see if message data was written correctly."
)
prog_inst.img_inst = Image.open(options.out)
prog_inst.rand_gen = random.RandomState(options.seed)
prog_inst.verbosity = False
msg = prog_inst.read()
check = cmp(msg, data)
if (check == 0):
print("[+] Message data was written correctly")
return 0
else:
print("[-] Message data was not written correctly,")
print(
" try changing a few parameters like seed or better still use a larger image"
)
return -1
except IOError as io:
sys.stderr.write("[!] 0x%08X: Could not find %s: %s.\n" %
(io.errno, io.filename, io.strerror))
sys.exit(io.errno)
if (options.read):
print prog_inst.read()
def __init__(self,
x,
y,
input_len,
sigma=1.0,
learning_rate=0.5,
decay_function=None,
neighborhood_function='gaussian',
random_seed=None,
trained_weights=None,
normalize_weights=True,
neigh_threshold=0.0):
"""Initializes a Self Organizing Maps.
Parameters
----------
decision_tree : decision tree
The decision tree to be exported.
x : int
x dimension of the SOM
y : int
y dimension of the SOM
input_len : int
Number of the elements of the vectors in input.
sigma : float, optional (default=1.0)
Spread of the neighborhood function, needs to be adequate
to the dimensions of the map.
(at the iteration t we have sigma(t) = sigma / (1 + t/T)
where T is #num_iteration/2)
learning_rate, initial learning rate
(at the iteration t we have
learning_rate(t) = learning_rate / (1 + t/T)
where T is #num_iteration/2)
decay_function : function (default=None)
Function that reduces learning_rate and sigma at each iteration
default function:
lambda x, current_iteration, max_iter :
x/(1+current_iteration/max_iter)
neighborhood_function : function, optional (default='gaussian')
Function that weights the neighborhood of a position in the map
possible values: 'gaussian', 'mexican_hat'
random_seed : int, optiona (default=None)
Random seed to use.
trained_weights : initial weights (to load from previously saved ones)
normalize_weights : whether to normalize weights (default: True)
neigh_threshold : limit neighborhood to cells where g[] is bigger than this value times g[bmu]
"""
if sigma >= x / 2.0 or sigma >= y / 2.0:
warn('Warning: sigma is too high for the dimension of the map.')
if random_seed:
self._random_generator = random.RandomState(random_seed)
else:
self._random_generator = random.RandomState(random_seed)
if decay_function:
self._decay_function = decay_function
else:
self._decay_function = lambda x, t, max_iter: x / (1 + t / max_iter
)
self._learning_rate = learning_rate
self._sigma = sigma
self._randomSeed = random_seed
self._normalizeWeights = normalize_weights
self._neigh_threshold = neigh_threshold
if trained_weights is not None:
# There's a pre-trained Weight matrix
assert trained_weights.shape[0] == x and trained_weights.shape[
1] == y
self._weights = trained_weights
else:
# New training -> random initialization
# Random inizialization between (-1,1)
self._weights = self._random_generator.rand(x, y,
input_len) * 2 - 1
if self._normalizeWeights:
for i in range(x):
for j in range(y):
# normalization
norm = fast_norm(self._weights[i, j])
self._weights[i, j] = self._weights[i, j] / norm
self._activation_map = zeros((x, y))
self._neigx = arange(x)
self._neigy = arange(y) # used to evaluate the neighborhood function
neig_functions = {
'gaussian': self._gaussian,
'mexican_hat': self._mexican_hat
}
if neighborhood_function not in neig_functions:
msg = '%s not supported. Functions available: %s'
raise ValueError(
msg %
(neighborhood_function, ', '.join(neig_functions.keys())))
self.neighborhood = neig_functions[neighborhood_function]
epochs = 1000
batch_size = 5
n_a0 = \
dict(n_input=f_height*f_width, # 784 image
encoder_net=[200,200],
n_z=2, # dimensionality of latent space
decoder_net=[200,200])
list_of_archs = [n_a0]
list_of_archs_i = [0]
rs = npr.RandomState(0)
#Ramdomly select a batch
batch = []
while len(batch) != batch_size:
datapoint = test_x[rs.randint(0,len(test_x))]
batch.append(datapoint)
for k in list_of_k_samples:
for m in list_of_models:
for arch in list_of_archs_i:
# saved_parameter_file = m + '_struc' + str(arch) + '_k' + str(k) + '_1000.ckpt'
def __init__(self,
x,
y,
input_len,
sigma=1.0,
learning_rate=0.5,
decay_function=asymptotic_decay,
neighborhood_function='gaussian',
random_seed=None):
"""Initializes a Self Organizing Maps.
A rule of thumb to set the size of the grid for a dimensionality
reduction task is that it should contain 5*sqrt(N) neurons
where N is the number of samples in the dataset to analyze.
E.g. if your dataset has 150 samples, 5*sqrt(150) = 61.23
hence a map 8-by-8 should perform well.
Parameters
----------
x : int
x dimension of the SOM.
y : int
y dimension of the SOM.
input_len : int
Number of the elements of the vectors in input.
sigma : float, optional (default=1.0)
Spread of the neighborhood function, needs to be adequate
to the dimensions of the map.
(at the iteration t we have sigma(t) = sigma / (1 + t/T)
where T is #num_iteration/2)
learning_rate, initial learning rate
(at the iteration t we have
learning_rate(t) = learning_rate / (1 + t/T)
where T is #num_iteration/2)
decay_function : function (default=None)
Function that reduces learning_rate and sigma at each iteration
the default function is:
learning_rate / (1+t/(max_iterarations/2))
A custom decay function will need to to take in input
three parameters in the following order:
1. learning rate
2. current iteration
3. maximum number of iterations allowed
Note that if a lambda function is used to define the decay
MiniSom will not be pickable anymore.
neighborhood_function : function, optional (default='gaussian')
Function that weights the neighborhood of a position in the map
possible values: 'gaussian', 'mexican_hat', 'bubble'
random_seed : int, optional (default=None)
Random seed to use.
"""
if sigma >= x or sigma >= y:
warn('Warning: sigma is too high for the dimension of the map.')
self._random_generator = random.RandomState(random_seed)
self._learning_rate = learning_rate
self._sigma = sigma
self._input_len = input_len
# random initialization
self._weights = self._random_generator.rand(x, y, input_len) * 2 - 1
for i in range(x):
for j in range(y):
# normalization
norm = fast_norm(self._weights[i, j])
self._weights[i, j] = self._weights[i, j] / norm
self._activation_map = zeros((x, y))
self._neigx = arange(x)
self._neigy = arange(y) # used to evaluate the neighborhood function
self._decay_function = decay_function
neig_functions = {
'gaussian': self._gaussian,
'mexican_hat': self._mexican_hat,
'bubble': self._bubble,
'triangle': self._triangle
}
if neighborhood_function not in neig_functions:
msg = '%s not supported. Functions available: %s'
raise ValueError(
msg %
(neighborhood_function, ', '.join(neig_functions.keys())))
if neighborhood_function in ['triangle', 'bubble'
] and divmod(sigma, 1)[1] != 0:
warn('sigma should be an integer when triangle or bubble' +
'are used as neighborhood function')
self.neighborhood = neig_functions[neighborhood_function]
import numpy as np
import numpy.random as nr
from dateutil.relativedelta import relativedelta
from scipy.stats import truncnorm
seed = nr.RandomState(0)
def get_truncated(lower, upper, mu, sigma):
return truncnorm((lower - mu) / sigma, (upper - mu) / sigma,
loc=mu,
scale=sigma)
def gen_birth(contract):
# Borrower has to be at least 25 at contract time, and not over 80 at contract end
# Hence, limiting age between 26 and 74 years old (9515, 27085 days)
return contract - relativedelta(days=nr.randint(9515, 27085))
def gen_amortization(birth, contract):
# Amortization is conditional on age and contract date
amortization = nr.randint(60, 360)
# Checking to see if oldest borrower at the end of contract is above age limit of 80 years old
count = 0
while (birth + relativedelta(years=79) <
contract + relativedelta(months=amortization)) is True:
amortization = nr.randint(60, 360)
count += 1
if count > 360:
amortization = 60
def rand_not_small(offset=10.):
post = lambda x: x + onp.where(x > 0, offset, -offset)
randn = npr.RandomState(0).randn
return partial(_rand_dtype, randn, scale=3., post=post)
def testScanRnn(self):
r = npr.RandomState(0)
n_in = 4
n_hid = 2
n_out = 1
length = 3
W_trans = r.randn(n_hid, n_hid + n_in)
W_out = r.randn(n_out, n_hid + n_in)
params = W_trans, W_out
inputs = r.randn(length, n_in)
targets = r.randn(length, n_out)
def step(params, state, input):
W_trans, W_out = params
stacked = np.concatenate([state, input])
output = np.tanh(np.dot(W_out, stacked))
next_state = np.tanh(np.dot(W_trans, stacked))
return next_state, output
def rnn(params, inputs):
init_state = np.zeros(n_hid)
_, outputs = lax.scan(partial(step, params), init_state, inputs)
return outputs
def loss(params, inputs, targets):
predictions = rnn(params, inputs)
return np.sum((predictions - targets)**2)
# evaluation doesn't crash
loss(params, inputs, targets)
# jvp evaluation doesn't crash
api.jvp(lambda params: loss(params, inputs, targets), (params, ),
(params, ))
# jvp numerical check passes
jtu.check_grads(loss, (params, inputs, targets),
order=2,
modes=["fwd"])
# linearize works
_, expected = api.jvp(loss, (params, inputs, targets),
(params, inputs, targets))
_, linfun = api.linearize(loss, params, inputs, targets)
ans = linfun(params, inputs, targets)
self.assertAllClose(ans, expected, check_dtypes=False)
# gradient evaluation doesn't crash
api.grad(loss)(params, inputs, targets)
# gradient check passes
jtu.check_grads(loss, (params, inputs, targets), order=2)
# we can vmap to batch things
batch_size = 7
batched_inputs = r.randn(batch_size, length, n_in)
batched_targets = r.randn(batch_size, length, n_out)
batched_loss = api.vmap(lambda x, y: loss(params, x, y))
losses = batched_loss(batched_inputs, batched_targets)
expected = onp.stack(
list(
map(lambda x, y: loss(params, x, y), batched_inputs,
batched_targets)))
self.assertAllClose(losses, expected, check_dtypes=False)
def rand_uniform(low=0.0, high=1.0):
assert low < high
rand = npr.RandomState(0).rand
post = lambda x: x * (high - low) + low
return partial(_rand_dtype, rand, post=post)
break
# 判断是否画线覆盖所有0，若覆盖，进行加减操作
if 0 not in c:
row_sub = list(set(row_and_col) - set(delete_count_of_row))
min_value = np.min(c)
for i in row_sub:
b[i] = b[i] - min_value
for i in delete_count_of_rol:
b[:, i] = b[:, i] + min_value
break
row_ind, col_ind = linear_sum_assignment(b)
min_cost = task_matrix[row_ind, col_ind].sum()
best_solution = list(task_matrix[row_ind, col_ind])
return min_cost, best_solution
# 生成开销矩阵
rd = random.RandomState(10000)
task_matrix = rd.randint(1, 10, size=(5, 5))
# 用全排列方法实现任务分配
ass_by_per = TaskAssignment(task_matrix, 'all_permutation')
# 用匈牙利方法实现任务分配
ass_by_Hun = TaskAssignment(task_matrix, 'Hungary')
print('cost matrix = ', '\n', task_matrix)
print('全排列方法任务分配：')
print('min cost = ', ass_by_per.min_cost)
print('best solution = ', ass_by_per.best_solution)
print('匈牙利方法任务分配：')
print('min cost = ', ass_by_Hun.min_cost)
print('best solution = ', ass_by_Hun.best_solution)
def rand_default(): randn = npr.RandomState(0).randn return partial(_rand_dtype, randn, scale=3)
def cases_from_list(xs):
rng = npr.RandomState(42)
xs = list(xs)
k = min(len(xs), FLAGS.num_generated_cases)
indices = rng.choice(onp.arange(len(xs)), k, replace=False)
return [xs[i] for i in indices]
def rand_positive(): post = lambda x: x + 1 rand = npr.RandomState(0).rand return partial(_rand_dtype, rand, scale=2, post=post)
def setup_method(self, method):
self.rs = random.RandomState(1234567890)