class EventletApiRequest(request.ApiRequest):
'''Eventlet-based ApiRequest class.
This class will form the basis for eventlet-based ApiRequest classes
'''
# Maximum number of green threads present in the system at one time.
API_REQUEST_POOL_SIZE = request.DEFAULT_API_REQUEST_POOL_SIZE
# Pool of green threads. One green thread is allocated per incoming
# request. Incoming requests will block when the pool is empty.
API_REQUEST_POOL = eventlet.GreenPool(API_REQUEST_POOL_SIZE)
# A unique id is assigned to each incoming request. When the current
# request id reaches MAXIMUM_REQUEST_ID it wraps around back to 0.
MAXIMUM_REQUEST_ID = request.DEFAULT_MAXIMUM_REQUEST_ID
# The request id for the next incoming request.
CURRENT_REQUEST_ID = 0
def __init__(self, client_obj, url, method="GET", body=None,
headers=None,
request_timeout=request.DEFAULT_REQUEST_TIMEOUT,
retries=request.DEFAULT_RETRIES,
auto_login=True,
redirects=request.DEFAULT_REDIRECTS,
http_timeout=request.DEFAULT_HTTP_TIMEOUT, client_conn=None):
'''Constructor.'''
self._api_client = client_obj
self._url = url
self._method = method
self._body = body
self._headers = headers or {}
self._request_timeout = request_timeout
self._retries = retries
self._auto_login = auto_login
self._redirects = redirects
self._http_timeout = http_timeout
self._client_conn = client_conn
self._abort = False
self._request_error = None
if "User-Agent" not in self._headers:
self._headers["User-Agent"] = USER_AGENT
self._green_thread = None
# Retrieve and store this instance's unique request id.
self._request_id = EventletApiRequest.CURRENT_REQUEST_ID
# Update the class variable that tracks request id.
# Request IDs wrap around at MAXIMUM_REQUEST_ID
next_request_id = self._request_id + 1
next_request_id %= self.MAXIMUM_REQUEST_ID
EventletApiRequest.CURRENT_REQUEST_ID = next_request_id
@classmethod
def _spawn(cls, func, *args, **kwargs):
'''Allocate a green thread from the class pool.'''
return cls.API_REQUEST_POOL.spawn(func, *args, **kwargs)
def spawn(self, func, *args, **kwargs):
'''Spawn a new green thread with the supplied function and args.'''
return self.__class__._spawn(func, *args, **kwargs)
@classmethod
def joinall(cls):
'''Wait for all outstanding requests to complete.'''
return cls.API_REQUEST_POOL.waitall()
def join(self):
'''Wait for instance green thread to complete.'''
if self._green_thread is not None:
return self._green_thread.wait()
return Exception(_('Joining an invalid green thread'))
def start(self):
'''Start request processing.'''
self._green_thread = self.spawn(self._run)
def copy(self):
'''Return a copy of this request instance.'''
return EventletApiRequest(
self._api_client, self._url, self._method, self._body,
self._headers, self._request_timeout, self._retries,
self._auto_login, self._redirects, self._http_timeout)
def _run(self):
'''Method executed within green thread.'''
if self._request_timeout:
# No timeout exception escapes the with block.
with eventlet.timeout.Timeout(self._request_timeout, False):
return self._handle_request()
LOG.info(_('[%d] Request timeout.'), self._rid())
self._request_error = Exception(_('Request timeout'))
return None
else:
return self._handle_request()
def _handle_request(self):
'''First level request handling.'''
attempt = 0
timeout = 0
response = None
while response is None and attempt <= self._retries:
eventlet.greenthread.sleep(timeout)
attempt += 1
req = self._issue_request()
# automatically raises any exceptions returned.
if isinstance(req, httplib.HTTPResponse):
timeout = 0
if attempt <= self._retries and not self._abort:
if req.status in (httplib.UNAUTHORIZED, httplib.FORBIDDEN):
continue
elif req.status == httplib.SERVICE_UNAVAILABLE:
timeout = 0.5
continue
# else fall through to return the error code
LOG.debug(_("[%(rid)d] Completed request '%(method)s %(url)s'"
": %(status)s"),
{'rid': self._rid(), 'method': self._method,
'url': self._url, 'status': req.status})
self._request_error = None
response = req
else:
LOG.info(_('[%(rid)d] Error while handling request: %(req)s'),
{'rid': self._rid(), 'req': req})
self._request_error = req
response = None
return response
class Client(object):
"""
Client for the Cisco Nexus1000V Neutron Plugin.
This client implements functions to communicate with
Cisco Nexus1000V VSM.
For every Neutron object, Cisco Nexus1000V Neutron Plugin
creates a corresponding object on the controller (Cisco
Nexus1000V VSM).
CONCEPTS:
Following are few concepts used in Nexus1000V VSM:
port-profiles:
Policy profiles correspond to port profiles on Nexus1000V VSM.
Port profiles are the primary mechanism by which network policy is
defined and applied to switch interfaces in a Nexus 1000V system.
network-segment:
Each network-segment represents a broadcast domain.
network-segment-pool:
A network-segment-pool contains one or more network-segments.
logical-network:
A logical-network contains one or more network-segment-pools.
bridge-domain:
A bridge-domain is created when the network-segment is of type VXLAN.
Each VXLAN <--> VLAN combination can be thought of as a bridge domain.
ip-pool:
Each ip-pool represents a subnet on the Nexus1000V VSM.
WORK FLOW:
For every network profile a corresponding logical-network and
a network-segment-pool, under this logical-network, will be created.
For every network created from a given network profile, a
network-segment will be added to the network-segment-pool corresponding
to that network profile.
A port is created on a network and associated with a policy-profile.
Hence for every unique combination of a network and a policy-profile, a
unique vm-network will be created and a reference to the port will be
added. If the same combination of network and policy-profile is used by
another port, the references to that port will be added to the same
vm-network.
"""
# Define paths for the URI where the client connects for HTTP requests.
port_profiles_path = "/virtual-port-profile"
ports_path = "/kvm/vm-network/%s/ports"
port_path = "/kvm/vm-network/%s/ports/%s"
network_segment_path = "/network-segment/%s"
network_segments_path = "/network-segment"
network_segment_pool_path = "/network-segment-pool/%s"
network_segment_pools_path = "/network-segment-pool"
ip_pool_path = "/ip-pool-template/%s"
ip_pools_path = "/ip-pool-template"
vm_networks_path = "/kvm/vm-network"
vm_network_path = "/kvm/vm-network/%s"
bridge_domains_path = "/kvm/bridge-domain"
bridge_domain_path = "/kvm/bridge-domain/%s"
logical_network_path = "/logical-network/%s"
md5_path = "/kvm/config-md5-hashes"
sync_notification_path = "/sync-notification"
pool = eventlet.GreenPool(cfg.CONF.ml2_cisco_n1kv.http_pool_size)
def __init__(self, **kwargs):
"""Initialize a new client for the plugin."""
self.format = 'json'
# Extract configuration parameters from the configuration file.
self.username = cfg.CONF.ml2_cisco_n1kv.username
self.password = cfg.CONF.ml2_cisco_n1kv.password
self.vsm_ips = config.get_vsm_hosts()
self.action_prefix = 'http://%s/api/n1k'
self.timeout = cfg.CONF.ml2_cisco_n1kv.http_timeout
self.max_vsm_retries = cfg.CONF.ml2_cisco_n1kv.max_vsm_retries
required_opts = ('vsm_ips', 'username', 'password')
# Validate whether required options are configured
for opt in required_opts:
if not getattr(self, opt):
raise cfg.RequiredOptError(opt, 'ml2_cisco_n1kv')
# Validate the configured VSM IP addresses
# Note: Currently only support IPv4
for vsm_ip in self.vsm_ips:
if not (netutils.is_valid_ipv4(vsm_ip)
or netutils.is_valid_ipv6(vsm_ip)):
raise cfg.Error(
_("Cisco Nexus1000V ML2 driver config: "
"Invalid format for VSM IP address: %s") % vsm_ip)
def send_sync_notification(self, msg, vsm_ip):
"""Send a start/end/no-change sync notification to the VSM.
:param vsm_ip: string representing the IP address of the VSM
:param msg: message string, start, end or no-change
"""
body = {'status': msg}
self._post(self.sync_notification_path, body=body, vsm_ip=vsm_ip)
def list_port_profiles(self, vsm_ip=None):
"""Fetch all policy profiles from the VSM.
:param vsm_ip: string representing the IP address of the VSM
:returns: JSON string
"""
return self._get(self.port_profiles_path, vsm_ip=vsm_ip)
def list_network_profiles(self, vsm_ip=None):
"""Fetch all network profiles from VSM.
:param vsm_ip: string representing the IP address of the VSM
:return: JSON string
"""
return self._get(self.network_segment_pools_path, vsm_ip=vsm_ip)
def list_networks(self, vsm_ip=None):
"""Fetch all networks from VSM.
:param vsm_ip: string representing the IP address of the VSM
:return: JSON string
"""
return self._get(self.network_segments_path, vsm_ip=vsm_ip)
def list_subnets(self, vsm_ip=None):
"""Fetch all subnets from VSM.
:param vsm_ip: string representing the IP address of the VSM
:return: JSON string
"""
return self._get(self.ip_pools_path, vsm_ip=vsm_ip)
def list_vmnetworks(self, vsm_ip=None):
"""Fetch all VM networks from VSM.
:param vsm_ip: string representing the IP address of the VSM
:return: JSON string
"""
return self._get(self.vm_networks_path, vsm_ip=vsm_ip)
def list_md5_hashes(self, vsm_ip=None):
"""Fetch MD5 hashes for all resources from VSM.
Fetch MD5 hashes for network profiles, networks, subnets, ports and
a consolidated hash of these hashes from the VSM
:param vsm_ip: string representing the IP address of the VSM
:return: JSON string
"""
return self._get(self.md5_path, vsm_ip=vsm_ip)
def list_bridge_domains(self, vsm_ip=None):
"""Fetch the list of all bridge domains on the VSM.
:param vsm_ip: string representing the IP address of the VSM
:return: JSON string
"""
return self._get(self.bridge_domains_path, vsm_ip=vsm_ip)
def show_network(self, network_id, vsm_ip=None):
"""Fetch details of a given network like segment type from the VSM.
:param network_id: UUID of the network whose details are needed
:param vsm_ip: string representing the IP address of the VSM
:return: JSON string
"""
return self._get(self.network_segment_path % network_id, vsm_ip=vsm_ip)
def _create_logical_network(self, network_profile, vsm_ip=None):
"""Create a logical network on the VSM.
:param network_profile: network profile dict
:param vsm_ip: string representing the IP address of the VSM
"""
body = {'description': network_profile['name']}
logical_network_name = (network_profile['id'] +
n1kv_const.LOGICAL_NETWORK_SUFFIX)
return self._post(self.logical_network_path % logical_network_name,
body=body,
vsm_ip=vsm_ip)
def delete_logical_network(self, logical_network_name, vsm_ip=None):
"""Delete a logical network on VSM.
:param logical_network_name: string representing name of the logical
network
:param vsm_ip: string representing the IP address of the VSM
"""
return self._delete(self.logical_network_path % logical_network_name,
vsm_ip=vsm_ip)
def create_network_segment_pool(self, network_profile, vsm_ip=None):
"""Create a network segment pool on the VSM.
:param network_profile: network profile dict
:param vsm_ip: string representing the IP address of the VSM
"""
self._create_logical_network(network_profile, vsm_ip=vsm_ip)
logical_network_name = (network_profile['id'] +
n1kv_const.LOGICAL_NETWORK_SUFFIX)
body = {
'name': network_profile['name'],
'description': network_profile['name'],
'id': network_profile['id'],
'logicalNetwork': logical_network_name
}
return self._post(self.network_segment_pool_path %
network_profile['id'],
body=body,
vsm_ip=vsm_ip)
def delete_network_segment_pool(self,
network_segment_pool_id,
vsm_ip=None):
"""Delete a network segment pool on the VSM.
:param network_segment_pool_id: UUID representing the network
segment pool
:param vsm_ip: string representing the IP address of the VSM
"""
return self._delete(self.network_segment_pool_path %
network_segment_pool_id,
vsm_ip=vsm_ip)
def create_network_segment(self, network, network_profile, vsm_ip=None):
"""Create a network segment on the VSM.
:param network: network dict
:param network_profile: network profile object
:param vsm_ip: string representing the IP address of the VSM
"""
body = {
'publishName': network['id'],
'description': network['name'],
'id': network['id'],
'tenantId': network['tenant_id'],
'mode': 'access',
'segmentType': network_profile['segment_type'],
'networkSegmentPool': network_profile['id']
}
# Override tenantId if network is shared
if network['shared']:
body['tenantId'] = '0'
if network[providernet.NETWORK_TYPE] == p_const.TYPE_VLAN:
body['vlan'] = network[providernet.SEGMENTATION_ID]
elif network[providernet.NETWORK_TYPE] == p_const.TYPE_VXLAN:
# Create a bridge domain on VSM
bd_name = network['id'] + n1kv_const.BRIDGE_DOMAIN_SUFFIX
self.create_bridge_domain(network, network_profile, vsm_ip=vsm_ip)
body['bridgeDomain'] = bd_name
try:
return self._post(self.network_segment_path % network['id'],
body=body,
vsm_ip=vsm_ip)
except (n1kv_exc.VSMError, n1kv_exc.VSMConnectionFailed):
with excutils.save_and_reraise_exception():
# Clean up the bridge domain from the VSM for VXLAN networks.
# Reraise the exception so that caller method executes further
# clean up.
if network[providernet.NETWORK_TYPE] == p_const.TYPE_VXLAN:
self.delete_bridge_domain(bd_name, vsm_ip=vsm_ip)
def update_network_segment(self, updated_network):
"""Update a network segment on the VSM.
:param updated_network: updated network dict
"""
body = {
'description': updated_network['name'],
'tenantId': updated_network['tenant_id']
}
if updated_network['shared']:
body['tenantId'] = '0'
return self._post(self.network_segment_path % updated_network['id'],
body=body)
def delete_network_segment(self,
network_segment_id,
network_type,
vsm_ip=None):
"""Delete a network segment on the VSM.
:param network_segment_id: UUID representing the network segment
:param network_type: type of network to be deleted
:param vsm_ip: string representing the IP address of the VSM
"""
if network_type == p_const.TYPE_VXLAN:
bd_name = network_segment_id + n1kv_const.BRIDGE_DOMAIN_SUFFIX
self.delete_bridge_domain(bd_name, vsm_ip=vsm_ip)
return self._delete(self.network_segment_path % network_segment_id,
vsm_ip=vsm_ip)
def create_bridge_domain(self, network, net_prof, vsm_ip=None):
"""Create a bridge domain on VSM.
:param network: network dict
:param net_prof: network profile dict
:param vsm_ip: string representing the IP address of the VSM
"""
if net_prof['sub_type'] == n1kv_const.CLI_VXLAN_MODE_ENHANCED:
vxlan_subtype = n1kv_const.MODE_UNICAST
else:
vxlan_subtype = n1kv_const.MODE_NATIVE_VXLAN
body = {
'name': network['id'] + n1kv_const.BRIDGE_DOMAIN_SUFFIX,
'segmentId': network[providernet.SEGMENTATION_ID],
'subType': vxlan_subtype,
'tenantId': network['tenant_id']
}
if vxlan_subtype == n1kv_const.MODE_NATIVE_VXLAN:
start_ip, end_ip = net_prof['multicast_ip_range'].split('-', 1)
body['groupIp'] = start_ip
return self._post(self.bridge_domains_path, body=body, vsm_ip=vsm_ip)
def delete_bridge_domain(self, name, vsm_ip=None):
"""Delete a bridge domain on VSM.
:param name: name of the bridge domain to be deleted
:param vsm_ip: string representing the IP address of the VSM
"""
return self._delete(self.bridge_domain_path % name, vsm_ip=vsm_ip)
def create_ip_pool(self, subnet, vsm_ip=None):
"""Create a subnet on VSM.
:param subnet: subnet dict
:param vsm_ip: string representing the IP address of the VSM
"""
if subnet['cidr']:
try:
ip = netaddr.IPNetwork(subnet['cidr'])
netmask = str(ip.netmask)
network_address = str(ip.network)
except (ValueError, netaddr.AddrFormatError):
msg = _("Invalid input for CIDR")
raise n_exc.InvalidInput(error_message=msg)
else:
netmask = network_address = ""
if subnet['allocation_pools']:
address_range_start = subnet['allocation_pools'][0]['start']
address_range_end = subnet['allocation_pools'][0]['end']
else:
address_range_start = None
address_range_end = None
body = {
'addressRangeStart': address_range_start,
'addressRangeEnd': address_range_end,
'ipAddressSubnet': netmask,
'description': subnet['name'],
'gateway': subnet['gateway_ip'],
'dhcp': subnet['enable_dhcp'],
'dnsServersList': subnet['dns_nameservers'],
'networkAddress': network_address,
'netSegmentName': subnet['network_id'],
'id': subnet['id'],
'tenantId': subnet['tenant_id']
}
return self._post(self.ip_pool_path % subnet['id'],
body=body,
vsm_ip=vsm_ip)
def update_ip_pool(self, subnet):
"""Update an ip-pool on the VSM.
:param subnet: subnet dictionary
"""
body = {
'description': subnet['name'],
'dhcp': subnet['enable_dhcp'],
'dnsServersList': subnet['dns_nameservers']
}
return self._post(self.ip_pool_path % subnet['id'], body=body)
def delete_ip_pool(self, subnet_id, vsm_ip=None):
"""Delete an ip-pool on the VSM.
:param subnet_id: UUID representing the subnet
:param vsm_ip: string representing the IP address of the VSM
"""
return self._delete(self.ip_pool_path % subnet_id, vsm_ip=vsm_ip)
def create_n1kv_port(self,
port,
vmnetwork_name,
policy_profile,
vsm_ip=None):
"""Create a port on the VSM.
:param port: port dict
:param vmnetwork_name: name of the VM network
:param policy_profile: policy profile object
:param vsm_ip: string representing the IP address of the VSM
"""
body = {
'name': vmnetwork_name,
'networkSegmentId': port['network_id'],
'networkSegment': port['network_id'],
'portProfile': policy_profile.name,
'portProfileId': policy_profile.id,
'tenantId': port['tenant_id'],
'portId': port['id'],
'macAddress': port['mac_address'],
'portType': port['device_owner'],
}
if port.get('fixed_ips'):
body['ipAddress'] = port['fixed_ips'][0]['ip_address']
body['subnetId'] = port['fixed_ips'][0]['subnet_id']
return self._post(self.vm_networks_path, body=body, vsm_ip=vsm_ip)
def delete_n1kv_port(self, vmnetwork_name, port_id, vsm_ip=None):
"""Delete a port on the VSM.
:param vmnetwork_name: name of the VM network which imports this port
:param port_id: UUID of the port
:param vsm_ip: string representing the IP address of the VSM
"""
return self._delete(self.port_path % (vmnetwork_name, port_id),
vsm_ip=vsm_ip)
def _do_request(self,
method,
action,
body=None,
headers=None,
vsm_ip=None):
"""Perform the HTTP request.
The response is in either JSON format or plain text. A GET method will
invoke a JSON response while a PUT/POST/DELETE returns message from the
VSM in plain text format.
Exception is raised when VSM replies with an INTERNAL SERVER ERROR HTTP
status code (500) i.e. an error has occurred on the VSM or SERVICE
UNAVAILABLE (404) i.e. VSM is not reachable.
:param method: type of the HTTP request. POST, GET, PUT or DELETE
:param action: path to which the client makes request
:param body: dict for arguments which are sent as part of the request
:param headers: header for the HTTP request
:param vsm_ip: vsm_ip for the HTTP request. If not provided then
request will be sent to all VSMs.
:returns: JSON or plain text in HTTP response
"""
action = self.action_prefix + action
if body:
body = jsonutils.dumps(body)
LOG.debug("req: %s", body)
hosts = []
if vsm_ip:
hosts.append(vsm_ip)
else:
hosts = self.vsm_ips
if not headers:
headers = self._get_auth_header()
headers['Content-Type'] = headers['Accept'] = "application/json"
for vsm_ip in hosts:
if netutils.is_valid_ipv6(vsm_ip):
# Enclose IPv6 address in [] in the URL
vsm_action = action % ("[%s]" % vsm_ip)
else:
# IPv4 address
vsm_action = action % vsm_ip
for attempt in range(self.max_vsm_retries + 1):
try:
LOG.debug("[VSM %(vsm)s attempt %(id)s]: Connecting.." % {
"vsm": vsm_ip,
"id": attempt
})
resp = self.pool.spawn(requests.request,
method,
url=vsm_action,
data=body,
headers=headers,
timeout=self.timeout).wait()
break
except Exception as e:
LOG.debug("[VSM %(vsm)s attempt %(id)s]: Conn timeout." % {
"vsm": vsm_ip,
"id": attempt
})
if attempt == self.max_vsm_retries:
LOG.error(_LE("VSM %s, Conn failed."), vsm_ip)
raise n1kv_exc.VSMConnectionFailed(reason=e)
if resp.status_code != requests.codes.OK:
LOG.error(_LE("VSM %(vsm)s, Got error: %(err)s"), {
"vsm": vsm_ip,
"err": resp.text
})
raise n1kv_exc.VSMError(reason=resp.text)
if 'application/json' in resp.headers['content-type']:
try:
return resp.json()
except ValueError:
return {}
elif 'text/plain' in resp.headers['content-type']:
LOG.info(_LI("VSM: %s"), resp.text)
def _delete(self, action, body=None, headers=None, vsm_ip=None):
return self._do_request("DELETE",
action,
body=body,
headers=headers,
vsm_ip=vsm_ip)
def _get(self, action, vsm_ip, body=None, headers=None):
return self._do_request("GET",
action,
body=body,
headers=headers,
vsm_ip=vsm_ip)
def _post(self, action, body=None, headers=None, vsm_ip=None):
return self._do_request("POST",
action,
body=body,
headers=headers,
vsm_ip=vsm_ip)
def _put(self, action, body=None, headers=None, vsm_ip=None):
return self._do_request("PUT",
action,
body=body,
headers=headers,
vsm_ip=vsm_ip)
def _get_auth_header(self):
"""Retrieve header with auth info for the VSM.
:return: authorization header dict
"""
auth = base64.encodestring(
six.b("%s:%s" % (self.username, self.password))).rstrip()
return {"Authorization": "Basic %s" % auth}
print("fetching %s" % url)
try:
r = requests.head(url, allow_redirects=True, timeout=timeout)
except requests.exceptions.RequestException:
res = timeout + 1
else:
if r.status_code == 200:
res = time.time() - now
client.zadd(key, timeout - res, url)
if __name__ == "__main__":
import eventlet
eventlet.monkey_patch()
import redis
import requests
key = "pypimirrors"
client = redis.StrictRedis()
client.delete(key)
pool = eventlet.GreenPool(20)
for i in range(ord("b"), ord("g") + 1):
c = chr(i)
if c in "bde":
continue
url = "http://%s.pypi.python.org/packages/" % chr(i)
pool.spawn_n(fetch, url)
pool.waitall()
for x in client.zrange(key, 0, 10, withscores=True):
print(x)
def __init__(self):
#
self.ts = ts()
self.pool = et.GreenPool(20)
def _process_loop(self):
LOG.debug("Starting _process_loop")
pool = eventlet.GreenPool(size=8)
while True:
pool.spawn_n(self._process_resource_update)
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import datetime
from heat.openstack.common import log as logging
from heat.openstack.common import timeutils
from heat.engine import timestamp
from heat.db import api as db_api
from heat.engine import parser
from heat.common import context as ctxtlib
import eventlet
logger = logging.getLogger('heat.engine.watchrule')
greenpool = eventlet.GreenPool()
class WatchRule(object):
WATCH_STATES = (ALARM, NORMAL, NODATA) = ('ALARM', 'NORMAL', 'NODATA')
ACTION_MAP = {
ALARM: 'AlarmActions',
NORMAL: 'OKActions',
NODATA: 'InsufficientDataActions'
}
created_at = timestamp.Timestamp(db_api.watch_rule_get, 'created_at')
updated_at = timestamp.Timestamp(db_api.watch_rule_get, 'updated_at')
def __init__(self,
def __init__(self, is_not_light=False):
super(DFDaemon, self).__init__()
self.pool = eventlet.GreenPool()
self.is_daemonize = False
self.thread = None
self.is_not_light = is_not_light
def process_service(self, device_ids=None, removed_devices_info=None):
try:
LOG.debug("Routing service processing started")
resources = {}
routers = []
removed_routers = []
all_routers_flag = False
if self.fullsync:
LOG.debug("FullSync flag is on. Starting fullsync")
# Setting all_routers_flag and clear the global full_sync flag
all_routers_flag = True
self.fullsync = False
self.updated_routers.clear()
self.removed_routers.clear()
self.sync_devices.clear()
routers = self._fetch_router_info(all_routers=True)
else:
if self.updated_routers:
router_ids = list(self.updated_routers)
LOG.debug("Updated routers:%s", router_ids)
self.updated_routers.clear()
routers = self._fetch_router_info(router_ids=router_ids)
if device_ids:
LOG.debug("Adding new devices:%s", device_ids)
self.sync_devices = set(device_ids) | self.sync_devices
if self.sync_devices:
sync_devices_list = list(self.sync_devices)
LOG.debug("Fetching routers on:%s", sync_devices_list)
routers.extend(self._fetch_router_info(
device_ids=sync_devices_list))
self.sync_devices.clear()
if removed_devices_info:
if removed_devices_info.get('deconfigure'):
ids = self._get_router_ids_from_removed_devices_info(
removed_devices_info)
self.removed_routers = self.removed_routers | set(ids)
if self.removed_routers:
removed_routers_ids = list(self.removed_routers)
LOG.debug("Removed routers:%s", removed_routers_ids)
for r in removed_routers_ids:
if r in self.router_info:
removed_routers.append(self.router_info[r].router)
# Sort on hosting device
if routers:
resources['routers'] = routers
if removed_routers:
resources['removed_routers'] = removed_routers
hosting_devices = self._sort_resources_per_hosting_device(
resources)
# Dispatch process_services() for each hosting device
pool = eventlet.GreenPool()
for device_id, resources in hosting_devices.items():
routers = resources.get('routers')
removed_routers = resources.get('removed_routers')
pool.spawn_n(self._process_routers, routers, removed_routers,
device_id, all_routers=all_routers_flag)
pool.waitall()
if removed_devices_info:
for hd_id in removed_devices_info['hosting_data']:
self._drivermgr.remove_driver_for_hosting_device(hd_id)
LOG.debug("Routing service processing successfully completed")
except Exception:
LOG.exception(_LE("Failed processing routers"))
self.fullsync = True
from abc import ABCMeta, abstractmethod
import uuid
import eventlet
from .message import (Down, Monitor, Unmonitor, Cancel, Kill, Fork,
ForkWithMonitor, ForkResponse)
from .mailbox import Mailbox, AckableMailbox, LocalMailbox, Receiver
_actor_map = {}
_actor_pool = eventlet.GreenPool(size=1000000)
class ActorBase(Receiver, metaclass=ABCMeta):
@abstractmethod
def encode(self):
pass
@staticmethod
@abstractmethod
def decode(params):
pass
class Actor(ActorBase):
__slots__ = [
'_ack', '_inbox', '_outbox', '_callback', '_greenlet', '_observers'
]
def __init__(self, callback, mailbox=None):
def test_update_while_iterating_agents(self):
pool = eventlet.GreenPool(2)
pool.spawn(self._list_agents)
pool.spawn(self._add_and_delete_agents)
pool.waitall()
self.assertEqual(self.names_ref, self.names_read)
def _start_process_queue(self):
LOG.debug("Starting ip_conntrack _process_queue_worker() threads")
pool = eventlet.GreenPool(size=WORKERS)
for i in range(WORKERS):
pool.spawn_n(self._process_queue_worker)
def multiobject_download(
urlList,
downloadDirectory,
log,
timeStamp=True,
timeout=180,
concurrentDownloads=10,
resetFilename=False,
credentials=False,
longTime=False,
indexFilenames=False
):
"""
*get multiple url documents and place them in specified download directory/directories*
**Key Arguments:**
- ``urlList`` -- list of document urls
- ``downloadDirectory`` -- directory(ies) to download the documents to - can be one directory path or a list of paths the same length as urlList
- ``log`` -- the logger
- ``timestamp`` -- append a timestamp the name of the URL (ensure unique filenames)
- ``longTime`` -- use a longer timestamp when appending to the filename (greater uniqueness)
- ``timeout`` -- the timeout limit for downloads (secs)
- ``concurrentDownloads`` -- the number of concurrent downloads allowed at any one time
- ``resetFilename`` -- a string to reset all filenames to
- ``credentials`` -- basic http credentials { 'username' : "...", "password", "..." }
- ``indexFilenames`` -- prepend filenames with index (where url appears in urllist)
**Return:**
- list of timestamped documents (same order as the input urlList)
**Usage:**
.. code-block:: python
# download the pages linked from the main list page
from fundamentals.download import multiobject_download
localUrls = multiobject_download(
urlList=["https://www.python.org/dev/peps/pep-0257/","https://en.wikipedia.org/wiki/Docstring"],
downloadDirectory="/tmp",
log="log",
timeStamp=True,
timeout=180,
concurrentDownloads=2,
resetFilename=False,
credentials=False, # { 'username' : "...", "password", "..." }
longTime=True
)
print localUrls
# OUT: ['/tmp/untitled_20160316t160650610780.html', '/tmp/Docstring_20160316t160650611136.html']
.. image:: https://i.imgur.com/QYoMm24.png width=600px
"""
## > IMPORTS ##
import sys
import os
import eventlet
import socket
import re
import base64
from fundamentals.download import _fetch, _dump_files_to_local_drive, append_now_datestamp_to_filename, extract_filename_from_url
## >SETTINGS ##
# TIMEOUT IN SECONDS
timeout = float(timeout)
socket.setdefaulttimeout(timeout)
###########################################################
# >ACTION(S) #
###########################################################
# BUILD THE 2D ARRAY FOR MULTI_THREADED DOWNLOADS
thisArray = []
bodies = []
localUrls = []
theseUrls = []
requestList = []
totalCount = len(urlList)
# IF ONLY ONE DOWNLOAD DIRECORY
if isinstance(downloadDirectory, ("".__class__, u"".__class__)):
for i, url in enumerate(urlList):
# EXTRACT THE FILENAME FROM THE URL
if resetFilename and len(resetFilename):
filename = resetFilename[i]
else:
filename = extract_filename_from_url(log, url)
if indexFilenames:
filename = """%(i)03d_%(filename)s""" % locals()
if not filename:
from datetime import datetime, date, time
now = datetime.now()
filename = now.strftime("%Y%m%dt%H%M%S%f")
if(timeStamp):
# APPEND TIMESTAMP TO THE FILENAME
filename = append_now_datestamp_to_filename(
log, filename, longTime=longTime)
# GENERATE THE LOCAL FILE URL
localFilepath = downloadDirectory + "/" + filename
thisArray.extend([[url, localFilepath]])
# GENERATE THE REQUESTS
request = urllib.request.Request(url)
if credentials != False:
username = credentials["username"]
password = credentials["password"]
base64string = base64.encodestring(
'%s:%s' % (username, password)).replace('\n', '')
request.add_header("Authorization", "Basic %s" % base64string)
requestList.append(request)
elif isinstance(downloadDirectory, list):
for u, d in zip(urlList, downloadDirectory):
# EXTRACT THE FILENAME FROM THE URL
if resetFilename:
filename = resetFilename
else:
filename = extract_filename_from_url(log, url)
if not filename:
continue
if(timeStamp):
# APPEND TIMESTAMP TO THE FILENAME
filename = append_now_datestamp_to_filename(
log, filename)
# GENERATE THE LOCAL FILE URL
localFilepath = d + "/" + filename
thisArray.extend([[u, localFilepath]])
log.debug(" about to download %s" % (u,))
# GENERATE THE REQUESTS
request = urllib.request.Request(u)
if credentials != False:
log.debug('adding the credentials')
username = credentials["username"]
password = credentials["password"]
base64string = base64.encodestring(
'%s:%s' % (username, password)).replace('\n', '')
request.add_header("Authorization", "Basic %s" % base64string)
requestList.append(request)
pool = eventlet.GreenPool(concurrentDownloads)
i = 0
try:
log.debug(
"starting mutli-threaded download batch - %s concurrent downloads" %
(concurrentDownloads,))
log.debug('len(requestList): %s' % (len(requestList),))
for url, body in pool.imap(_fetch, requestList):
urlNum = i + 1
if urlNum > 1:
# CURSOR UP ONE LINE AND CLEAR LINE
sys.stdout.write("\x1b[1A\x1b[2K")
percent = (float(urlNum) / float(totalCount)) * 100.
print(
" %(urlNum)s / %(totalCount)s (%(percent)1.1f%%) URLs downloaded" % locals())
if(body):
bodies.extend([body])
theseUrls.extend([thisArray[i][1]])
else:
theseUrls.extend([None])
bodies.extend([None])
# DUMP THE FILES FROM MEMORY EVERY CONCURRENT DOWNLOAD CYCLE
if i % concurrentDownloads == 0:
_dump_files_to_local_drive(bodies, theseUrls, log)
localUrls.extend(theseUrls)
# RESET THE TMP ARRAYS
bodies = []
theseUrls = []
i += 1
except Exception as e:
log.error(
"something went wrong with the mutli-threaded download : " + str(e) + "\n")
# DUMP REMAINING FILES TO THE LOCAL DRIVE
_dump_files_to_local_drive(bodies, theseUrls, log)
localUrls.extend(theseUrls)
return localUrls
def main(argv):
global urllib2
usage = "async.py <sync | async>"
# Parse the command line args.
opts = parse(argv, {}, ".splunkrc")
# We have to see if we got either the "sync" or
# "async" command line arguments.
allowed_args = ["sync", "async"]
if len(opts.args) == 0 or opts.args[0] not in allowed_args:
error("Must supply either of: %s" % allowed_args, 2)
# Note whether or not we are async.
is_async = opts.args[0] == "async"
# If we're async, we'' import `eventlet` and `eventlet`'s version
# of `urllib2`. Otherwise, import the stdlib version of `urllib2`.
#
# The reason for the funky import syntax is that Python imports
# are scoped to functions, and we need to make it global.
# In a real application, you would only import one of these.
if is_async:
urllib2 = __import__('eventlet.green', globals(), locals(),
['urllib2'], -1).urllib2
else:
urllib2 = __import__("urllib2", globals(), locals(), [], -1)
# Create the service instance using our custom HTTP request handler.
service = client.Service(handler=request, **opts.kwargs)
service.login()
# Record the current time at the start of the
# "benchmark".
oldtime = datetime.datetime.now()
def do_search(query):
# Create a search job for the query.
# In the async case, eventlet will "relinquish" the coroutine
# worker, and let others go through. In the sync case, we will
# block the entire thread waiting for the request to complete.
job = service.jobs.create(query, exec_mode="blocking")
# We fetch the results, and cancel the job
results = job.results()
job.cancel()
return results
# We specify many queries to get show the advantages
# of paralleism.
queries = [
'search * | head 100',
'search * | head 100',
'search * | head 100',
'search * | head 100',
'search * | head 100',
'search * | head 100',
'search * | head 100',
'search * | head 100',
'search * | head 100',
'search * | head 100',
'search * | head 100',
'search * | head 100',
'search * | head 100',
'search * | head 100',
'search * | head 100',
'search * | head 100',
'search * | head 100',
'search * | head 100',
'search * | head 100',
'search * | head 100',
'search * | head 100',
'search * | head 100',
]
# Check if we are async or not, and execute all the
# specified queries.
if is_async:
import eventlet
# Create an `eventlet` pool of workers.
pool = eventlet.GreenPool(16)
# If we are async, we use our worker pool to farm
# out all the queries. We just pass, as we don't
# actually care about the result.
for results in pool.imap(do_search, queries):
pass
else:
# If we are sync, then we just execute the queries one by one,
# and we can also ignore the result.
for query in queries:
do_search(query)
# Record the current time at the end of the benchmark,
# and print the delta elapsed time.
newtime = datetime.datetime.now()
print("Elapsed Time: %s" % (newtime - oldtime))
class Client(object):
"""
Client for the Cisco Nexus1000V Neutron Plugin.
This client implements functions to communicate with
Cisco Nexus1000V VSM.
For every Neutron objects, Cisco Nexus1000V Neutron Plugin
creates a corresponding object in the controller (Cisco
Nexus1000V VSM).
CONCEPTS:
Following are few concepts used in Nexus1000V VSM:
port-profiles:
Policy profiles correspond to port profiles on Nexus1000V VSM.
Port profiles are the primary mechanism by which network policy is
defined and applied to switch interfaces in a Nexus 1000V system.
network-segment:
Each network-segment represents a broadcast domain.
network-segment-pool:
A network-segment-pool contains one or more network-segments.
logical-network:
A logical-network contains one or more network-segment-pools.
bridge-domain:
A bridge-domain is created when the network-segment is of type VXLAN.
Each VXLAN <--> VLAN combination can be thought of as a bridge domain.
ip-pool:
Each ip-pool represents a subnet on the Nexus1000V VSM.
vm-network:
vm-network refers to a network-segment and policy-profile.
It maintains a list of ports that uses the network-segment and
policy-profile this vm-network refers to.
events:
Events correspond to commands that are logged on Nexus1000V VSM.
Events are used to poll for a certain resource on Nexus1000V VSM.
Event type of port_profile: Return all updates/create/deletes
of port profiles from the VSM.
Event type of port_profile_update: Return only updates regarding
policy-profiles.
Event type of port_profile_delete: Return only deleted policy profiles.
WORK FLOW:
For every network profile a corresponding logical-network and
a network-segment-pool, under this logical-network, will be created.
For every network created from a given network profile, a
network-segment will be added to the network-segment-pool corresponding
to that network profile.
A port is created on a network and associated with a policy-profile.
Hence for every unique combination of a network and a policy-profile, a
unique vm-network will be created and a reference to the port will be
added. If the same combination of network and policy-profile is used by
another port, the references to that port will be added to the same
vm-network.
"""
# Define paths for the URI where the client connects for HTTP requests.
port_profiles_path = "/virtual-port-profile"
network_segment_path = "/network-segment/%s"
network_segment_pool_path = "/network-segment-pool/%s"
ip_pool_path = "/ip-pool-template/%s"
ports_path = "/kvm/vm-network/%s/ports"
port_path = "/kvm/vm-network/%s/ports/%s"
vm_networks_path = "/kvm/vm-network"
vm_network_path = "/kvm/vm-network/%s"
bridge_domains_path = "/kvm/bridge-domain"
bridge_domain_path = "/kvm/bridge-domain/%s"
logical_network_path = "/logical-network/%s"
events_path = "/kvm/events"
clusters_path = "/cluster"
encap_profiles_path = "/encapsulation-profile"
encap_profile_path = "/encapsulation-profile/%s"
pool = eventlet.GreenPool(c_conf.CISCO_N1K.http_pool_size)
def __init__(self, **kwargs):
"""Initialize a new client for the plugin."""
self.format = 'json'
self.hosts = self._get_vsm_hosts()
self.action_prefix = 'http://%s/api/n1k' % self.hosts[0]
self.timeout = c_conf.CISCO_N1K.http_timeout
def list_port_profiles(self):
"""
Fetch all policy profiles from the VSM.
:returns: JSON string
"""
return self._get(self.port_profiles_path)
def create_bridge_domain(self, network, overlay_subtype):
"""
Create a bridge domain on VSM.
:param network: network dict
:param overlay_subtype: string representing subtype of overlay network
"""
body = {
'name': network['id'] + c_const.BRIDGE_DOMAIN_SUFFIX,
'segmentId': network[providernet.SEGMENTATION_ID],
'subType': overlay_subtype,
'tenantId': network['tenant_id']
}
if overlay_subtype == c_const.NETWORK_SUBTYPE_NATIVE_VXLAN:
body['groupIp'] = network[n1kv.MULTICAST_IP]
return self._post(self.bridge_domains_path, body=body)
def delete_bridge_domain(self, name):
"""
Delete a bridge domain on VSM.
:param name: name of the bridge domain to be deleted
"""
return self._delete(self.bridge_domain_path % name)
def create_network_segment(self, network, network_profile):
"""
Create a network segment on the VSM.
:param network: network dict
:param network_profile: network profile dict
"""
body = {
'publishName': network['id'],
'description': network['name'],
'id': network['id'],
'tenantId': network['tenant_id'],
'networkSegmentPool': network_profile['id'],
}
if network[providernet.NETWORK_TYPE] == c_const.NETWORK_TYPE_VLAN:
body['vlan'] = network[providernet.SEGMENTATION_ID]
elif network[providernet.NETWORK_TYPE] == c_const.NETWORK_TYPE_OVERLAY:
body['bridgeDomain'] = (network['id'] +
c_const.BRIDGE_DOMAIN_SUFFIX)
if network_profile['segment_type'] == c_const.NETWORK_TYPE_TRUNK:
body['mode'] = c_const.NETWORK_TYPE_TRUNK
body['segmentType'] = network_profile['sub_type']
if network_profile['sub_type'] == c_const.NETWORK_TYPE_VLAN:
body['addSegments'] = network['add_segment_list']
body['delSegments'] = network['del_segment_list']
else:
body['encapProfile'] = (network['id'] +
c_const.ENCAPSULATION_PROFILE_SUFFIX)
else:
body['mode'] = 'access'
body['segmentType'] = network_profile['segment_type']
return self._post(self.network_segment_path % network['id'], body=body)
def update_network_segment(self, network_segment_id, body):
"""
Update a network segment on the VSM.
Network segment on VSM can be updated to associate it with an ip-pool
or update its description and segment id.
:param network_segment_id: UUID representing the network segment
:param body: dict of arguments to be updated
"""
return self._post(self.network_segment_path % network_segment_id,
body=body)
def delete_network_segment(self, network_segment_id):
"""
Delete a network segment on the VSM.
:param network_segment_id: UUID representing the network segment
"""
return self._delete(self.network_segment_path % network_segment_id)
def create_logical_network(self, network_profile, tenant_id):
"""
Create a logical network on the VSM.
:param network_profile: network profile dict
:param tenant_id: UUID representing the tenant
"""
LOG.debug(_("Logical network"))
body = {'description': network_profile['name'], 'tenantId': tenant_id}
logical_network_name = (network_profile['id'] +
c_const.LOGICAL_NETWORK_SUFFIX)
return self._post(self.logical_network_path % logical_network_name,
body=body)
def delete_logical_network(self, logical_network_name):
"""
Delete a logical network on VSM.
:param logical_network_name: string representing name of the logical
network
"""
return self._delete(self.logical_network_path % logical_network_name)
def create_network_segment_pool(self, network_profile, tenant_id):
"""
Create a network segment pool on the VSM.
:param network_profile: network profile dict
:param tenant_id: UUID representing the tenant
"""
LOG.debug(_("network_segment_pool"))
logical_network_name = (network_profile['id'] +
c_const.LOGICAL_NETWORK_SUFFIX)
body = {
'name': network_profile['name'],
'description': network_profile['name'],
'id': network_profile['id'],
'logicalNetwork': logical_network_name,
'tenantId': tenant_id
}
if network_profile['segment_type'] == c_const.NETWORK_TYPE_OVERLAY:
body['subType'] = network_profile['sub_type']
return self._post(self.network_segment_pool_path %
network_profile['id'],
body=body)
def update_network_segment_pool(self, network_profile):
"""
Update a network segment pool on the VSM.
:param network_profile: network profile dict
"""
body = {
'name': network_profile['name'],
'description': network_profile['name']
}
return self._post(self.network_segment_pool_path %
network_profile['id'],
body=body)
def delete_network_segment_pool(self, network_segment_pool_id):
"""
Delete a network segment pool on the VSM.
:param network_segment_pool_id: UUID representing the network
segment pool
"""
return self._delete(self.network_segment_pool_path %
network_segment_pool_id)
def create_ip_pool(self, subnet):
"""
Create an ip-pool on the VSM.
:param subnet: subnet dict
"""
if subnet['cidr']:
try:
ip = netaddr.IPNetwork(subnet['cidr'])
netmask = str(ip.netmask)
network_address = str(ip.network)
except (ValueError, netaddr.AddrFormatError):
msg = _("Invalid input for CIDR")
raise n_exc.InvalidInput(error_message=msg)
else:
netmask = network_address = ""
if subnet['allocation_pools']:
address_range_start = subnet['allocation_pools'][0]['start']
address_range_end = subnet['allocation_pools'][0]['end']
else:
address_range_start = None
address_range_end = None
body = {
'addressRangeStart': address_range_start,
'addressRangeEnd': address_range_end,
'ipAddressSubnet': netmask,
'description': subnet['name'],
'gateway': subnet['gateway_ip'],
'dhcp': subnet['enable_dhcp'],
'dnsServersList': subnet['dns_nameservers'],
'networkAddress': network_address,
'netSegmentName': subnet['network_id'],
'id': subnet['id'],
'tenantId': subnet['tenant_id']
}
return self._post(self.ip_pool_path % subnet['id'], body=body)
def update_ip_pool(self, subnet):
"""
Update an ip-pool on the VSM.
:param subnet: subnet dictionary
"""
body = {
'description': subnet['name'],
'dhcp': subnet['enable_dhcp'],
'dnsServersList': subnet['dns_nameservers']
}
return self._post(self.ip_pool_path % subnet['id'], body=body)
def delete_ip_pool(self, subnet_id):
"""
Delete an ip-pool on the VSM.
:param subnet_id: UUID representing the subnet
"""
return self._delete(self.ip_pool_path % subnet_id)
def create_vm_network(self, port, vm_network_name, policy_profile):
"""
Create a VM network on the VSM.
:param port: port dict
:param vm_network_name: name of the VM network
:param policy_profile: policy profile dict
"""
body = {
'name': vm_network_name,
'networkSegmentId': port['network_id'],
'networkSegment': port['network_id'],
'portProfile': policy_profile['name'],
'portProfileId': policy_profile['id'],
'tenantId': port['tenant_id'],
'portId': port['id'],
'macAddress': port['mac_address'],
}
if port.get('fixed_ips'):
body['ipAddress'] = port['fixed_ips'][0]['ip_address']
body['subnetId'] = port['fixed_ips'][0]['subnet_id']
return self._post(self.vm_networks_path, body=body)
def delete_vm_network(self, vm_network_name):
"""
Delete a VM network on the VSM.
:param vm_network_name: name of the VM network
"""
return self._delete(self.vm_network_path % vm_network_name)
def create_n1kv_port(self, port, vm_network_name):
"""
Create a port on the VSM.
:param port: port dict
:param vm_network_name: name of the VM network which imports this port
"""
body = {'id': port['id'], 'macAddress': port['mac_address']}
if port.get('fixed_ips'):
body['ipAddress'] = port['fixed_ips'][0]['ip_address']
body['subnetId'] = port['fixed_ips'][0]['subnet_id']
return self._post(self.ports_path % vm_network_name, body=body)
def update_n1kv_port(self, vm_network_name, port_id, body):
"""
Update a port on the VSM.
Update the mac address associated with the port
:param vm_network_name: name of the VM network which imports this port
:param port_id: UUID of the port
:param body: dict of the arguments to be updated
"""
return self._post(self.port_path % (vm_network_name, port_id),
body=body)
def delete_n1kv_port(self, vm_network_name, port_id):
"""
Delete a port on the VSM.
:param vm_network_name: name of the VM network which imports this port
:param port_id: UUID of the port
"""
return self._delete(self.port_path % (vm_network_name, port_id))
def _do_request(self, method, action, body=None, headers=None):
"""
Perform the HTTP request.
The response is in either JSON format or plain text. A GET method will
invoke a JSON response while a PUT/POST/DELETE returns message from the
VSM in plain text format.
Exception is raised when VSM replies with an INTERNAL SERVER ERROR HTTP
status code (500) i.e. an error has occurred on the VSM or SERVICE
UNAVAILABLE (503) i.e. VSM is not reachable.
:param method: type of the HTTP request. POST, GET, PUT or DELETE
:param action: path to which the client makes request
:param body: dict for arguments which are sent as part of the request
:param headers: header for the HTTP request
:returns: JSON or plain text in HTTP response
"""
action = self.action_prefix + action
if not headers and self.hosts:
headers = self._get_auth_header(self.hosts[0])
headers['Content-Type'] = self._set_content_type('json')
headers['Accept'] = self._set_content_type('json')
if body:
body = jsonutils.dumps(body, indent=2)
LOG.debug(_("req: %s"), body)
try:
resp = self.pool.spawn(requests.request,
method,
url=action,
data=body,
headers=headers,
timeout=self.timeout).wait()
except Exception as e:
raise c_exc.VSMConnectionFailed(reason=e)
LOG.debug(_("status_code %s"), resp.status_code)
if resp.status_code == requests.codes.OK:
if 'application/json' in resp.headers['content-type']:
try:
return resp.json()
except ValueError:
return {}
elif 'text/plain' in resp.headers['content-type']:
LOG.debug(_("VSM: %s"), resp.text)
else:
raise c_exc.VSMError(reason=resp.text)
def _set_content_type(self, format=None):
"""
Set the mime-type to either 'xml' or 'json'.
:param format: format to be set.
:return: mime-type string
"""
if not format:
format = self.format
return "application/%s" % format
def _delete(self, action, body=None, headers=None):
return self._do_request("DELETE", action, body=body, headers=headers)
def _get(self, action, body=None, headers=None):
return self._do_request("GET", action, body=body, headers=headers)
def _post(self, action, body=None, headers=None):
return self._do_request("POST", action, body=body, headers=headers)
def _put(self, action, body=None, headers=None):
return self._do_request("PUT", action, body=body, headers=headers)
def _get_vsm_hosts(self):
"""
Retrieve a list of VSM ip addresses.
:return: list of host ip addresses
"""
return [
cr[c_const.CREDENTIAL_NAME]
for cr in network_db_v2.get_all_n1kv_credentials()
]
def _get_auth_header(self, host_ip):
"""
Retrieve header with auth info for the VSM.
:param host_ip: IP address of the VSM
:return: authorization header dict
"""
username = c_cred.Store.get_username(host_ip)
password = c_cred.Store.get_password(host_ip)
auth = base64.encodestring("%s:%s" % (username, password)).rstrip()
header = {"Authorization": "Basic %s" % auth}
return header
def get_clusters(self):
"""Fetches a list of all vxlan gateway clusters."""
return self._get(self.clusters_path)
def create_encapsulation_profile(self, encap):
"""
Create an encapsulation profile on VSM.
:param encap: encapsulation dict
"""
body = {
'name': encap['name'],
'addMappings': encap['add_segment_list'],
'delMappings': encap['del_segment_list']
}
return self._post(self.encap_profiles_path, body=body)
def update_encapsulation_profile(self, context, profile_name, body):
"""
Adds a vlan to bridge-domain mapping to an encapsulation profile.
:param profile_name: Name of the encapsulation profile
:param body: mapping dictionary
"""
return self._post(self.encap_profile_path % profile_name, body=body)
def delete_encapsulation_profile(self, name):
"""
Delete an encapsulation profile on VSM.
:param name: name of the encapsulation profile to be deleted
"""
return self._delete(self.encap_profile_path % name)
import eventlet
def handle(fd):
print("client connected")
while True:
# pass through every non-eof line
x = fd.readline()
if not x:
break
fd.write(x)
fd.flush()
print("echoed", x, end=" ")
print("client disconnected")
if __name__ == "__main__":
server = eventlet.listen(("0.0.0.0", 6000))
pool = eventlet.GreenPool(10000)
while True:
try:
new_sock, address = server.accept()
print("accepted", address)
pool.spawn_n(handle, new_sock.makefile("rw"))
except (SystemExit, KeyboardInterrupt):
break
conn.close()
def run(filename_list):
conn = httplib.HTTPSConnection(CONNECTION_ENDPOINT)
for filename in filename_list:
start = time.time()
with open(filename, 'rb') as f:
conn.request('PUT',
CONTAINER_PATH + '/' + filename,
body=f,
headers=SEND_HEADERS)
resp = conn.getresponse()
resp.read()
print '%s uploaded in %.4f seconds' % (filename, (time.time() - start))
sys.stdout.flush()
conn.close()
data_list = [
'test_data/%s' % x for x in os.listdir('test_data') if x.endswith('.dat')
]
len_data_list = len(data_list)
concurrency = min(len_data_list, 20)
print 'uploading %d files with a concurrency of %d' % (len_data_list,
concurrency)
pool = eventlet.GreenPool(size=concurrency)
for i in xrange(len_data_list / concurrency):
pool.spawn(run, data_list[concurrency * i:concurrency * (i + 1)])
pool.waitall()
def big_object_get(self,
bucket_name,
object_name,
file_path=None,
thread_size=4,
part_size=32 * 1024 * 124):
if not file_path:
file_path = object_name
if os.path.exists(file_path): os.remove(file_path)
# Get the total length of the big object
resource = '/%s/%s' % (bucket_name, object_name)
conn = self.conn.get_request_conn('GET', resource)
response = conn.getresponse()
if response.status >= 400:
raise exceptions.from_response(response.status,
body=response.read())
content_length = int(dict(response.getheaders())['content-length'])
# Inner Method, Download a part_size piece of data and write file.
@utils.retries(5)
def get_piece(start_pos, end_pos):
if end_pos >= content_length:
bytes_str = 'bytes=%s-' % start_pos
else:
bytes_str = 'bytes=%s-%s' % (start_pos, end_pos - 1)
headers = {'Range': bytes_str}
cli = self.new_httpclient()
cli_conn = cli.get_request_conn('GET', resource, headers)
resp = cli_conn.getresponse()
if resp.status >= 400:
raise exceptions.from_response(resp.status, resp.read())
# receive 64K every time
with open(file_path, 'awb') as fd:
fd.seek(start_pos)
start = 0
while start < (end_pos - start_pos):
data = resp.read(self.io_buffer_size)
if len(data) <= 0:
break
fd.write(data)
start += self.io_buffer_size
# close socket
resp.close()
# start position, end position
green_pool = eventlet.GreenPool(thread_size)
start_pos = 0
end_pos = part_size
while start_pos < content_length:
green_pool.spawn(get_piece, start_pos, end_pos)
start_pos = end_pos
end_pos += part_size
# Wait all green finished
green_pool.waitall()
local_size = os.path.getsize(file_path)
error_msg = 'Size of local object(%s bytes) is not different from'\
' the size of remote jss object(%s bytes)' % (local_size,
content_length)
assert local_size == content_length, error_msg
def create_pool(self):
return eventlet.GreenPool(size=self.threads)
def __init__(self,
name,
app,
host=None,
port=None,
pool_size=None,
protocol=eventlet.wsgi.HttpProtocol,
backlog=128):
"""Initialize, but do not start, a WSGI server.
:param name: Pretty name for logging.
:param app: The WSGI application to serve.
:param host: IP address to serve the application.
:param port: Port number to server the application.
:param pool_size: Maximum number of eventlets to spawn concurrently.
:returns: None
"""
# Allow operators to customize http requests max header line size.
eventlet.wsgi.MAX_HEADER_LINE = CONF.max_header_line
self.name = name
self.app = app
self._host = host or "0.0.0.0"
self._port = port or 0
self._server = None
self._socket = None
self._protocol = protocol
self.pool_size = pool_size or self.default_pool_size
self._pool = eventlet.GreenPool(self.pool_size)
self._logger = logging.getLogger("eventlet.wsgi.server")
self._wsgi_logger = logging.WritableLogger(self._logger)
if backlog < 1:
raise exception.InvalidInput(
reason='The backlog must be more than 1')
bind_addr = (host, port)
# TODO(dims): eventlet's green dns/socket module does not actually
# support IPv6 in getaddrinfo(). We need to get around this in the
# future or monitor upstream for a fix
try:
info = socket.getaddrinfo(bind_addr[0], bind_addr[1],
socket.AF_UNSPEC, socket.SOCK_STREAM)[0]
family = info[0]
bind_addr = info[-1]
except Exception:
family = socket.AF_INET
cert_file = CONF.ssl_cert_file
key_file = CONF.ssl_key_file
ca_file = CONF.ssl_ca_file
self._use_ssl = cert_file or key_file
if cert_file and not os.path.exists(cert_file):
raise RuntimeError(_("Unable to find cert_file : %s") % cert_file)
if ca_file and not os.path.exists(ca_file):
raise RuntimeError(_("Unable to find ca_file : %s") % ca_file)
if key_file and not os.path.exists(key_file):
raise RuntimeError(_("Unable to find key_file : %s") % key_file)
if self._use_ssl and (not cert_file or not key_file):
raise RuntimeError(
_("When running server in SSL mode, you "
"must specify both a cert_file and "
"key_file option value in your "
"configuration file."))
retry_until = time.time() + 30
while not self._socket and time.time() < retry_until:
try:
self._socket = eventlet.listen(bind_addr,
backlog=backlog,
family=family)
except socket.error as err:
if err.args[0] != errno.EADDRINUSE:
raise
eventlet.sleep(0.1)
if not self._socket:
raise RuntimeError(
_("Could not bind to %(host)s:%(port)s "
"after trying for 30 seconds") % {
'host': host,
'port': port
})
(self._host, self._port) = self._socket.getsockname()[0:2]
LOG.info(
_("%(name)s listening on %(_host)s:%(_port)s") % self.__dict__)
def __init__(self, host, conf=None):
if conf:
self.conf = conf
else:
self.conf = cfg.CONF
self.router_info = {}
self._check_config_params()
self.process_monitor = external_process.ProcessMonitor(
config=self.conf,
resource_type='router')
self.driver = common_utils.load_interface_driver(self.conf)
self._context = n_context.get_admin_context_without_session()
self.plugin_rpc = L3PluginApi(topics.L3PLUGIN, host)
self.fullsync = True
self.sync_routers_chunk_size = SYNC_ROUTERS_MAX_CHUNK_SIZE
# Get the HA router count from Neutron Server
# This is the first place where we contact neutron-server on startup
# so retry in case its not ready to respond.
while True:
try:
self.ha_router_count = int(
self.plugin_rpc.get_host_ha_router_count(self.context))
except oslo_messaging.MessagingTimeout as e:
LOG.warning('l3-agent cannot contact neutron server '
'to retrieve HA router count. '
'Check connectivity to neutron server. '
'Retrying... '
'Detailed message: %(msg)s.', {'msg': e})
continue
break
LOG.info("Agent HA routers count %s", self.ha_router_count)
self.init_extension_manager(self.plugin_rpc)
self.metadata_driver = None
if self.conf.enable_metadata_proxy:
self.metadata_driver = metadata_driver.MetadataDriver(self)
self.namespaces_manager = namespace_manager.NamespaceManager(
self.conf,
self.driver,
self.metadata_driver)
# L3 agent router processing green pool
self._pool_size = ROUTER_PROCESS_GREENLET_MIN
self._pool = eventlet.GreenPool(size=self._pool_size)
self._queue = queue.ResourceProcessingQueue()
super(L3NATAgent, self).__init__(host=self.conf.host)
self.target_ex_net_id = None
self.use_ipv6 = ipv6_utils.is_enabled_and_bind_by_default()
self.pd = pd.PrefixDelegation(self.context, self.process_monitor,
self.driver,
self.plugin_rpc.process_prefix_update,
self.create_pd_router_update,
self.conf)
# Consume network updates to trigger router resync
consumers = [[topics.NETWORK, topics.UPDATE]]
agent_rpc.create_consumers([self], topics.AGENT, consumers)
self._check_ha_router_process_status()
def __init__(self, threads=1000):
self.pool = eventlet.GreenPool(threads)
class GenericApiRequest(request.ApiRequest):
'''A non Eventlet-based ApiRequest class.
This class will form the basis for eventlet-based ApiRequest classes
'''
# Maximum number of green threads present in the system at one time.
API_REQUEST_POOL_SIZE = request.DEFAULT_API_REQUEST_POOL_SIZE
# Pool of green threads. One green thread is allocated per incoming
# request. Incoming requests will block when the pool is empty.
API_REQUEST_POOL = eventlet.GreenPool(API_REQUEST_POOL_SIZE)
# A unique id is assigned to each incoming request. When the current
# request id reaches MAXIMUM_REQUEST_ID it wraps around back to 0.
MAXIMUM_REQUEST_ID = request.DEFAULT_MAXIMUM_REQUEST_ID
# The request id for the next incoming request.
CURRENT_REQUEST_ID = 0
def __init__(self,
client_obj,
url,
method="GET",
body=None,
headers=None,
request_id=CURRENT_REQUEST_ID,
retries=DEFAULT_RETRIES,
auto_login=True,
redirects=DEFAULT_REDIRECTS,
http_timeout=DEFAULT_HTTP_TIMEOUT,
client_conn=None):
'''Constructor.'''
self._api_client = client_obj
self._url = url
self._method = method
self._body = body if body else None
self._headers = headers or {}
self._request_timeout = http_timeout * retries
self._retries = retries
self._auto_login = auto_login
self._redirects = redirects
self._http_timeout = http_timeout
self._client_conn = client_conn
self._abort = False
self._request_error = None
if "User-Agent" not in self._headers:
self._headers["User-Agent"] = DEFAULT_USER_AGENT
# Retrieve and store this instance's unique request id.
self._request_id = request_id
def join(self):
'''Wait for instance green thread to complete.'''
pass
def start(self):
'''Start request processing.'''
return self._handle_request()
def _handle_request(self):
'''First level request handling.'''
attempt = 0
timeout = 0
badstatus = 0
response = None
while response is None and attempt <= self._retries:
attempt += 1
try:
req = self._issue_request()
except (httplib.BadStatusLine, socket.error):
if badstatus <= DEFAULT_RETRIES:
badstatus += 1
attempt -= 1
LOG.error("request {method} {url} {body} error".format(
method=self._method, url=self._url, body=self._body))
continue
# automatically raises any exceptions returned.
if isinstance(req, httplib.HTTPResponse):
timeout = 0
if attempt <= self._retries and not self._abort:
# currently there is a bug in fortios, it return 401 and
# 400 when a cookie is invalid, the change is to tolerant
# the bug to handle return 400 situation.
# when fortios fix the bug, here should use
# 'req.status in (401, 403)' instead
if req.status in (400, 401, 403):
continue
elif req.status == 503:
timeout = 0.5
continue
# else fall through to return the error code
LOG.info(
"[%(rid)d] Completed request '%(method)s %(url)s'"
": %(status)s", {
'rid': self._rid(),
'method': self._method,
'url': self._url,
'status': req.status
})
self._request_error = None
response = req
else:
LOG.info(
_LI('[%(rid)d] Error while handling request: '
'%(req)s'), {
'rid': self._rid(),
'req': req
})
self._request_error = req
response = None
return response
def __init__(self,
name,
app,
host='0.0.0.0',
port=0,
pool_size=None,
protocol=eventlet.wsgi.HttpProtocol,
backlog=128,
use_ssl=False,
max_url_len=None):
"""Initialize, but do not start, a WSGI server.
:param name: Pretty name for logging.
:param app: The WSGI application to serve.
:param host: IP address to serve the application.
:param port: Port number to server the application.
:param pool_size: Maximum number of eventlets to spawn concurrently.
:param backlog: Maximum number of queued connections.
:param max_url_len: Maximum length of permitted URLs.
:returns: None
:raises: masakari.exception.InvalidInput
"""
# Allow operators to customize http requests max header line size.
eventlet.wsgi.MAX_HEADER_LINE = CONF.wsgi.max_header_line
self.name = name
self.app = app
self._server = None
self._protocol = protocol
self.pool_size = pool_size or self.default_pool_size
self._pool = eventlet.GreenPool(self.pool_size)
self._logger = logging.getLogger("masakari.%s.wsgi.server" % self.name)
self._use_ssl = use_ssl
self._max_url_len = max_url_len
self.client_socket_timeout = CONF.wsgi.client_socket_timeout or None
if backlog < 1:
raise exception.InvalidInput(
reason=_('The backlog must be more than 0'))
bind_addr = (host, port)
try:
info = socket.getaddrinfo(bind_addr[0], bind_addr[1],
socket.AF_UNSPEC, socket.SOCK_STREAM)[0]
family = info[0]
bind_addr = info[-1]
except Exception:
family = socket.AF_INET
try:
self._socket = eventlet.listen(bind_addr, family, backlog=backlog)
except EnvironmentError:
LOG.error("Could not bind to %(host)s:%(port)d", {
'host': host,
'port': port
})
raise
(self.host, self.port) = self._socket.getsockname()[0:2]
LOG.info("%(name)s listening on %(host)s:%(port)d", {
'name': self.name,
'host': self.host,
'port': self.port
})
def __init__(self, port, handler=None):
Publisher.__init__(self)
self.port = port
self.handler = handler
self.pool = eventlet.GreenPool(10000)
self.connections = weakref.WeakValueDictionary()
rec = sock.recv(1024)
if not rec:
break
if rec.decode() in model:
data = eval(rec.decode().replace('.', '_'))()
logger.debug('execute function %s' %
rec.decode().replace('.', '_'))
if isinstance(data, int) or isinstance(data, float):
sock.send(str(data).encode())
elif isinstance(data, str):
sock.sendall(data.encode())
else:
sock.sendall(b'no')
try:
server = eventlet.listen((agent_conf.listen, agent_conf.agent_port))
pool = eventlet.GreenPool(agent_conf.maxclient)
except Exception as msg:
raise msg
finally:
logger.info("listen %s:%s" % (agent_conf.listen, agent_conf.agent_port))
while True:
try:
sock, address = server.accept()
pool.spawn_n(handle, sock)
logger.info(address)
except Exception as msg:
raise msg
