def setUp(self):
logger = logging.getLogger()
logger.level = logging.DEBUG
self.log_file = tempfile.NamedTemporaryFile()
logger.addHandler(logging.FileHandler(self.log_file.name))
logger.addHandler(logging.StreamHandler(sys.stdout))
self.chewie = Chewie('lo', logger, auth_handler, failure_handler,
logoff_handler, '127.0.0.1', 1812, 'SECRET',
'44:44:44:44:44:44')
self.fake_scheduler = FakeTimerScheduler()
self.chewie.timer_scheduler = self.fake_scheduler
global FROM_SUPPLICANT # pylint: disable=global-statement
global TO_SUPPLICANT # pylint: disable=global-statement
global FROM_RADIUS # pylint: disable=global-statement
global TO_RADIUS # pylint: disable=global-statement
global FROM_SUPPLICANT_ACTIVITY # pylint: disable=global-statement
FROM_SUPPLICANT = Queue()
FROM_SUPPLICANT_ACTIVITY = Queue()
TO_SUPPLICANT = Queue()
FROM_RADIUS = Queue()
TO_RADIUS = Queue()
def __init__(self,
local_as,
peer_as,
router_id,
local_address,
neighbor,
hold_time=DEFAULT_HOLD_TIME,
open_handler=None):
self.local_as = local_as
if local_as > 65535:
self.local_as2 = 23456
else:
self.local_as2 = self.local_as
self.peer_as = peer_as
self.router_id = IPAddress.from_string(router_id)
self.local_address = IPAddress.from_string(local_address)
self.neighbor = IPAddress.from_string(neighbor)
self.hold_time = hold_time
self.open_handler = open_handler
self.keepalive_time = hold_time // 3
self.output_messages = Queue()
self.route_updates = Queue()
self.routes_to_advertise = []
self.fourbyteas = False
self.timers = {
"hold": Timer(self.hold_time),
"keepalive": Timer(self.keepalive_time),
}
self.state = "active"
def __init__(self):
self.queue_send = Queue()
self.queue_recv = Queue()
self.appid = None
# interface
self.sock = None
def __init__(self,
interface_name,
logger=None,
auth_handler=None,
failure_handler=None,
logoff_handler=None,
radius_server_ip=None,
radius_server_port=None,
radius_server_secret=None,
chewie_id=None):
self.interface_name = interface_name
self.log_name = Chewie.__name__
if logger:
self.log_name = logger.name + "." + Chewie.__name__
self.logger = get_logger(self.log_name)
self.auth_handler = auth_handler
self.failure_handler = failure_handler
self.logoff_handler = logoff_handler
self.radius_server_ip = radius_server_ip
self.radius_secret = radius_server_secret
self.radius_server_port = self.RADIUS_UDP_PORT
if radius_server_port:
self.radius_server_port = radius_server_port
self.radius_listen_ip = "0.0.0.0"
self.radius_listen_port = 0
self.chewie_id = "44-44-44-44-44-44:" # used by the RADIUS Attribute
# 'Called-Station' in Access-Request
if chewie_id:
self.chewie_id = chewie_id
self.state_machines = {} # port_id_str: { mac : state_machine}
self.port_to_eapol_id = {
} # port_id: last ID used in preemptive identity request.
# TODO for port_to_eapol_id - may want to set ID to null (-1...) if sent from the
# state machine.
self.port_status = {} # port_id: status (true=up, false=down)
self.port_to_identity_job = {} # port_id: timerJob
self.eap_output_messages = Queue()
self.radius_output_messages = Queue()
self.radius_lifecycle = RadiusLifecycle(self.radius_secret,
self.chewie_id, self.logger)
self.timer_scheduler = timer_scheduler.TimerScheduler(self.logger)
self.eap_socket = None
self.mab_socket = None
self.pool = None
self.eventlets = None
self.radius_socket = None
self.interface_index = None
self.eventlets = []
def init_portlist(self):
self.minicap_q = Queue(50)
self.minitouch_q = Queue(50)
self.stfservices_q = Queue(50)
for port in range(1300, 1350):
self.minicap_q.put(port)
self.minitouch_q.put(port - 100)
for port in range(1100, 1190, 2):
self.stfservices_q.put(port)
def setUp(self):
self.logger = logging.getLogger()
self.logger.level = logging.DEBUG
self.log_file = tempfile.NamedTemporaryFile()
self.logger.addHandler(logging.FileHandler(self.log_file.name))
self.logger.addHandler(logging.StreamHandler(sys.stdout))
self.fake_scheduler = FakeTimerScheduler()
self.timer_scheduler = self.fake_scheduler
self.managed_port = None
self.eap_output_messages = Queue() # pylint: disable=global-statement
self.radius_output_messages = Queue() # pylint: disable=global-statement
def _make_app_iter(self, node, source):
"""
Returns an iterator over the contents of the source (via its read
func). There is also quite a bit of cleanup to ensure garbage
collection works and the underlying socket of the source is closed.
:param source: The httplib.Response object this iterator should read
from.
:param node: The node the source is reading from, for logging purposes.
"""
try:
# Spawn reader to read from the source and place in the queue.
# We then drop any reference to the source or node, for garbage
# collection purposes.
queue = Queue(1)
spawn_n(self._make_app_iter_reader, node, source, queue,
self.app.logger.thread_locals)
source = node = None
while True:
chunk = queue.get(timeout=self.app.node_timeout)
if isinstance(chunk, bool): # terminator
success = chunk
if not success:
raise Exception(
_('Failed to read all data'
' from the source'))
break
yield chunk
except Empty:
raise ChunkReadTimeout()
except (GeneratorExit, Timeout):
self.app.logger.warn(_('Client disconnected on read'))
except Exception:
self.app.logger.exception(_('Trying to send to client'))
raise
def __init__(self, *args, **kwargs):
Connection.__init__(self, *args, **kwargs)
self.connected_event = Event()
self._write_queue = Queue()
self._callbacks = {}
self._push_watchers = defaultdict(set)
sockerr = None
addresses = socket.getaddrinfo(self.host, self.port, socket.AF_UNSPEC,
socket.SOCK_STREAM)
for (af, socktype, proto, canonname, sockaddr) in addresses:
try:
self._socket = socket.socket(af, socktype, proto)
self._socket.settimeout(1.0)
self._socket.connect(sockaddr)
sockerr = None
break
except socket.error as err:
sockerr = err
if sockerr:
raise socket.error(
sockerr.errno, "Tried connecting to %s. Last error: %s" %
([a[4] for a in addresses], sockerr.strerror))
if self.sockopts:
for args in self.sockopts:
self._socket.setsockopt(*args)
self._read_watcher = eventlet.spawn(lambda: self.handle_read())
self._write_watcher = eventlet.spawn(lambda: self.handle_write())
self._send_options_message()
def __init__(self, *args, **kwargs):
super(AsyncClient, self).__init__(*args, **kwargs)
self.pool = eventlet.greenpool.GreenPool(DEFAULT_POOL_SIZE)
self.reader_thread = None
self.writer_thread = None
self.queue = Queue(DEFAULT_MAX_QUEUE_SIZE)
self.max_pending = MAX_PENDING
self.closing = False
def __init__(self,
interface_name,
logger=None,
auth_handler=None,
failure_handler=None,
logoff_handler=None,
radius_server_ip=None,
radius_server_port=None,
radius_server_secret=None,
chewie_id=None):
self.interface_name = interface_name
self.logger = get_logger(logger.name + "." + Chewie.__name__)
self.auth_handler = auth_handler
self.failure_handler = failure_handler
self.logoff_handler = logoff_handler
self.radius_server_ip = radius_server_ip
self.radius_secret = radius_server_secret
self.radius_server_port = self.RADIUS_UDP_PORT
if radius_server_port:
self.radius_server_port = radius_server_port
self.radius_listen_ip = "0.0.0.0"
self.radius_listen_port = 0
self.chewie_id = "44-44-44-44-44-44:" # used by the RADIUS Attribute
# 'Called-Station' in Access-Request
if chewie_id:
self.chewie_id = chewie_id
self.state_machines = {} # mac: state_machine
self.eap_output_messages = Queue()
self.radius_output_messages = Queue()
self.radius_lifecycle = RadiusLifecycle(self.radius_secret,
self.chewie_id, self.logger)
self.timer_scheduler = timer_scheduler.TimerScheduler(self.logger)
self.eap_socket = None
self.pool = None
self.eventlets = None
self.radius_socket = None
self.interface_index = None
self.eventlets = []
def __init__(self, *args, **kwargs):
Connection.__init__(self, *args, **kwargs)
self._write_queue = Queue()
self._connect_socket()
self._read_watcher = eventlet.spawn(lambda: self.handle_read())
self._write_watcher = eventlet.spawn(lambda: self.handle_write())
self._send_options_message()
def _recvData_init(self):
self.__dataq=Queue(1000)
self.readBannerBytes=0
self.bannerLength=0
self.readFrameBytes=0
self.frameBodyLength=0
self.frameBodyLengthStr=b''
self.frameBody=b''
self.banner={'version':0,'length':0,'pid':0,'realWidth':0,'realHeight':0,'virtualWidth':0,'realHeight':0,'orientation':0,'quirks':0}
def __init__(self, addr, serial):
self.serial = serial
self.addr = addr
# self.desiredState=StateQueue()
self.serviceQueue = Queue(500)
self.send_status = 0 #0:初始化;1:接收状态;-1:关闭状态;-2:关闭ing
self.get_status = 0 #0:初始化;1:接收状态;-1:关闭状态;-2:关闭ing
self.service = Service(serial, [self.addr[1], self.addr[1] + 1])
self.msgQ = {}
self.phone = None
def add(self, key, addr):
if self.clients.get(key):
return -1
else:
try:
c = RealtimeScreenCap(addr, key)
q = Queue(500)
q2 = Queue(500)
t = RealtimeScreenTouch(('localhost', 1111), key, q)
s = stfServices(('localhost', 1100), key, q2)
self.clients[key] = {
'touch': t,
'cap': c,
'touchQ': q,
'services': s,
'servicesQ': q2
}
print(self.keys(), 'keys')
return 1
except Exception as e:
print(str(e))
return 0
def __init__(self, src_mac, auth_success=None):
self.src_mac = src_mac
self.auth_success = auth_success
self.txn_id = None
self.challenge = None
self.expected_response = None
# TODO - some way to query for this based on identity
self.password = "******"
self.state = "idle"
self.output_messages = Queue()
self.txn_id_method = generate_txn_id
self.challenge_method = generate_random_bytes
def __init__(self, pipe):
self._pipe = pipe
self._queue = Queue()
_queue = self._queue
def _enqueue_output():
for line in iter(pipe.readline, b''):
_queue.put(line)
pipe.close()
self._green_thread = eventlet.spawn(_enqueue_output)
def __init__(self, addr, serial, deviceInfos):
self.serial = serial
self.addr = addr
self.pid = -1
self.desiredState = StateQueue()
self.socket = None
self.runningState = 1 #0 STATE_STOPPED = 1 STATE_STARTING = 2 STATE_STARTED = 3 STATE_STOPPING = 4
self.minitouchService = MinitouchService(serial, self.addr[1],
deviceInfos)
self.touchQueue = Queue(500)
self.send_status = 0 #0:stoped;1:start;-2:stoping
self.get_status = 0 #0:stoped;1:start;-2:stoping
self._init()
self.actionStatus = None
def __init__(self, logger: Logger, interval: float = 10):
self._logger = logger
self._q = Queue()
self._ok_submits = 0
self._bad_submits = 0
self._connections = 0
self._interval = interval
self._was_ok = self._ok_submits
self._was_bad = self._bad_submits
self._was_conn = self._connections
self._labels = ['attacker_id', 'victim_id', 'task_id', 'submit_ok']
def __init__(self, addr, serial):
self.serial = serial
self.addr = addr
self.socket = None
self.socket22 = None
self.desiredState = StateQueue()
self.serviceQueue = Queue(500)
self.runningState = 1 #0 STATE_STOPPED = 1 STATE_STARTING = 2 STATE_STARTED = 3 STATE_STOPPING = 4
self.send_status = 0 #0:初始化;1:接收状态;-1:关闭状态;-2:关闭ing
self.get_status = 0 #0:初始化;1:接收状态;-1:关闭状态;-2:关闭ing
self.service = Service(serial, [self.addr[1], self.addr[1] + 1])
self.msgQ = {}
self.phone = None
self._init()
self.namespace = '/screen%s' % self.serial
def __init__(self,addr,channel,manager,key):
self.key=key
self.manager=manager
self.channel=''
self.addr=addr
self.__dataq=Queue(1000)
self.recv_status=True
self.push_status=True
self.readBannerBytes=0
self.bannerLength=0
self.readFrameBytes=0
self.frameBodyLength=0
self.frameBodyLengthStr=b''
self.frameBody=b''
self.banner={'version':0,'length':0,'pid':0,'realWidth':0,'realHeight':0,'virtualWidth':0,'realHeight':0,'orientation':0,'quirks':0}
def __init__(self, config, queue=None, quit_event=None, error_queue=None, pool=None):
self._quit_event = quit_event or Event()
self.changed = set()
self.queue = queue or Queue()
self.error_queue = error_queue
self._property_collector = None
self.down_ports = {}
self.untried_ports = {} # The host is simply down
self.iteration = 0
self.connection = None
# Map of the VMs and their NICs by the hardware key
# e.g vmobrefs -> keys -> _DVSPortMonitorDesc
self._hardware_map = defaultdict(dict)
# super(VCenterMonitor, self).__init__(target=self._run, args=(config,))
pool = pool or eventlet
self.thread = pool.spawn(self._run_safe, config)
def __init__(self, socket, address):
super(Connection, self).__init__()
self.socket = socket
self.address = address
self.is_active = True
# The limit is arbitrary. We need to limit queue size to
# prevent it from eating memory up
self.send_q = Queue(128)
# data structures for BGP
self.peer_ip = None
self.peer_as = None
self.peer_id = None
self.peer_capabilities = []
self._4or6 = 0
self.hold_time = 240
def __init__(self, *args, **kwargs):
Connection.__init__(self, *args, **kwargs)
self.connected_event = Event()
self._iobuf = StringIO()
self._write_queue = Queue()
self._callbacks = {}
self._push_watchers = defaultdict(set)
self._socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self._socket.settimeout(1.0)
self._socket.connect((self.host, self.port))
if self.sockopts:
for args in self.sockopts:
self._socket.setsockopt(*args)
self._read_watcher = eventlet.spawn(lambda: self.handle_read())
self._write_watcher = eventlet.spawn(lambda: self.handle_write())
self._send_options_message()
def __init__(self, conf):
TRUE_VALUES = set(('true', '1', 'yes', 'on', 't', 'y'))
self.conf = conf
self.logger = logging.getLogger('statsdlogd')
self.logger.setLevel(logging.INFO)
self.syslog = SysLogHandler(address='/dev/log')
self.formatter = logging.Formatter('%(name)s: %(message)s')
self.syslog.setFormatter(self.formatter)
self.logger.addHandler(self.syslog)
self.debug = conf.get('debug', 'false').lower() in TRUE_VALUES
self.statsd_host = conf.get('statsd_host', '127.0.0.1')
self.statsd_port = int(conf.get('statsd_port', '8125'))
self.listen_addr = conf.get('listen_addr', '127.0.0.1')
self.listen_port = int(conf.get('listen_port', 8126))
self.report_internal_stats = conf.get('report_internal_stats',
'true').lower() in TRUE_VALUES
self.int_stats_interval = int(conf.get('internal_stats_interval', 5))
self.buff = int(conf.get('buffer_size', 8192))
self.max_q_size = int(conf.get('max_line_backlog', 512))
self.statsd_sample_rate = float(conf.get('statsd_sample_rate', '.5'))
self.counter = 0
self.skip_counter = 0
self.hits = 0
self.q = Queue(maxsize=self.max_q_size)
# key: regex
self.patterns_file = conf.get('patterns_file',
'/etc/statsdlog/patterns.json')
self.json_patterns = conf.get('json_pattern_file',
'true').lower() in TRUE_VALUES
try:
self.patterns = self.load_patterns()
except Exception as err:
self.logger.exception(err)
print "Encountered exception at startup: %s" % err
sys.exit(1)
self.statsd_addr = (self.statsd_host, self.statsd_port)
self.comp_patterns = {}
for item in self.patterns:
self.comp_patterns[item] = re.compile(self.patterns[item])
def __init__(self, config=None, pool=None, agent=None):
self.pool = pool
self.agent = agent
self.config = config or CONF.ML2_VMWARE
self.connection = _create_session(self.config)
self.context = agent.context
self._monitor_process = VCenterMonitor(self,
self.config,
connection=self.connection,
pool=self.pool)
self.queue = Queue(None)
self.uuid_port_map = {}
self.mac_port_map = {}
self.uuid_dvs_map = {}
self.network_dvs_map = {}
for network, dvs in six.iteritems(
dvs_util.create_network_map_from_config(
self.config, connection=self.connection, pool=pool)):
self.network_dvs_map[network] = dvs
self.uuid_dvs_map[dvs.uuid] = dvs
def PUT(self, req):
"""HTTP PUT request handler."""
if req.if_none_match is not None and '*' not in req.if_none_match:
# Sending an etag with if-none-match isn't currently supported
return HTTPBadRequest(request=req, content_type='text/plain',
body='If-None-Match only supports *')
container_info = self.container_info(
self.account_name, self.container_name, req)
policy_index = req.headers.get('X-Backend-Storage-Policy-Index',
container_info['storage_policy'])
obj_ring = self.app.get_object_ring(policy_index)
# pass the policy index to storage nodes via req header
req.headers['X-Backend-Storage-Policy-Index'] = policy_index
container_partition = container_info['partition']
containers = container_info['nodes']
req.acl = container_info['write_acl']
req.environ['swift_sync_key'] = container_info['sync_key']
object_versions = container_info['versions']
if 'swift.authorize' in req.environ:
aresp = req.environ['swift.authorize'](req)
if aresp:
return aresp
if not containers:
return HTTPNotFound(request=req)
try:
ml = req.message_length()
except ValueError as e:
return HTTPBadRequest(request=req, content_type='text/plain',
body=str(e))
except AttributeError as e:
return HTTPNotImplemented(request=req, content_type='text/plain',
body=str(e))
if ml is not None and ml > constraints.MAX_FILE_SIZE:
return HTTPRequestEntityTooLarge(request=req)
if 'x-delete-after' in req.headers:
try:
x_delete_after = int(req.headers['x-delete-after'])
except ValueError:
return HTTPBadRequest(request=req,
content_type='text/plain',
body='Non-integer X-Delete-After')
req.headers['x-delete-at'] = normalize_delete_at_timestamp(
time.time() + x_delete_after)
partition, nodes = obj_ring.get_nodes(
self.account_name, self.container_name, self.object_name)
# do a HEAD request for container sync and checking object versions
if 'x-timestamp' in req.headers or \
(object_versions and not
req.environ.get('swift_versioned_copy')):
# make sure proxy-server uses the right policy index
_headers = {'X-Backend-Storage-Policy-Index': policy_index,
'X-Newest': 'True'}
hreq = Request.blank(req.path_info, headers=_headers,
environ={'REQUEST_METHOD': 'HEAD'})
hresp = self.GETorHEAD_base(
hreq, _('Object'), obj_ring, partition,
hreq.swift_entity_path)
# Used by container sync feature
if 'x-timestamp' in req.headers:
try:
req_timestamp = Timestamp(req.headers['X-Timestamp'])
if hresp.environ and 'swift_x_timestamp' in hresp.environ and \
hresp.environ['swift_x_timestamp'] >= req_timestamp:
return HTTPAccepted(request=req)
except ValueError:
return HTTPBadRequest(
request=req, content_type='text/plain',
body='X-Timestamp should be a UNIX timestamp float value; '
'was %r' % req.headers['x-timestamp'])
req.headers['X-Timestamp'] = req_timestamp.internal
else:
req.headers['X-Timestamp'] = Timestamp(time.time()).internal
# Sometimes the 'content-type' header exists, but is set to None.
content_type_manually_set = True
detect_content_type = \
config_true_value(req.headers.get('x-detect-content-type'))
if detect_content_type or not req.headers.get('content-type'):
guessed_type, _junk = mimetypes.guess_type(req.path_info)
req.headers['Content-Type'] = guessed_type or \
'application/octet-stream'
if detect_content_type:
req.headers.pop('x-detect-content-type')
else:
content_type_manually_set = False
error_response = check_object_creation(req, self.object_name) or \
check_content_type(req)
if error_response:
return error_response
if object_versions and not req.environ.get('swift_versioned_copy'):
if hresp.status_int != HTTP_NOT_FOUND:
# This is a version manifest and needs to be handled
# differently. First copy the existing data to a new object,
# then write the data from this request to the version manifest
# object.
lcontainer = object_versions.split('/')[0]
prefix_len = '%03x' % len(self.object_name)
lprefix = prefix_len + self.object_name + '/'
ts_source = hresp.environ.get('swift_x_timestamp')
if ts_source is None:
ts_source = time.mktime(time.strptime(
hresp.headers['last-modified'],
'%a, %d %b %Y %H:%M:%S GMT'))
new_ts = Timestamp(ts_source).internal
vers_obj_name = lprefix + new_ts
copy_headers = {
'Destination': '%s/%s' % (lcontainer, vers_obj_name)}
copy_environ = {'REQUEST_METHOD': 'COPY',
'swift_versioned_copy': True
}
copy_req = Request.blank(req.path_info, headers=copy_headers,
environ=copy_environ)
copy_resp = self.COPY(copy_req)
if is_client_error(copy_resp.status_int):
# missing container or bad permissions
return HTTPPreconditionFailed(request=req)
elif not is_success(copy_resp.status_int):
# could not copy the data, bail
return HTTPServiceUnavailable(request=req)
reader = req.environ['wsgi.input'].read
data_source = iter(lambda: reader(self.app.client_chunk_size), '')
source_header = req.headers.get('X-Copy-From')
source_resp = None
if source_header:
if req.environ.get('swift.orig_req_method', req.method) != 'POST':
req.environ.setdefault('swift.log_info', []).append(
'x-copy-from:%s' % source_header)
src_container_name, src_obj_name = check_copy_from_header(req)
ver, acct, _rest = req.split_path(2, 3, True)
if isinstance(acct, unicode):
acct = acct.encode('utf-8')
source_header = '/%s/%s/%s/%s' % (ver, acct,
src_container_name, src_obj_name)
source_req = req.copy_get()
# make sure the source request uses it's container_info
source_req.headers.pop('X-Backend-Storage-Policy-Index', None)
source_req.path_info = source_header
source_req.headers['X-Newest'] = 'true'
orig_obj_name = self.object_name
orig_container_name = self.container_name
self.object_name = src_obj_name
self.container_name = src_container_name
sink_req = Request.blank(req.path_info,
environ=req.environ, headers=req.headers)
source_resp = self.GET(source_req)
# This gives middlewares a way to change the source; for example,
# this lets you COPY a SLO manifest and have the new object be the
# concatenation of the segments (like what a GET request gives
# the client), not a copy of the manifest file.
hook = req.environ.get(
'swift.copy_hook',
(lambda source_req, source_resp, sink_req: source_resp))
source_resp = hook(source_req, source_resp, sink_req)
if source_resp.status_int >= HTTP_MULTIPLE_CHOICES:
return source_resp
self.object_name = orig_obj_name
self.container_name = orig_container_name
data_source = iter(source_resp.app_iter)
sink_req.content_length = source_resp.content_length
if sink_req.content_length is None:
# This indicates a transfer-encoding: chunked source object,
# which currently only happens because there are more than
# CONTAINER_LISTING_LIMIT segments in a segmented object. In
# this case, we're going to refuse to do the server-side copy.
return HTTPRequestEntityTooLarge(request=req)
if sink_req.content_length > constraints.MAX_FILE_SIZE:
return HTTPRequestEntityTooLarge(request=req)
sink_req.etag = source_resp.etag
# we no longer need the X-Copy-From header
del sink_req.headers['X-Copy-From']
if not content_type_manually_set:
sink_req.headers['Content-Type'] = \
source_resp.headers['Content-Type']
if not config_true_value(
sink_req.headers.get('x-fresh-metadata', 'false')):
copy_headers_into(source_resp, sink_req)
copy_headers_into(req, sink_req)
# copy over x-static-large-object for POSTs and manifest copies
if 'X-Static-Large-Object' in source_resp.headers and \
req.params.get('multipart-manifest') == 'get':
sink_req.headers['X-Static-Large-Object'] = \
source_resp.headers['X-Static-Large-Object']
req = sink_req
if 'x-delete-at' in req.headers:
try:
x_delete_at = normalize_delete_at_timestamp(
int(req.headers['x-delete-at']))
if int(x_delete_at) < time.time():
return HTTPBadRequest(
body='X-Delete-At in past', request=req,
content_type='text/plain')
except ValueError:
return HTTPBadRequest(request=req, content_type='text/plain',
body='Non-integer X-Delete-At')
req.environ.setdefault('swift.log_info', []).append(
'x-delete-at:%s' % x_delete_at)
delete_at_container = normalize_delete_at_timestamp(
int(x_delete_at) /
self.app.expiring_objects_container_divisor *
self.app.expiring_objects_container_divisor)
delete_at_part, delete_at_nodes = \
self.app.container_ring.get_nodes(
self.app.expiring_objects_account, delete_at_container)
else:
delete_at_container = delete_at_part = delete_at_nodes = None
node_iter = GreenthreadSafeIterator(
self.iter_nodes_local_first(obj_ring, partition))
pile = GreenPile(len(nodes))
te = req.headers.get('transfer-encoding', '')
chunked = ('chunked' in te)
outgoing_headers = self._backend_requests(
req, len(nodes), container_partition, containers,
delete_at_container, delete_at_part, delete_at_nodes)
for nheaders in outgoing_headers:
# RFC2616:8.2.3 disallows 100-continue without a body
if (req.content_length > 0) or chunked:
nheaders['Expect'] = '100-continue'
pile.spawn(self._connect_put_node, node_iter, partition,
req.swift_entity_path, nheaders,
self.app.logger.thread_locals)
conns = [conn for conn in pile if conn]
min_conns = quorum_size(len(nodes))
if req.if_none_match is not None and '*' in req.if_none_match:
statuses = [conn.resp.status for conn in conns if conn.resp]
if HTTP_PRECONDITION_FAILED in statuses:
# If we find any copy of the file, it shouldn't be uploaded
self.app.logger.debug(
_('Object PUT returning 412, %(statuses)r'),
{'statuses': statuses})
return HTTPPreconditionFailed(request=req)
if len(conns) < min_conns:
self.app.logger.error(
_('Object PUT returning 503, %(conns)s/%(nodes)s '
'required connections'),
{'conns': len(conns), 'nodes': min_conns})
return HTTPServiceUnavailable(request=req)
bytes_transferred = 0
try:
with ContextPool(len(nodes)) as pool:
for conn in conns:
conn.failed = False
conn.queue = Queue(self.app.put_queue_depth)
pool.spawn(self._send_file, conn, req.path)
while True:
with ChunkReadTimeout(self.app.client_timeout):
try:
chunk = next(data_source)
except StopIteration:
if chunked:
for conn in conns:
conn.queue.put('0\r\n\r\n')
break
bytes_transferred += len(chunk)
if bytes_transferred > constraints.MAX_FILE_SIZE:
return HTTPRequestEntityTooLarge(request=req)
for conn in list(conns):
if not conn.failed:
conn.queue.put(
'%x\r\n%s\r\n' % (len(chunk), chunk)
if chunked else chunk)
else:
conns.remove(conn)
if len(conns) < min_conns:
self.app.logger.error(_(
'Object PUT exceptions during'
' send, %(conns)s/%(nodes)s required connections'),
{'conns': len(conns), 'nodes': min_conns})
return HTTPServiceUnavailable(request=req)
for conn in conns:
if conn.queue.unfinished_tasks:
conn.queue.join()
conns = [conn for conn in conns if not conn.failed]
except ChunkReadTimeout as err:
self.app.logger.warn(
_('ERROR Client read timeout (%ss)'), err.seconds)
self.app.logger.increment('client_timeouts')
return HTTPRequestTimeout(request=req)
except (Exception, Timeout):
self.app.logger.exception(
_('ERROR Exception causing client disconnect'))
return HTTPClientDisconnect(request=req)
if req.content_length and bytes_transferred < req.content_length:
req.client_disconnect = True
self.app.logger.warn(
_('Client disconnected without sending enough data'))
self.app.logger.increment('client_disconnects')
return HTTPClientDisconnect(request=req)
statuses, reasons, bodies, etags = self._get_put_responses(req, conns,
nodes)
if len(etags) > 1:
self.app.logger.error(
_('Object servers returned %s mismatched etags'), len(etags))
return HTTPServerError(request=req)
etag = etags.pop() if len(etags) else None
resp = self.best_response(req, statuses, reasons, bodies,
_('Object PUT'), etag=etag)
if source_header:
resp.headers['X-Copied-From'] = quote(
source_header.split('/', 3)[3])
if 'last-modified' in source_resp.headers:
resp.headers['X-Copied-From-Last-Modified'] = \
source_resp.headers['last-modified']
copy_headers_into(req, resp)
resp.last_modified = math.ceil(
float(Timestamp(req.headers['X-Timestamp'])))
return resp
def __getattr__(self, key):
return self._wrap(key)
def _wrap(self, method):
@functools.wraps(app.route)
def func(*v, **kw):
kw["methods"] = [method.upper()]
return app.route(*v, **kw)
return func
rest = _rest()
# @TODO: all (web-)socket related components shall be in one place...
sig_queue = Queue()
#sig_archive = {}
class SignalManager(object):
none_repl = "n/a"
def __init__(self):
pass
def parse_signal(self, when, sender, sig, uuid, more_args, more_kw):
if signals_debug:
print("\n\n")
print("---- SignalManager ----")
print("when:", when, "sender:", sender, "sig:", sig, "uuid:", uuid)
print("args:", more_args, more_kw)
def __init__(self):
super().__init__()
self._queue = Queue(1024)
def stream(self, source, size=None):
bytes_transferred = 0
meta_chunk = self.meta_chunk
meta_checksum = hashlib.md5()
pile = GreenPile(len(meta_chunk))
failed_chunks = []
current_conns = []
for chunk in meta_chunk:
pile.spawn(self._connect_put, chunk)
for conn, chunk in [d for d in pile]:
if not conn:
failed_chunks.append(chunk)
else:
current_conns.append(conn)
self.quorum_or_fail([co.chunk for co in current_conns], failed_chunks)
bytes_transferred = 0
try:
with green.ContextPool(len(meta_chunk)) as pool:
for conn in current_conns:
conn.failed = False
conn.queue = Queue(io.PUT_QUEUE_DEPTH)
pool.spawn(self._send_data, conn)
while True:
if size is not None:
remaining_bytes = size - bytes_transferred
if io.WRITE_CHUNK_SIZE < remaining_bytes:
read_size = io.WRITE_CHUNK_SIZE
else:
read_size = remaining_bytes
else:
read_size = io.WRITE_CHUNK_SIZE
with green.SourceReadTimeout(self.read_timeout):
try:
data = source.read(read_size)
except (ValueError, IOError) as e:
raise SourceReadError(str(e))
if len(data) == 0:
for conn in current_conns:
if not conn.failed:
conn.queue.put('0\r\n\r\n')
break
self.checksum.update(data)
meta_checksum.update(data)
bytes_transferred += len(data)
# copy current_conns to be able to remove a failed conn
for conn in current_conns[:]:
if not conn.failed:
conn.queue.put('%x\r\n%s\r\n' % (len(data), data))
else:
current_conns.remove(conn)
failed_chunks.append(conn.chunk)
self.quorum_or_fail([co.chunk for co in current_conns],
failed_chunks)
for conn in current_conns:
if conn.queue.unfinished_tasks:
conn.queue.join()
except green.SourceReadTimeout:
logger.warn('Source read timeout')
raise
except SourceReadError:
logger.warn('Source read error')
raise
except Timeout as to:
logger.exception('Timeout writing data')
raise exc.OioTimeout(to)
except Exception:
logger.exception('Exception writing data')
raise
success_chunks = []
for conn in current_conns:
if conn.failed:
failed_chunks.append(conn.chunk)
continue
pile.spawn(self._get_response, conn)
meta_checksum_hex = meta_checksum.hexdigest()
for (conn, resp) in pile:
if resp:
self._handle_resp(conn, resp, meta_checksum_hex,
success_chunks, failed_chunks)
self.quorum_or_fail(success_chunks, failed_chunks)
for chunk in success_chunks:
chunk["size"] = bytes_transferred
chunk["hash"] = meta_checksum_hex
return bytes_transferred, meta_checksum_hex, success_chunks
def PUT(self, req):
"""HTTP PUT request handler."""
if req.if_none_match is not None and '*' not in req.if_none_match:
# Sending an etag with if-none-match isn't currently supported
return HTTPBadRequest(request=req,
content_type='text/plain',
body='If-None-Match only supports *')
container_info = self.container_info(self.account_name,
self.container_name, req)
policy_index = req.headers.get('X-Backend-Storage-Policy-Index',
container_info['storage_policy'])
obj_ring = self.app.get_object_ring(policy_index)
# pass the policy index to storage nodes via req header
req.headers['X-Backend-Storage-Policy-Index'] = policy_index
container_partition = container_info['partition']
containers = container_info['nodes']
req.acl = container_info['write_acl']
req.environ['swift_sync_key'] = container_info['sync_key']
object_versions = container_info['versions']
if 'swift.authorize' in req.environ:
aresp = req.environ['swift.authorize'](req)
if aresp:
return aresp
if not containers:
return HTTPNotFound(request=req)
# Sometimes the 'content-type' header exists, but is set to None.
content_type_manually_set = True
detect_content_type = \
config_true_value(req.headers.get('x-detect-content-type'))
if detect_content_type or not req.headers.get('content-type'):
guessed_type, _junk = mimetypes.guess_type(req.path_info)
req.headers['Content-Type'] = guessed_type or \
'application/octet-stream'
if detect_content_type:
req.headers.pop('x-detect-content-type')
else:
content_type_manually_set = False
error_response = check_object_creation(req, self.object_name) or \
check_content_type(req)
if error_response:
return error_response
partition, nodes = obj_ring.get_nodes(self.account_name,
self.container_name,
self.object_name)
#################################### CHANGED_CODE ############################################################
# Change the nodes list to contain only one dictionary item instead of the original 3 returned by the ring.
d = dict()
# d[partition] = nodes[1:]
# f.write(str(d)+"\n")
# f.close()
print("===Original Nodes===")
print(nodes)
temp_nodes = []
flag = 0
f = open("/home/hduser/swift/swift/proxy/controllers/spindowndevices",
"r")
sdlist = f.read().split("\n")
print("===Spun down devices===:", sdlist)
f.close()
upnodes = [item for item in nodes if item['device'] not in sdlist]
downnodes = [item for item in nodes if item['device'] in sdlist]
temp_nodes = upnodes
if (len(downnodes) > 0):
d = ast.literal_eval(
open("/home/hduser/swift/swift/proxy/controllers/nodes.txt",
"r").read())
# d_temp=pickle.load("/home/hduser/swift/proxy/controllers/nodes.p","rb")
# print("===Current dict===:",d)
for item in downnodes:
if (partition in d):
d[partition].append(item)
# print("===Modified dict===:",d)
else:
d[partition] = [item]
# print("===Modified dict===:",d)
# pickle.dump(d,open("/home/hduser/nodes.p","wb"))
# print("Before writing:",d)
fo = open("/home/hduser/swift/swift/proxy/controllers/nodes.txt", "w")
fo.write(str(d) + "\n")
fo.close()
# pickle.dump(d,open("/home/hduser/swift/swift/proxy/controllers/nodes.p","wb"))
## Old method, IGNORE
# for item in nodes:
# device = item['device']
# if(device not in sdlist):
# # if(os.path.ismount("path"))
# temp_nodes.append(item)
# flag = 1
# break
# else:
# pickle.dump(d,open("/home/hduser/nodes.p","wb"))
# # d = pickle.load(open("/home/hduser/nodes.p","rb"))
# import ast
# d = ast.literal_eval(open("/home/hduser/nodes.txt","r").read())
# print("===Current dict===:",d)
# if(partition in d):
# print("In IF")
# d[partition].append(item)
# print("===Modified dict===:",d)
# else:
# print("In ELSE")
# d[partition] = [item]
# print("===Modified dict===:",d)
# pickle.dump(d,open("/home/hduser/nodes.p","wb"))
# fo = open("/home/hduser/nodes.txt","w")
# fo.write(str(d)+"\n")
# Code to spin up a device if none are running already.
if (len(upnodes) == 0):
dev = nodes[0]['device']
print("===ALL NODES DOWN===")
print("===Mounting device===", dev)
os.system("mount /dev/" + str(dev))
print('===In controller PUT===:')
print("===Partition===", partition)
nodes = temp_nodes
print('===In controller PUT===:')
print("===Partition===", partition)
nodes = temp_nodes
print("===Nodes===:", nodes)
check_ssd()
############################################ CHANGED_CODE ########################################################
# do a HEAD request for checking object versions
if object_versions and not req.environ.get('swift_versioned_copy'):
# make sure proxy-server uses the right policy index
_headers = {
'X-Backend-Storage-Policy-Index': policy_index,
'X-Newest': 'True'
}
hreq = Request.blank(req.path_info,
headers=_headers,
environ={'REQUEST_METHOD': 'HEAD'})
hresp = self.GETorHEAD_base(hreq, _('Object'), obj_ring, partition,
hreq.swift_entity_path)
# Used by container sync feature
if 'x-timestamp' in req.headers:
try:
req_timestamp = Timestamp(req.headers['X-Timestamp'])
except ValueError:
return HTTPBadRequest(
request=req,
content_type='text/plain',
body='X-Timestamp should be a UNIX timestamp float value; '
'was %r' % req.headers['x-timestamp'])
req.headers['X-Timestamp'] = req_timestamp.internal
else:
req.headers['X-Timestamp'] = Timestamp(time.time()).internal
if object_versions and not req.environ.get('swift_versioned_copy'):
is_manifest = 'X-Object-Manifest' in req.headers or \
'X-Object-Manifest' in hresp.headers
if hresp.status_int != HTTP_NOT_FOUND and not is_manifest:
# This is a version manifest and needs to be handled
# differently. First copy the existing data to a new object,
# then write the data from this request to the version manifest
# object.
lcontainer = object_versions.split('/')[0]
prefix_len = '%03x' % len(self.object_name)
lprefix = prefix_len + self.object_name + '/'
ts_source = hresp.environ.get('swift_x_timestamp')
if ts_source is None:
ts_source = time.mktime(
time.strptime(hresp.headers['last-modified'],
'%a, %d %b %Y %H:%M:%S GMT'))
new_ts = Timestamp(ts_source).internal
vers_obj_name = lprefix + new_ts
copy_headers = {
'Destination': '%s/%s' % (lcontainer, vers_obj_name)
}
copy_environ = {
'REQUEST_METHOD': 'COPY',
'swift_versioned_copy': True
}
copy_req = Request.blank(req.path_info,
headers=copy_headers,
environ=copy_environ)
copy_resp = self.COPY(copy_req)
if is_client_error(copy_resp.status_int):
# missing container or bad permissions
return HTTPPreconditionFailed(request=req)
elif not is_success(copy_resp.status_int):
# could not copy the data, bail
return HTTPServiceUnavailable(request=req)
reader = req.environ['wsgi.input'].read
data_source = iter(lambda: reader(self.app.client_chunk_size), '')
source_header = req.headers.get('X-Copy-From')
source_resp = None
if source_header:
if req.environ.get('swift.orig_req_method', req.method) != 'POST':
req.environ.setdefault('swift.log_info', []).append(
'x-copy-from:%s' % source_header)
ver, acct, _rest = req.split_path(2, 3, True)
src_account_name = req.headers.get('X-Copy-From-Account', None)
if src_account_name:
src_account_name = check_account_format(req, src_account_name)
else:
src_account_name = acct
src_container_name, src_obj_name = check_copy_from_header(req)
source_header = '/%s/%s/%s/%s' % (ver, src_account_name,
src_container_name, src_obj_name)
source_req = req.copy_get()
# make sure the source request uses it's container_info
source_req.headers.pop('X-Backend-Storage-Policy-Index', None)
source_req.path_info = source_header
source_req.headers['X-Newest'] = 'true'
orig_obj_name = self.object_name
orig_container_name = self.container_name
orig_account_name = self.account_name
self.object_name = src_obj_name
self.container_name = src_container_name
self.account_name = src_account_name
sink_req = Request.blank(req.path_info,
environ=req.environ,
headers=req.headers)
source_resp = self.GET(source_req)
# This gives middlewares a way to change the source; for example,
# this lets you COPY a SLO manifest and have the new object be the
# concatenation of the segments (like what a GET request gives
# the client), not a copy of the manifest file.
hook = req.environ.get(
'swift.copy_hook',
(lambda source_req, source_resp, sink_req: source_resp))
source_resp = hook(source_req, source_resp, sink_req)
if source_resp.status_int >= HTTP_MULTIPLE_CHOICES:
return source_resp
self.object_name = orig_obj_name
self.container_name = orig_container_name
self.account_name = orig_account_name
data_source = iter(source_resp.app_iter)
sink_req.content_length = source_resp.content_length
if sink_req.content_length is None:
# This indicates a transfer-encoding: chunked source object,
# which currently only happens because there are more than
# CONTAINER_LISTING_LIMIT segments in a segmented object. In
# this case, we're going to refuse to do the server-side copy.
return HTTPRequestEntityTooLarge(request=req)
if sink_req.content_length > constraints.MAX_FILE_SIZE:
return HTTPRequestEntityTooLarge(request=req)
sink_req.etag = source_resp.etag
# we no longer need the X-Copy-From header
del sink_req.headers['X-Copy-From']
if 'X-Copy-From-Account' in sink_req.headers:
del sink_req.headers['X-Copy-From-Account']
if not content_type_manually_set:
sink_req.headers['Content-Type'] = \
source_resp.headers['Content-Type']
if config_true_value(
sink_req.headers.get('x-fresh-metadata', 'false')):
# post-as-copy: ignore new sysmeta, copy existing sysmeta
condition = lambda k: is_sys_meta('object', k)
remove_items(sink_req.headers, condition)
copy_header_subset(source_resp, sink_req, condition)
else:
# copy/update existing sysmeta and user meta
copy_headers_into(source_resp, sink_req)
copy_headers_into(req, sink_req)
# copy over x-static-large-object for POSTs and manifest copies
if 'X-Static-Large-Object' in source_resp.headers and \
req.params.get('multipart-manifest') == 'get':
sink_req.headers['X-Static-Large-Object'] = \
source_resp.headers['X-Static-Large-Object']
req = sink_req
req, delete_at_container, delete_at_part, \
delete_at_nodes = self._config_obj_expiration(req)
node_iter = GreenthreadSafeIterator(
self.iter_nodes_local_first(obj_ring, partition))
pile = GreenPile(len(nodes))
te = req.headers.get('transfer-encoding', '')
chunked = ('chunked' in te)
outgoing_headers = self._backend_requests(
req, len(nodes), container_partition, containers,
delete_at_container, delete_at_part, delete_at_nodes)
for nheaders in outgoing_headers:
# RFC2616:8.2.3 disallows 100-continue without a body
if (req.content_length > 0) or chunked:
nheaders['Expect'] = '100-continue'
################################# CHANGED_CODE ###################################################################
# Replaced node_iter by nodes in the following line to make sure that a new list with different order isnt used.
# Change from node_iter to nodes to make sure it writes to the same device.
# Without this, it gets a new list of nodes from the ring in a different order and connects to the first one.
pile.spawn(self._connect_put_node, nodes, partition,
req.swift_entity_path, nheaders,
self.app.logger.thread_locals)
################################# CHANGED_CODE ###################################################################
conns = [conn for conn in pile if conn]
min_conns = quorum_size(len(nodes))
if req.if_none_match is not None and '*' in req.if_none_match:
statuses = [conn.resp.status for conn in conns if conn.resp]
if HTTP_PRECONDITION_FAILED in statuses:
# If we find any copy of the file, it shouldn't be uploaded
self.app.logger.debug(
_('Object PUT returning 412, %(statuses)r'),
{'statuses': statuses})
return HTTPPreconditionFailed(request=req)
if any(conn for conn in conns
if conn.resp and conn.resp.status == HTTP_CONFLICT):
timestamps = [
HeaderKeyDict(
conn.resp.getheaders()).get('X-Backend-Timestamp')
for conn in conns if conn.resp
]
self.app.logger.debug(
_('Object PUT returning 202 for 409: '
'%(req_timestamp)s <= %(timestamps)r'), {
'req_timestamp': req.timestamp.internal,
'timestamps': ', '.join(timestamps)
})
return HTTPAccepted(request=req)
if len(conns) < min_conns:
self.app.logger.error(
_('Object PUT returning 503, %(conns)s/%(nodes)s '
'required connections'), {
'conns': len(conns),
'nodes': min_conns
})
return HTTPServiceUnavailable(request=req)
bytes_transferred = 0
try:
with ContextPool(len(nodes)) as pool:
for conn in conns:
conn.failed = False
conn.queue = Queue(self.app.put_queue_depth)
pool.spawn(self._send_file, conn, req.path)
while True:
with ChunkReadTimeout(self.app.client_timeout):
try:
chunk = next(data_source)
except StopIteration:
if chunked:
for conn in conns:
conn.queue.put('0\r\n\r\n')
break
bytes_transferred += len(chunk)
if bytes_transferred > constraints.MAX_FILE_SIZE:
return HTTPRequestEntityTooLarge(request=req)
for conn in list(conns):
if not conn.failed:
conn.queue.put('%x\r\n%s\r\n' %
(len(chunk),
chunk) if chunked else chunk)
else:
conns.remove(conn)
if len(conns) < min_conns:
self.app.logger.error(
_('Object PUT exceptions during'
' send, %(conns)s/%(nodes)s required connections'
), {
'conns': len(conns),
'nodes': min_conns
})
return HTTPServiceUnavailable(request=req)
for conn in conns:
if conn.queue.unfinished_tasks:
conn.queue.join()
conns = [conn for conn in conns if not conn.failed]
except ChunkReadTimeout as err:
self.app.logger.warn(_('ERROR Client read timeout (%ss)'),
err.seconds)
self.app.logger.increment('client_timeouts')
return HTTPRequestTimeout(request=req)
except (Exception, Timeout):
self.app.logger.exception(
_('ERROR Exception causing client disconnect'))
return HTTPClientDisconnect(request=req)
if req.content_length and bytes_transferred < req.content_length:
req.client_disconnect = True
self.app.logger.warn(
_('Client disconnected without sending enough data'))
self.app.logger.increment('client_disconnects')
return HTTPClientDisconnect(request=req)
statuses, reasons, bodies, etags = self._get_put_responses(
req, conns, nodes)
if len(etags) > 1:
self.app.logger.error(
_('Object servers returned %s mismatched etags'), len(etags))
return HTTPServerError(request=req)
etag = etags.pop() if len(etags) else None
resp = self.best_response(req,
statuses,
reasons,
bodies,
_('Object PUT'),
etag=etag)
if source_header:
acct, path = source_header.split('/', 3)[2:4]
resp.headers['X-Copied-From-Account'] = quote(acct)
resp.headers['X-Copied-From'] = quote(path)
if 'last-modified' in source_resp.headers:
resp.headers['X-Copied-From-Last-Modified'] = \
source_resp.headers['last-modified']
copy_headers_into(req, resp)
resp.last_modified = math.ceil(
float(Timestamp(req.headers['X-Timestamp'])))
return resp
