def compile(l):
ts = current_milli_time()
d = {
"rootDoc_id": l.websocket.root_folder['_id'],
"draft": False,
"_csrf": l.csrf_token
}
r1 = l.client.post("/project/%s/compile" % l.project_id,
json=d,
name="compile")
resp = r1.json()
if resp["status"] == "too-recently-compiled":
return
files = resp["outputFiles"]
# l.client.get("/project/%s/output/output.log" % l.project_id,
# params={"build": files[0]["build"]},
# name="get_compile_log")
l.client.get("/project/%s/output/output.pdf" % l.project_id,
params={
"build": files[0]["build"],
"compileGroup": "standard",
"pdfng": True
},
name="get_compile_pdf")
te = current_milli_time()
request_success.fire(request_type='GET',
name="full_compile",
response_time=(te - ts),
response_length=0)
def sendMessage(self):
ws = create_connection("wss://echo.websocket.org",
sslopt={"cert_reqs": ssl.CERT_NONE})
user_id = six.text_type(uuid.uuid4())
start_at = time.time()
json_val = {
'id':user_id,
'salutation': "Hello",
'name': "James",
'start_at': start_at
}
msg = json.dumps(json_val)
ws.send(msg)
print(f"Sent {msg}")
res = ws.recv()
data = json.loads(res)
end_at = time.time()
response_time = int((end_at - data['start_at']) * 1000)
print(f"Received {res}")
print(f"Got back id {data['id']}")
request_success.fire(
request_type='WebSocket Response',
name='ws',
response_time=response_time,
response_length=len(res),
)
graphyte.send('websocket.response_time',response_time)
def my_task(self):
# this is just an example, but it shows off some of the things you might want to do in a Webdriver test
self.client.delete_all_cookies()
customer = CUSTOMER_READER.get()
CUSTOMER_READER.release(customer)
start_at = time.time()
self.client.get("https://example.com/")
try:
self.client.find_element_by_css_selector(".btn-inverted").click()
except NoSuchElementException:
pass # dont really care about this
self.client.find_element_by_css_selector(".js-nav-list-item-link-login").click()
time.sleep(0.1)
self.client.find_element_by_xpath('//*[@id="bankidTab"]/button[1]').click()
time.sleep(0.1)
self.client.find_element_by_xpath('//*[@id="ssn"]')
ssn = customer["ssn"]
ssn_input = self.client.switch_to.active_element
# Sometimes send_keys has issues due to client side javascript. This is a workaround.
self.client.execute_script(
"arguments[0].setAttribute('value', arguments[1])", ssn_input, ssn[:8] + "-" + ssn[-4:]
)
ssn_input.send_keys(Keys.RETURN)
try:
self.client.find_element_by_xpath('//*[@id="last-login-time"]/div/div[4]/a/span').click()
except StaleElementReferenceException:
# retry...
self.client.find_element_by_xpath('//*[@id="last-login-time"]/div/div[4]/a/span').click()
time.sleep(0.1)
request_success.fire(
request_type="Selenium", name="Logged in", response_time=(time.time() - start_at) * 1000, response_length=0
)
def run(self):
log.info("Starting MobileStreamingUpdateProcessor connecting to uri: " + self._uri)
self._running = True
while self._running:
try:
init_start = time.time()
messages = self._connect()
for msg in messages:
if not self._running:
break
message_ok = self.process_message(self._store, self._requester, msg)
if message_ok is True and self._ready.is_set() is False:
log.info("MobileStreamingUpdateProcessor initialized ok.")
init_duration = int((time.time() - init_start) * 1000)
request_success.fire(request_type="ld:init", name="mobile", response_time=init_duration, response_length=0)
self._ready.set()
except UnsuccessfulResponseException as e:
log.error(http_error_message(e.status, "stream connection"))
init_duration = int((time.time() - init_start) * 1000)
request_failure.fire(request_type="ld:init", name="mobile", response_time=init_duration, response_length=0, exception=e)
if not is_http_error_recoverable(e.status):
self._ready.set() # if client is initializing, make it stop waiting; has no effect if already inited
self.stop()
break
except Exception as e:
init_duration = int((time.time() - init_start) * 1000)
request_failure.fire(request_type="ld:init", name="mobile", response_time=init_duration, response_length=0, exception=e)
log.warning("Caught exception. Restarting stream connection after one second. %s" % e)
# no stacktrace here because, for a typical connection error, it'll just be a lengthy tour of urllib3 internals
time.sleep(1)
def on_start(self):
url = 'http://stream.moneyball.com/connect'
start_at = time.time()
response = self.with_requests(url) # or with_requests(url)
client = sseclient.SSEClient(response)
for event in client.events():
start_at_click = time.time()
end_at = time.time()
data = json.loads(event.data)
pprint.pprint(data)
if 'lc' in data:
response_time = int((end_at - start_at) * 1000)
request_success.fire(
request_type='Connection Received',
name='test/sse/connect',
response_time=response_time,
response_length=len(event.data),
)
elif 'ts' in data:
response_time_click = int(
(float(data['ts']) - start_at_click * 1000) * 1000)
request_success.fire(
request_type='Click received',
name='sse/clickEvent',
response_time=response_time_click,
response_length=len(event.data),
)
def on_receive_publish(c, u, m):
elapsed = time.time() - float(m.payload)
request_success.fire(
request_type='Recv Publish',
name=m.topic,
response_time=int((elapsed if elapsed >= 0 else 0) * 1000),
response_length=len(m.payload),
)
def _receive():
sub_mapping = {}
while True:
start_time = time.time()
try:
res = ws.recv()
except:
log.exception("Error receiving message")
break
message = json.loads(res)
try:
sub_id = message["id"]
except KeyError:
pass
else:
try:
sub_mapping[message["result"]] = sub_id
continue
except KeyError:
pass
# except TypeError:
# print(sub_mapping)
# print(message)
# print(sub_id)
end_time = time.time()
# if output:
# print(message)
try:
message["params"]["subscription"] = sub_mapping[
message["params"]["subscription"]]
except KeyError:
log.warning("Mapping missing for %s",
message["params"]["subscription"])
end_at = time.time()
response_time = int(round((end_time - start_time) * 1000))
num = len(self.sends) // 10
request_failure.fire(request_type='WebSocket Recv',
name=f'gotsomething{num}',
response_time=1,
response_length=len(res),
exception="Mapping missing for %s" %
message["params"]["subscription"])
end_at = time.time()
num = len(self.sends) // 10
# print("recv")
# print(message)
# if num == 0:
# print(len(self.sends))
# print(self.sends)
response_time = int(round((end_time - start_time) * 1000))
request_success.fire(
request_type='WebSocket Recv',
name=f'gotsomething{num}',
response_time=response_time,
response_length=len(res),
)
def _send(self, pkt):
start_at = time.time()
msg = encode(pkt)
debug(">> " + msg)
self.ws.send(msg)
request_success.fire(
request_type='WebSocketSent',
name="socket.io/%s#%s" % (pkt.get("name", ""), pkt["type"]),
response_time=int((time.time() - start_at) * 1000000),
response_length=len(msg))
def sent(self):
start_at = time.time()
# body = json.dumps({'message': 'hello, world', 'user_id': self.user_id, 'start_at': start_at})
self.ws.send(command_data)
request_success.fire(
request_type='WebSocketSent',
name='test/ws/echo',
response_time=int((time.time() - start_at) * 1000000),
response_length=len(command_data),
)
def echo(self):
start_at = time.time()
self.sock.send(b'Hello world!!')
res = self.sock.recv(self.bufsize)
end_at = time.time()
request_success.fire(
request_type='TCP Recv',
name='test/echo',
response_time=int((end_at - start_at) * 1000000),
response_length=len(res),
)
def urlopen(self, method, url, redirect=True, **kw):
is_stream = not kw.get('preload_content', True)
start_time = time.time()
x = urlparse(url)
name = url
req_type = method
if method == 'GET' and (x.path.find('/meval/') == 0
or x.path.find('/users/') > -1
or x.path.find('/eval/') == 0):
parts = x.path.split('/')
parts.pop()
parts.append('[user]')
newpath = '/'.join(parts)
x = x._replace(path=newpath)
name = urlunparse(x)
if is_stream and kw.get('headers', {}).get(
'Accept', 'lol') == 'text/event-stream':
req_type = 'sse:connect'
resp = None
content_len = 0
try:
resp = super(LocustPoolManager,
self).urlopen(method, url, redirect, **kw)
if is_stream:
content_len = int(resp.headers.get("content-length") or 0)
else:
content_len = len(resp.data or b"")
throw_if_unsuccessful_response(resp)
except UnsuccessfulResponseException as e:
request_failure.fire(request_type=req_type,
name=name,
exception=e,
response_length=content_len,
response_time=int(
(time.time() - start_time) * 1000))
return resp
except Exception as e:
request_failure.fire(request_type=req_type,
name=name,
exception=e,
response_length=content_len,
response_time=int(
(time.time() - start_time) * 1000))
raise e
request_success.fire(request_type=req_type,
name=name,
response_length=content_len,
response_time=int(
(time.time() - start_time) * 1000))
return resp
def _receive():
while True:
res = ws.recv()
data = json.loads(res)
end_at = time.time()
response_time = int((end_at - data['start_at']) * 1000000)
request_success.fire(
request_type='WebSocket Recv',
name='test/ws/chat',
response_time=response_time,
response_length=len(res),
)
def on_chat(self, args):
rec_ts = current_milli_time()
if 'client_ts' in args[
0] and self.l.parent.email != args[0]['user']['email']:
request_success.fire(request_type='WebSocket',
name="receive_chat_message",
response_time=rec_ts -
int(args[0]['client_ts']),
response_length=0)
# print('user %s received chat from %s' % (self.l.parent.email, args[0]['user']['email']))
pass
def _recv(self):
start_at = time.time()
res = self.ws.recv()
debug("<< " + res)
data = decode(res)
name = data.get("name", "")
request_success.fire(request_type='WebSocketRecv',
name="socket.io/%s#%s" % (name, data["type"]),
response_time=int(
(time.time() - start_at) * 1000000),
response_length=len(res))
return data
def _receive():
while True:
res = self.ws.recv()
data = json.loads(res.decode())
end_at = time.time()
response_time = int(round(end_at - data['start_at'], 3) * 1000)
request_success.fire(
request_type='WebSocketTest',
name='recv',
response_time=response_time,
response_length=len(res),
)
def on_update_position(self, args):
rec_ts = current_milli_time()
if 'client_ts' in args[0] and self.l.parent.email != args[0]['email']:
request_success.fire(request_type='WebSocket',
name="update_cursor_position",
response_time=rec_ts - args[0]['client_ts'],
response_length=0)
# print('user %s saw user %s moving at [%s:%s]' % (self.l.parent.email, args[0]['email'], args[0]['row'], args[0]['column']))
global pos_registered
pos_registered += 1
pass
def sent(self):
start_at = time.time()
body = json.dumps({
'message': 'hello, world',
'connection_id': self.connection_id,
'start_at': start_at
})
self.ws.send_binary(body.encode())
request_success.fire(
request_type='WebSocketTest',
name='send',
response_time=int(round(time.time() - start_at, 3) * 1000),
response_length=len(body),
)
def send(self, body):
if body == "2":
action = "2 heartbeat"
else:
m = re.search(r'(\d*)\["([a-z]*)"', body)
code = m.group(1)
action = m.group(2)
url_part = re.search(r'"url": *"([^"]*)"', body)
url = re.sub(r"/[0-9_]*/", "/:id/", url_part.group(1))
action = f"{code} {action} url: {url}"
request_success.fire(request_type="WSS", name=action, response_time=None, response_length=len(body))
logging.debug(f"WSS: {body}")
self.ws.send(body)
def receive(self):
while True:
start_at = time.time()
res = self.ws.recv()
data = json.loads(res)
print(data)
end_at = time.time()
response_time = int((end_at - start_at) * 1000000)
request_success.fire(
request_type='WebSocket Recv',
name='test/ws/echo',
response_time=response_time,
response_length=len(res),
)
def update_version(self, args):
rec_ts = current_milli_time()
if 'client_ts' in args[0] and self.sent_doc_version != args[0]["v"]:
request_success.fire(request_type='WebSocket',
name="update_text",
response_time=rec_ts - args[0]['client_ts'],
response_length=0)
# print("update in %s ms" % str(rec_ts - args[0]['client_ts']))
# if self.sent_doc_version != args[0]["v"]:
# print('user %s saw an update it didn\'t emit' % (self.l.parent.email))
self.doc_version = args[0]["v"] + 1
if self.pending_text is not None:
self.doc_text = self.pending_text
self.pending_text = None
def reg(self, reconnect):
try:
self.ws.send('Reg:loadroom:%s' % self.user_id)
regname = 'Sent Reg'
if reconnect is True:
regname = 'Sent Retry Reg'
request_success.fire(
request_type='WebSocket Sent',
name=regname,
response_time=1,
response_length=1,
)
except WebSocketException:
self.ws = None
except:
p.ws = None
def _receive2():
while True:
try:
res = ws2.recv()
except:
log.exception("Error receiving message")
break
end_time = time.time()
response_time = int(
round((end_time - self.check_start_time) * 1000))
request_success.fire(
request_type='WebSocket Recv',
name=f'eth_blockNumberRPC',
response_time=response_time,
response_length=len(res),
)
def my_task(self):
short_sleep = 1 if __name__ == "__main__" else 0.1
# this is just an example, but it shows off some of the things you might want to do in a Webdriver test
self.client.delete_all_cookies()
customer = customer_reader.get()
start_at = time.time()
self.client.get(self.locust.host)
try:
self.client.find_element_by_css_selector(".btn-inverted").click()
except NoSuchElementException:
pass # dont really care about this
self.client.find_element_by_css_selector(
".js-nav-list-item-link-login").click()
time.sleep(short_sleep)
self.client.find_element_by_xpath(
'//*[@id="bankidTab"]/button[1]').click()
time.sleep(short_sleep)
self.client.find_element_by_xpath('//*[@id="ssn"]')
ssn = customer["ssn"]
ssn_input = self.client.switch_to.active_element
# Sometimes send_keys has issues due to client side javascript. This is a workaround.
self.client.execute_script(
"arguments[0].setAttribute('value', arguments[1])", ssn_input,
ssn[:8] + "-" + ssn[-4:])
time.sleep(short_sleep)
ssn_input.send_keys(Keys.RETURN)
time.sleep(short_sleep)
try:
self.client.find_element_by_xpath(
'//*[@id="last-login-time"]/div/div[4]/a/span').click()
except StaleElementReferenceException:
# retry...
self.client.find_element_by_xpath(
'//*[@id="last-login-time"]/div/div[4]/a/span').click()
# typically you would release the customer only after its task has finished,
# but in this case I dont care, and dont want to block another test run from using it
# (particularly if this test crashes)
customer_reader.release(customer)
request_success.fire(request_type="Selenium",
name="Log in",
response_time=(time.time() - start_at) * 1000,
response_length=0)
time.sleep(short_sleep * 2)
def receive(self):
message_regex = re.compile(r"(\d*)(.*)")
description_regex = re.compile(r"<([0-9]+)>$")
response_time = None
while True:
message = self.ws.recv()
logging.debug(f"WSR: {message}")
m = message_regex.match(message)
code = m.group(1)
json_string = m.group(2)
if code == "0":
name = "0 open"
elif code == "3":
name = "3 heartbeat"
elif code == "40":
name = "40 message ok"
elif code == "42":
# this is rather specific to our use case. Some messages contain an originating timestamp,
# and we use that to calculate the delay & report it as locust response time
# see it as inspiration rather than something you just pick up and use
obj = json.loads(json_string)
name = f"{code} {obj[0]} apiUri: {obj[1]['apiUri']}"
if obj[1]["value"] != "":
description = obj[1]["value"]["draw"]["description"]
description_match = description_regex.search(description)
if description_match:
sent_timestamp = int(description_match.group(1))
current_timestamp = int(round(time.time() * 1000))
response_time = current_timestamp - sent_timestamp
else:
# differentiate samples that have no timestamps from ones that do
name += "_"
else:
name += "_missingTimestamp"
else:
print(f"Received unexpected message: {message}")
continue
# Locust mail fail if no response time was detected (response_time = None)
# There is a small locust patch in my fork, install it like this:
# pip install -e git+https://github.com/cyberw/locust.git@allow-samples-with-None-response-time#egg=locustio
request_success.fire(request_type="WSR",
name=name,
response_time=response_time,
response_length=len(message))
def parse(cls, raw, instance=None):
"""
Given a possibly-multiline string representing an SSE message, parse it
and return a Event object.
"""
msg = cls()
for line in raw.split('\n'):
m = cls.sse_line_pattern.match(line)
if m is None:
# Malformed line. Discard but warn.
log.warning('Invalid SSE line: "%s"' % line)
continue
name = m.groupdict()['name']
value = m.groupdict()['value']
if name == '':
if instance is not None:
now = time.time()
last_heartbeat = instance._last_heartbeat or instance._connect_start
duration = int((now - last_heartbeat) * 1000)
request_success.fire(request_type='sse:heartbeat',
name=clean_name('GET', instance.url),
response_time=duration,
response_length=0)
instance._last_heartbeat = now
# line began with a ":", so is a comment. Ignore
continue
if name == 'data':
# If we already have some data, then join to it with a newline.
# Else this is it.
if msg.data:
msg.data = '%s\n%s' % (msg.data, value)
else:
msg.data = value
elif name == 'event':
msg.event = value
elif name == 'id':
msg.id = value
elif name == 'retry':
msg.retry = int(value)
return msg
def urlopen(self, method, url, redirect=True, **kw):
is_stream = not kw.get('preload_content', True)
start_time = time.time()
name = clean_name(method, url)
req_type = method
if is_stream and kw.get('headers', {}).get(
'Accept', 'lol') == 'text/event-stream':
req_type = 'sse:connect'
resp = None
content_len = 0
try:
resp = super(LocustProxyPoolManager,
self).urlopen(method, url, redirect, **kw)
if is_stream:
content_len = int(resp.headers.get("content-length") or 0)
else:
content_len = len(resp.data or b"")
throw_if_unsuccessful_response(resp)
except UnsuccessfulResponseException as e:
request_failure.fire(request_type=req_type,
name=name,
exception=e,
response_length=content_len,
response_time=int(
(time.time() - start_time) * 1000))
return resp
except Exception as e:
request_failure.fire(request_type=req_type,
name=name,
exception=e,
response_length=content_len,
response_time=int(
(time.time() - start_time) * 1000))
raise e
request_success.fire(request_type=req_type,
name=name,
response_length=content_len,
response_time=int(
(time.time() - start_time) * 1000))
return resp
def process_message(store, requester, msg):
if msg.event == 'put':
all_data = json.loads(msg.data)
for k,v in all_data.items():
v['key'] = k
init_data = {
FEATURES: all_data
}
log.debug("Received put event with %d flags",
len(init_data[FEATURES]))
store.init(init_data)
return True
elif msg.event == 'patch':
recv_time = time.time() * 1000
payload = json.loads(msg.data)
if payload.get('key') == 'locust-heartbeat':
value = int(payload.get('value') or 0)
duration = int((recv_time - value))
request_success.fire(request_type='sse:flag-update', name='/meval', response_time=duration, response_length=0)
log.debug("Received patch event for %s, New version: [%d]", payload.get('key'), payload.get("version"))
store.upsert(FEATURES, payload)
elif msg.event == 'ping':
log.debug('Received ping event')
all_data = requester.get_all_data()
store.init(all_data)
log.debug("Received flags after ping event with %d flags", len(all_data[FEATURES]))
return True
elif msg.event == 'delete':
payload = json.loads(msg.data)
key = payload.get('key')
# noinspection PyShadowingNames
version = payload['version']
log.debug("Received delete event for %s, New version: [%d]", key, version)
target = ParsedPath(kind = FEATURES, key = key)
store.delete(target.kind, target.key, version)
else:
log.warning('Unhandled event in stream processor: ' + msg.event)
return False
def sent(self):
try:
start_at = time.time()
body = json.dumps({
'message': randomword(255),
'user_id': self.user_id,
'start_at': start_at
})
msg = 'Say:loadroom:msg:%s' % body
self.ws.send(msg)
request_success.fire(
request_type='WebSocket Sent',
name='Sent Say',
response_time=int((time.time() - start_at) * 1000000),
response_length=len(body),
)
except WebSocketException:
self.ws = None
except:
self.ws = None
def _receive(p):
while p.run:
try:
res = ws.recv()
data = json.loads(res)
if data['Type'] == 'quit':
p.run = False
# end_at = time.time()
# response_time = int((end_at - data['start_at']) * 1000000)
request_success.fire(
request_type=data['Type'],
name=data['Name'],
# response_time=response_time,
response_time=1,
# response_length=len(res),
response_length=1,
)
except:
logger.info("ws recv error")
break
def put_record(self):
start_at = time.time()
print(start_at)
try:
res = self.conn.put_record(stream_name="test-stream",
data="test",
partition_key="partition_key1")
print(res)
request_success.fire(
request_type='Kinesis put record',
name='test-stream',
response_time=int((time.time() - start_at) * 1000),
response_length=len(json.dumps(res)),
)
except Exception as e:
print(e)
request_failure.fire(request_type='Kinesis put record',
name='test-stream',
response_time=int(
(time.time() - start_at) * 1000),
exception=e)
def handle_message(self, msg, **kwargs):
init_time = kwargs.get('init_time', time.time())
if "CONNECTED" in msg:
logging.info("%s: Connection initiated", self.instanceId)
request_success.fire(request_type='WebSocket Init',
name='Init websocket connection',
response_time=(time.time() - init_time) *
1000,
response_length=len(msg))
elif "ERROR" in msg:
logging.warning("%s: Received error: \n %s", self.instanceId,
str(msg))
request_failure.fire(request_type='WebSocket Error',
name='Websocket server Error',
response_time=(time.time() - init_time) *
1000,
response_length=len(msg),
exception=InterruptTaskSet())
raise StopLocust()
elif "MESSAGE" in msg:
logging.info("%s: Message received: %s", self.instanceId, str(msg))
message = str(msg)
idx = message.find("{")
content = message[idx:-1]
json_msg = json.loads(content)
res = stomper.send(
"/app/device",
'{ "time": "' + datetime.datetime.now().isoformat() + '",' +
'"status": "OK",' + '"id": "' + json_msg["id"] + '",' +
'"data": ' + str([1, 0, -1, 0]) + '}',
content_type="application/json")
self.ws.send(res)
request_success.fire(request_type='Msg Receive',
name='Msg received',
response_time=(time.time() - init_time) *
1000,
response_length=len(msg))
def _receive(p):
while True:
if p.ws is None:
logger.info("ws reconnect happen")
p.create_new_ws(True)
try:
res = p.ws.recv()
except WebSocketException:
p.ws = None
except:
p.ws = None
if res.startswith('msg'):
jres = res.split('msg:')[1]
data = json.loads(jres)
end_at = time.time()
response_time = int((end_at - data['start_at']) * 1000000)
request_success.fire(
request_type='WebSocket Recv',
name='Recv Say',
response_time=response_time,
response_length=len(jres),
)
elif res.startswith('Room') or res.startswith('UnReg'):
request_success.fire(
request_type='WebSocket Recv',
name='Recv Someone Joinin/UnReg',
response_time=1,
response_length=1,
)
else:
request_success.fire(
request_type='WebSocket Recv',
name='Recv Unknown',
response_time=1,
response_length=1,
)
def send_kinesis_success_fire(time):
request_success.fire(request_type='PUT',
name='Kinesis',
response_time=time,
response_length=0)