def test_can_get_with_auto_start(self):
otto = Octopus(concurrency=1, auto_start=True)
def handle_url_response(url, response):
self.response = response
otto.enqueue('http://www.twitter.com', handle_url_response)
otto.wait(5)
expect(self.response).not_to_be_null()
expect(self.response.status_code).to_equal(200)
def test_can_handle_cached_responses(self):
response = Mock(status_code=200, body="whatever")
url = 'http://www.google.com'
otto = Octopus(concurrency=1, cache=True)
otto.response_cache.put(url, response)
def handle_url_response(url, response):
self.response = response
otto.enqueue(url, handle_url_response)
expect(self.response).not_to_be_null()
expect(self.response.status_code).to_equal(200)
expect(self.response.body).to_equal("whatever")
def otto_cached_requests(repetitions, concurrency, urls_to_retrieve):
message = "Retrieving URLs concurrently with Octopus with caching enabled..."
print
print("=" * len(message))
print(message)
print("=" * len(message))
print
otto = Octopus(concurrency=concurrency, cache=True, auto_start=True)
for url in urls_to_retrieve:
otto.enqueue(url, handle_url_response)
start_time = time()
otto.wait(0)
return time() - start_time
def test_can_handle_cached_responses_when_not_cached(self):
url = 'http://www.twitter.com'
otto = Octopus(concurrency=1, cache=True)
def handle_url_response(url, response):
self.response = response
otto.enqueue(url, handle_url_response)
otto.enqueue(url, handle_url_response)
otto.enqueue(url, handle_url_response)
otto.enqueue(url, handle_url_response)
otto.start()
otto.wait(5)
expect(self.response).not_to_be_null()
expect(self.response.status_code).to_equal(200)
def main_octopus():
from octopus import Octopus
from ase.build import molecule
system = molecule('H2')
system.center(vacuum=1.5)
system.pbc = 1
calc = Octopus()
system.set_calculator(calc)
system.get_potential_energy()
check_interface(calc)
def test_can_wait(self):
otto = Octopus(concurrency=1)
def handle_url_response(url, response):
self.response = response
otto.enqueue('http://www.twitter.com', handle_url_response)
otto.start()
otto.wait(0)
expect(self.response).not_to_be_null()
expect(self.response.status_code).to_equal(200)
def __init__(self):
pygame.init()
self.settings = Settings()
self.screen = pygame.display.set_mode((0, 0), pygame.FULLSCREEN)
self.settings.screen_width = self.screen.get_rect().width
self.settings.screen_height = self.screen.get_rect().height
pygame.display.set_caption("Ocean Adventure")
self.stats = GameStats(self)
self.sb = Scoreboard(self)
self.octopus = Octopus(self)
self.bullets = pygame.sprite.Group()
self.fishes = pygame.sprite.Group()
self._create_fleet()
self.play_button = Button(self, "Play")
self.bg_color = (187, 230, 247)
def test_wait_returns_automatically_when_empty(self):
otto = Octopus(concurrency=1)
otto.start()
otto.wait(5)
expect(otto.is_empty).to_be_true()
def test_can_handle_timeouts(self):
url = 'http://baidu.com'
otto = Octopus(concurrency=1, request_timeout_in_seconds=0.1)
def handle_url_response(url, response):
self.response = response
otto.enqueue(url, handle_url_response)
otto.start()
otto.wait(5)
expect(self.response.text).to_include('Connection to baidu.com timed out')
expect(self.response.error).to_include('Connection to baidu.com timed out. (connect timeout=0.1)')
def otto_requests(repetitions, concurrency, urls_to_retrieve):
message = "Retrieving URLs concurrently with Octopus..."
print
print("=" * len(message))
print(message)
print("=" * len(message))
print
otto = Octopus(concurrency=concurrency)
for url in urls_to_retrieve:
otto.enqueue(url, handle_url_response)
start_time = time()
otto.start()
otto.wait(0)
return time() - start_time
def test_can_handle_invalid_urls(self):
url = 'http://kagdjdkjgka.fk'
otto = Octopus(concurrency=1)
def handle_url_response(url, response):
self.response = response
otto.enqueue(url, handle_url_response)
otto.start()
otto.wait(5)
expect(self.response).not_to_be_null()
expect(self.response.status_code).to_equal(599)
expect(self.response.text).to_include("HTTPConnectionPool(host='kagdjdkjgka.fk', port=80)")
expect(self.response.text).to_include('Max retries exceeded with url: /')
expect(self.response.error).to_equal(self.response.text)
def test_times_out_on_wait(self):
otto = Octopus(concurrency=1)
def handle_url_response(url, response):
self.response = response
otto.enqueue('http://www.google.com', handle_url_response)
try:
otto.wait(0.1)
except TimeoutError:
err = sys.exc_info()[1]
expect(err).to_have_an_error_message_of("")
else:
assert False, "Should not have gotten this far"
def test_can_handle_more_urls_concurrently(self):
urls = [
'http://www.twitter.com',
'http://www.cnn.com',
'http://www.bbc.com',
'http://www.facebook.com'
]
otto = Octopus(concurrency=4)
def handle_url_response(url, response):
self.responses[url] = response
for url in urls:
otto.enqueue(url, handle_url_response)
otto.start()
otto.wait(10)
expect(self.responses).to_length(4)
for url in urls:
expect(self.responses).to_include(url)
expect(self.responses[url].status_code).to_equal(200)
def octopus_create_GET_request(urls):
data = []
otto = Octopus(concurrency=4,
auto_start=True,
cache=True,
expiration_in_seconds=10,
request_timeout_in_seconds=8)
def handle_url_response(url, response):
if "Not found" == response.text:
print("URL Not Found: %s" % url)
else:
data.append(response.text)
for url in urls:
otto.enqueue(url, handle_url_response)
otto.wait(25)
return data
def request(urls):
data = []
otto = Octopus(
concurrency = 4, auto_start = True, cache = True, expiration_in_seconds=10
)
def handle_url_response(url, response):
if "Not found" == response.text:
print("URL Not Found: %s" %url)
else:
data.append(response.text)
for url in urls:
otto.enqueue(url, handle_url_response)
otto.wait()
json_data = json.JSONEncoder(indent = None,
separators = (',', ':')).encode(data)
return pprint(json_data)
def create_request(urls, method, kwargs):
data = []
otto = Octopus(
concurrency=1, auto_start=True, cache=False, expiration_in_seconds=20
)
def handle_url_response(url, response):
if "Not found" == response.text:
print ("URL Not Found: %s" % url)
else:
data.append(response.text)
for url in urls:
otto.enqueue(url, handle_url_response, method, data=kwargs, headers={ "Content-type": "application/x-www-form-urlencoded" })
otto.wait()
json_data = json.JSONEncoder(indent=None,
separators=(',', ': ')).encode(data)
return json_data
def create_request(urls, buscar):
data = []
otto = Octopus(concurrency=10,
auto_start=True,
cache=True,
expiration_in_seconds=30)
def handle_url_response(url, response):
if "Not found" == response.text:
print("URL Not Found: %s" % url)
else:
aux_dict = json.loads(response.text)
if aux_dict["url"][-3] == '/':
data.append((aux_dict[buscar], aux_dict["url"][-2]))
else:
data.append((aux_dict[buscar], aux_dict["url"][-3:-1]))
for url in urls:
otto.enqueue(url, handle_url_response)
otto.wait()
return data
def test_can_create_octopus(self):
otto = Octopus(concurrency=20)
expect(otto.concurrency).to_equal(20)
expect(otto.auto_start).to_be_false()
expect(otto.cache).to_be_false()
] #categories to be tested and saved to file. These can be 'performance', 'seo', 'accessibility', 'best-practices'
threadsNumber = 15 #number of requests sent asynchronically at once
startTime = datetime.datetime.now()
print(f'Starting process - {startTime.strftime("%H:%M")}')
pathlib.Path('./results').mkdir(
parents=True, exist_ok=True) #create results fir if not exists
filePath = f'./results/{projectName}-{strategy} performance-{startTime.strftime("%d-%m-%Y_%H:%M:%S")}.csv'
resultsFile = open(filePath, 'w')
columnNames = "Number, URL, First Contentful Paint, First Interactive, Performance, SEO, Accessibility, Best-Practices\n"
resultsFile.write(columnNames)
# settings for octopus threads
otto = Octopus(concurrency=threadsNumber,
auto_start=True,
cache=True,
request_timeout_in_seconds=30)
def handle_url_response(url, response):
number = url.split('&num=')
splitUrl = url.split('?url=')
reqUrl = splitUrl[1].split('&strategy=')
row = f'{number[1]}'
try:
parsedResponse = json.loads(response.text)
contentfulPaint = parsedResponse['lighthouseResult']['audits'][
'first-contentful-paint']['displayValue']
firstInteractive = parsedResponse['lighthouseResult']['audits'][
'interactive']['displayValue']
performanceScore = parsedResponse['lighthouseResult']['categories'][
#!/usr/bin/env python3
'''
octo.py
sample octopus http for python3 training
at www.jasaplus.com
'''
from octopus import Octopus
otto = Octopus(concurrency=2, auto_start=True)
def handle_url_response(url, response):
print(response.text)
otto.enqueue(
'https://raw.githubusercontent.com/brandiqa/json-examples/master/src/google_markers.json',
handle_url_response)
otto.enqueue(
'https://raw.githubusercontent.com/brandiqa/json-examples/master/src/products.json',
handle_url_response)
otto.wait()
def test_can_enqueue_url(self):
otto = Octopus()
otto.enqueue('http://www.google.com', None)
expect(otto.queue_size).to_equal(1)
def test_queue_is_empty(self):
otto = Octopus()
expect(otto.is_empty).to_be_true()
def test_has_default_concurrency(self):
otto = Octopus()
expect(otto.concurrency).to_equal(10)
def test_should_not_get_more_than_one_url_for_same_domain_concurrently(self):
limiter = PerDomainRedisLimiter(
{'http://g1.globo.com': 1},
{'http://globoesporte.globo.com': 1},
redis=self.redis
)
otto = Octopus(concurrency=10, auto_start=True, limiter=limiter)
otto.enqueue('http://globoesporte.globo.com', self.handle_url_response)
otto.enqueue('http://globoesporte.globo.com/futebol/times/flamengo/', self.handle_url_response)
otto.enqueue('http://g1.globo.com', self.handle_url_response)
otto.enqueue('http://g1.globo.com/economia', self.handle_url_response)
otto.wait(10)
expect(self.responses).to_length(4)
expect(self.redis.zcard('limit-for-http://g1.globo.com')).to_equal(0)
expect(self.redis.zcard('limit-for-http://globoesporte.globo.com')).to_equal(0)
def test_should_call_limiter_miss_twice(self):
limiter = PerDomainInMemoryLimiter(
{'http://g1.globo.com': 1},
{'http://globoesporte.globo.com': 1},
)
limiter.subscribe_to_lock_miss(self.handle_limiter_miss)
otto = Octopus(concurrency=10, auto_start=True, limiter=limiter)
otto.enqueue('http://globoesporte.globo.com/', self.handle_url_response)
otto.enqueue('http://globoesporte.globo.com/futebol/times/flamengo/', self.handle_url_response)
otto.enqueue('http://g1.globo.com/', self.handle_url_response)
otto.enqueue('http://g1.globo.com/economia/', self.handle_url_response)
otto.wait()
expect(self.cache_miss).to_length(2)
class OceanAdventure:
def __init__(self):
pygame.init()
self.settings = Settings()
self.screen = pygame.display.set_mode((0, 0), pygame.FULLSCREEN)
self.settings.screen_width = self.screen.get_rect().width
self.settings.screen_height = self.screen.get_rect().height
pygame.display.set_caption("Ocean Adventure")
self.stats = GameStats(self)
self.sb = Scoreboard(self)
self.octopus = Octopus(self)
self.bullets = pygame.sprite.Group()
self.fishes = pygame.sprite.Group()
self._create_fleet()
self.play_button = Button(self, "Play")
self.bg_color = (187, 230, 247)
def run_game(self):
while True:
self._check_events()
if self.stats.game_active:
self.octopus.update()
self._update_bullets()
self._update_fishes()
self._update_screen()
def _check_events(self):
for event in pygame.event.get():
if event.type == pygame.QUIT:
sys.exit()
elif event.type == pygame.KEYDOWN:
self._check_keydown_events(event)
elif event.type == pygame.KEYUP:
self._check_keyup_events(event)
elif event.type == pygame.MOUSEBUTTONDOWN:
mouse_pos = pygame.mouse.get_pos()
self._check_play_button(mouse_pos)
def _check_play_button(self, mouse_pos):
button_clicked = self.play_button.rect.collidepoint(mouse_pos)
if button_clicked and not self.stats.game_active:
self.settings.initialize_dynamic_settings()
self.stats.reset_stats()
self.stats.game_active = True
self.sb.prep_score()
self.fishes.empty()
self.bullets.empty()
self._create_fleet()
self.octopus.center_octopus()
pygame.mouse.set_visible(False)
def _check_keydown_events(self, event):
if event.key == pygame.K_RIGHT:
self.octopus.moving_right = True
elif event.key == pygame.K_LEFT:
self.octopus.moving_left = True
elif event.key == pygame.K_q:
sys.exit()
elif event.key == pygame.K_SPACE:
self._fire_bullet()
def _check_keyup_events(self, event):
if event.key == pygame.K_RIGHT:
self.octopus.moving_right = False
elif event.key == pygame.K_LEFT:
self.octopus.moving_left = False
def _fire_bullet(self):
if len(self.bullets) < self.settings.bullets_allowed:
new_bullet = Bullet(self)
self.bullets.add(new_bullet)
def _update_bullets(self):
self.bullets.update()
for bullet in self.bullets.copy():
if bullet.rect.bottom <= 0:
self.bullets.remove(bullet)
self._check_bullet_alien_collisions()
def _check_bullet_alien_collisions(self):
collisions = pygame.sprite.groupcollide(self.bullets, self.fishes,
True, True)
if collisions:
for fishes in collisions.values():
self.stats.score += self.settings.fish_points * len(fishes)
self.sb.prep_score()
hitSound.play()
if not self.fishes:
self.bullets.empty()
self._create_fleet()
self.settings.increase_speed()
def _update_fishes(self):
self._check_fleet_edges()
self.fishes.update()
if pygame.sprite.spritecollideany(self.octopus, self.fishes):
self._octopus_hit()
self._check_fishes_bottom()
def _create_fleet(self):
fish = Fish(self)
fish_width, fish_height = fish.rect.size
available_space_x = self.settings.screen_width - (2 * fish_width)
number_fishes_x = available_space_x // (2 * fish_width)
octopus_height = self.octopus.rect.height
available_space_y = (self.settings.screen_height - (3 * fish_height) -
octopus_height)
number_rows = available_space_y // (2 * fish_height)
for row_number in range(number_rows):
for fish_number in range(number_fishes_x):
self._create_fish(fish_number, row_number)
def _create_fish(self, fish_number, row_number):
fish = Fish(self)
fish_width, fish_height = fish.rect.size
fish.x = fish_width + 2 * fish_width * fish_number
fish.rect.x = fish.x
fish.rect.y = fish.rect.height + 2 * fish.rect.height * row_number
self.fishes.add(fish)
def _check_fleet_edges(self):
for fish in self.fishes.sprites():
if fish.check_edges():
self._change_fleet_direction()
break
def _change_fleet_direction(self):
for fish in self.fishes.sprites():
fish.rect.y += self.settings.fleet_drop_speed
self.settings.fleet_direction *= -1
def _check_fishes_bottom(self):
screen_rect = self.screen.get_rect()
for fish in self.fishes.sprites():
if fish.rect.bottom >= screen_rect.bottom:
self._octopus_hit()
break
def _octopus_hit(self):
if self.stats.octopi_left > 0:
self.fishes.empty()
self.bullets.empty()
self._create_fleet()
self.octopus.center_octopus()
sleep(0.5)
else:
self.stats.game_active = False
pygame.mouse.set_visible(True)
def _update_screen(self):
self.screen.fill(self.settings.bg_color)
self.octopus.blitme()
for bullet in self.bullets.sprites():
bullet.draw_bullet()
self.fishes.draw(self.screen)
self.sb.show_score()
if not self.stats.game_active:
self.play_button.draw_button()
pygame.display.flip()