def download_from_esi(killmails): cache = FileCache(path=os.path.join(ROOT_DIR, 'cache/esipy_swagger')) esi_app = EsiApp(cache=cache, cache_time=60 * 60 * 24) app = esi_app.get_latest_swagger client = EsiClient( retry_requests=True, # set to retry on http 5xx error (default False) headers={'User-Agent': 'Something CCP can use to contact you and that define your app'}, raw_body_only=True, # default False, set to True to never parse response and only return raw JSON string content. ) operations = [] for killmail in killmails: operations.append( app.op['get_killmails_killmail_id_killmail_hash']( killmail_hash=killmail['zkb']['hash'], killmail_id=killmail['killmail_id'] ) ) results = client.multi_request(operations) full_killmails = [] for result in results: full_killmails.append(json.loads(result[1].raw)) return full_killmails
# to update the security object, security.update_token({ 'access_token': '', # leave this empty 'expires_in': -1, # seconds until expiry, so we force refresh anyway 'refresh_token': 'yUI96Q6kOxvQ6ulwDnQ7VLD1I-4LH-Jz8DOY9Di4NhA1' }) tokens = security.refresh() operations = [] for page in range(1, 20): operations.append(app.op['get_markets_structures_structure_id']( structure_id=1022734985679, page=page)) results = client.multi_request(operations) for (req, res) in results: for order in res.data: print str(order.type_id) + ', ' + str(order.price) + ', ' + str( order.is_buy_order) with open('test.csv', 'wb') as csvfile: datawriter = csv.writer(csvfile, delimiter=',') datawriter.writerow(['type_id', 'price', 'is_buy_order']) for (req, res) in results: for order in res.data: datawriter.writerow( [order.type_id, order.price, order.is_buy_order])
class TestEsiPy(unittest.TestCase): CALLBACK_URI = "https://foo.bar/baz/callback" LOGIN_EVE = "https://login.eveonline.com" OAUTH_VERIFY = "%s/oauth/verify" % LOGIN_EVE OAUTH_TOKEN = "%s/oauth/token" % LOGIN_EVE CLIENT_ID = 'foo' SECRET_KEY = 'bar' BASIC_TOKEN = six.u('Zm9vOmJhcg==') SECURITY_NAME = 'evesso' @mock.patch('six.moves.urllib.request.urlopen') def setUp(self, urlopen_mock): # I hate those mock... thx urlopen instead of requests... urlopen_mock.return_value = open('test/resources/swagger.json') self.app = App.create('https://esi.tech.ccp.is/latest/swagger.json') self.security = EsiSecurity( app=self.app, redirect_uri=TestEsiPy.CALLBACK_URI, client_id=TestEsiPy.CLIENT_ID, secret_key=TestEsiPy.SECRET_KEY, ) self.cache = DictCache() self.client = EsiClient(self.security, cache=self.cache) self.client_no_auth = EsiClient(cache=self.cache, retry_requests=True) def tearDown(self): """ clear the cache so we don't have residual data """ self.cache._dict = {} def test_esipy_client_no_args(self): client_no_args = EsiClient() self.assertIsNone(client_no_args.security) self.assertTrue(isinstance(client_no_args.cache, DictCache)) self.assertEqual(client_no_args._session.headers['User-Agent'], 'EsiPy/Client - https://github.com/Kyria/EsiPy') self.assertEqual(client_no_args.raw_body_only, False) def test_esipy_client_with_headers(self): client_with_headers = EsiClient(headers={'User-Agent': 'foobar'}) self.assertEqual(client_with_headers._session.headers['User-Agent'], 'foobar') def test_esipy_client_with_adapter(self): transport_adapter = HTTPAdapter() client_with_adapters = EsiClient(transport_adapter=transport_adapter) self.assertEqual(client_with_adapters._session.get_adapter('http://'), transport_adapter) self.assertEqual(client_with_adapters._session.get_adapter('https://'), transport_adapter) def test_esipy_client_without_cache(self): client_without_cache = EsiClient(cache=None) self.assertTrue(isinstance(client_without_cache.cache, DummyCache)) def test_esipy_client_with_cache(self): cache = DictCache() client_with_cache = EsiClient(cache=cache) self.assertTrue(isinstance(client_with_cache.cache, BaseCache)) self.assertEqual(client_with_cache.cache, cache) def test_esipy_client_wrong_cache(self): with self.assertRaises(ValueError): EsiClient(cache=DictCache) def test_esipy_request_public(self): with httmock.HTTMock(public_incursion): incursions = self.client_no_auth.request( self.app.op['get_incursions']()) self.assertEqual(incursions.data[0].type, 'Incursion') self.assertEqual(incursions.data[0].faction_id, 500019) def test_esipy_request_authed(self): with httmock.HTTMock(*_all_auth_mock_): self.security.auth('let it bee') char_location = self.client.request( self.app.op['get_characters_character_id_location']( character_id=123456789)) self.assertEqual(char_location.data.station_id, 60004756) # force expire self.security.token_expiry = 0 char_location_with_refresh = self.client.request( self.app.op['get_characters_character_id_location']( character_id=123456789)) self.assertEqual(char_location_with_refresh.data.station_id, 60004756) def test_client_cache_request(self): @httmock.all_requests def fail_if_request(url, request): self.fail('Cached data is not supposed to do requests') incursion_operation = self.app.op['get_incursions'] with httmock.HTTMock(public_incursion_no_expires): incursions = self.client_no_auth.request(incursion_operation()) self.assertEqual(incursions.data[0].state, 'mobilizing') with httmock.HTTMock(public_incursion_no_expires_second): incursions = self.client_no_auth.request(incursion_operation()) self.assertEqual(incursions.data[0].state, 'established') with httmock.HTTMock(public_incursion): incursions = self.client_no_auth.request(incursion_operation()) self.assertEqual(incursions.data[0].state, 'mobilizing') with httmock.HTTMock(fail_if_request): incursions = self.client_no_auth.request(incursion_operation()) self.assertEqual(incursions.data[0].state, 'mobilizing') def test_client_warning_header(self): with httmock.HTTMock(public_incursion_warning): warnings.simplefilter('error') incursion_operation = self.app.op['get_incursions'] with self.assertRaises(UserWarning): self.client_no_auth.request(incursion_operation()) def test_client_raw_body_only(self): client = EsiClient(raw_body_only=True) self.assertEqual(client.raw_body_only, True) with httmock.HTTMock(public_incursion): incursions = client.request(self.app.op['get_incursions']()) self.assertIsNone(incursions.data) self.assertTrue(len(incursions.raw) > 0) incursions = client.request(self.app.op['get_incursions'](), raw_body_only=False) self.assertIsNotNone(incursions.data) def test_esipy_reuse_operation(self): operation = self.app.op['get_incursions']() with httmock.HTTMock(public_incursion): incursions = self.client_no_auth.request(operation) self.assertEqual(incursions.data[0].faction_id, 500019) # this shouldn't create any errors incursions = self.client_no_auth.request(operation) self.assertEqual(incursions.data[0].faction_id, 500019) def test_esipy_multi_request(self): operation = self.app.op['get_incursions']() with httmock.HTTMock(public_incursion): count = 0 for req, incursions in self.client_no_auth.multi_request( [operation, operation, operation], threads=2): self.assertEqual(incursions.data[0].faction_id, 500019) count += 1 # Check we made 3 requests self.assertEqual(count, 3) def test_esipy_backoff(self): operation = self.app.op['get_incursions']() start_calls = time.time() with httmock.HTTMock(public_incursion_server_error): incursions = self.client_no_auth.request(operation) self.assertEqual(incursions.data.error, 'broke') end_calls = time.time() # Check we retried 5 times self.assertEqual(incursions.data.count, 5) # Check that backoff slept for a sum > 2 seconds self.assertTrue(end_calls - start_calls > 2) def test_esipy_timeout(self): def send_function(*args, **kwargs): """ manually create a ConnectionError to test the retry and be sure no exception is thrown """ send_function.count += 1 raise ConnectionError send_function.count = 0 self.client_no_auth._session.send = mock.MagicMock( side_effect=send_function) operation = self.app.op['get_incursions']() with httmock.HTTMock(public_incursion): incursions = self.client_no_auth.request(operation) # there shouldn't be any exceptions self.assertEqual(incursions.status, 500) self.assertEqual(send_function.count, 5)
class ESI: def __init__(self): self.db = Database() self.config = Config() self.scopes = self.config.getConfig()["settings"]["esiScopes"] self.esi_app = App.create( url=self.config.getConfig()["settings"]["esiURL"], ) self.security = EsiSecurity( app=self.esi_app, redirect_uri=self.config.getConfig()["settings"]["esiCallback"], client_id=self.config.getConfig()["settings"]["esiClientID"], secret_key=self.config.getConfig()["settings"]["esiSecretKey"], headers={ 'User-Agent': self.config.getConfig()["settings"]["esiCustomHeader"] }) self.client = EsiClient( security=self.security, retry_requests=True, headers={ 'User-Agent': self.config.getConfig()["settings"]["esiCustomHeader"] }) def getAuthURI(self): return self.security.get_auth_uri(scopes=self.scopes) def getToken(self, code): return self.security.auth(code) def getESIChar(self, token): self.security.update_token(token) try: self.security.refresh() except APIException as e: if str(e) == "HTTP Error 400: invalid_token": session.pop('token', None) session.pop('char', None) return redirect(url_for('page_routes.logout')) return self.security.verify() def isVerified(self, token): try: self.security.update_token(token) except: return False try: self.security.refresh() character = self.security.verify() except: return False session["char"] = character return True def getESIInfo(self, endpoint, obj): info = self.esi_app.op[endpoint](**obj) res = self.client.request(info) result = res.data try: if "response" in result: result = result["response"] except: pass return result def getESIInfoMP(self, endpoint, obj): info = self.esi_app.op[endpoint](**obj) res = self.client.head(info) if res.status == 200: number_of_pages = res.header["X-Pages"][0] ops = [] for page in range(1, number_of_pages + 1): obj["page"] = page ops.append(self.esi_app.op[endpoint](**obj)) results = self.client.multi_request(ops) return results return {} def subToken(self, refresh_token): self.security.update_token({ 'access_token': '', 'expires_in': -1, 'refresh_token': refresh_token }) def getForceRefresh(self): return self.security.refresh()
class TestEsiPy(unittest.TestCase): CALLBACK_URI = "https://foo.bar/baz/callback" LOGIN_EVE = "https://login.eveonline.com" OAUTH_VERIFY = "%s/oauth/verify" % LOGIN_EVE OAUTH_TOKEN = "%s/oauth/token" % LOGIN_EVE CLIENT_ID = 'foo' SECRET_KEY = 'bar' BASIC_TOKEN = six.u('Zm9vOmJhcg==') SECURITY_NAME = 'evesso' RSC_SSO_ENDPOINTS = "test/resources/oauth-authorization-server.json" RSC_JWKS = "test/resources/jwks.json" @mock.patch('six.moves.urllib.request.urlopen') def setUp(self, urlopen_mock): # I hate those mock... thx urlopen instead of requests... urlopen_mock.return_value = open('test/resources/swagger.json') warnings.simplefilter('ignore') self.app = App.create('https://esi.evetech.net/latest/swagger.json') with open(TestEsiPy.RSC_SSO_ENDPOINTS, 'r') as sso_endpoints: with open(TestEsiPy.RSC_JWKS, "r") as jwks: self.security = EsiSecurity( app=self.app, redirect_uri=TestEsiPy.CALLBACK_URI, client_id=TestEsiPy.CLIENT_ID, secret_key=TestEsiPy.SECRET_KEY, sso_endpoints=json.load(sso_endpoints), jwks_key=json.load(jwks)) self.cache = DictCache() self.client = EsiClient(self.security, cache=self.cache) self.client_no_auth = EsiClient(cache=self.cache, retry_requests=True) def tearDown(self): """ clear the cache so we don't have residual data """ self.cache._dict = {} def test_esipy_client_no_args(self): client_no_args = EsiClient() self.assertIsNone(client_no_args.security) self.assertTrue(isinstance(client_no_args.cache, DictCache)) self.assertEqual(client_no_args._session.headers['User-Agent'], 'EsiPy/Client - https://github.com/Kyria/EsiPy') self.assertEqual(client_no_args.raw_body_only, False) def test_esipy_client_with_headers(self): client_with_headers = EsiClient(headers={'User-Agent': 'foobar'}) self.assertEqual(client_with_headers._session.headers['User-Agent'], 'foobar') def test_esipy_client_with_adapter(self): transport_adapter = HTTPAdapter() client_with_adapters = EsiClient(transport_adapter=transport_adapter) self.assertEqual(client_with_adapters._session.get_adapter('http://'), transport_adapter) self.assertEqual(client_with_adapters._session.get_adapter('https://'), transport_adapter) def test_esipy_client_without_cache(self): client_without_cache = EsiClient(cache=None) self.assertTrue(isinstance(client_without_cache.cache, DummyCache)) def test_esipy_client_with_cache(self): cache = DictCache() client_with_cache = EsiClient(cache=cache) self.assertTrue(isinstance(client_with_cache.cache, BaseCache)) self.assertEqual(client_with_cache.cache, cache) def test_esipy_client_wrong_cache(self): with self.assertRaises(ValueError): EsiClient(cache=DictCache) def test_esipy_request_public(self): with httmock.HTTMock(public_incursion): incursions = self.client_no_auth.request( self.app.op['get_incursions']()) self.assertEqual(incursions.data[0].type, 'Incursion') self.assertEqual(incursions.data[0].faction_id, 500019) def test_esipy_request_authed(self): with httmock.HTTMock(*_all_auth_mock_): self.security.auth('let it bee') char_location = self.client.request( self.app.op['get_characters_character_id_location']( character_id=123456789)) self.assertEqual(char_location.data.station_id, 60004756) # force expire self.security.token_expiry = 0 char_location_with_refresh = self.client.request( self.app.op['get_characters_character_id_location']( character_id=123456789)) self.assertEqual(char_location_with_refresh.data.station_id, 60004756) def test_client_cache_request(self): @httmock.all_requests def fail_if_request(url, request): self.fail('Cached data is not supposed to do requests') incursion_operation = self.app.op['get_incursions'] with httmock.HTTMock(public_incursion_no_expires): incursions = self.client_no_auth.request(incursion_operation()) self.assertEqual(incursions.data[0].state, 'mobilizing') with httmock.HTTMock(public_incursion_no_expires_second): incursions = self.client_no_auth.request(incursion_operation()) self.assertEqual(incursions.data[0].state, 'established') with httmock.HTTMock(public_incursion): incursions = self.client_no_auth.request(incursion_operation()) self.assertEqual(incursions.data[0].state, 'mobilizing') with httmock.HTTMock(fail_if_request): incursions = self.client_no_auth.request(incursion_operation()) self.assertEqual(incursions.data[0].state, 'mobilizing') def test_client_warning_header(self): # deprecated warning warnings.simplefilter('error') with httmock.HTTMock(public_incursion_warning): incursion_operation = self.app.op['get_incursions'] with self.assertRaises(UserWarning): self.client_no_auth.request(incursion_operation()) with self.assertRaises(UserWarning): self.client_no_auth.head(incursion_operation()) def test_client_raw_body_only(self): client = EsiClient(raw_body_only=True) self.assertEqual(client.raw_body_only, True) with httmock.HTTMock(public_incursion): incursions = client.request(self.app.op['get_incursions']()) self.assertIsNone(incursions.data) self.assertTrue(len(incursions.raw) > 0) incursions = client.request(self.app.op['get_incursions'](), raw_body_only=False) self.assertIsNotNone(incursions.data) def test_esipy_reuse_operation(self): operation = self.app.op['get_incursions']() with httmock.HTTMock(public_incursion): incursions = self.client_no_auth.request(operation) self.assertEqual(incursions.data[0].faction_id, 500019) # this shouldn't create any errors incursions = self.client_no_auth.request(operation) self.assertEqual(incursions.data[0].faction_id, 500019) def test_esipy_multi_request(self): operation = self.app.op['get_incursions']() with httmock.HTTMock(public_incursion): count = 0 for req, incursions in self.client_no_auth.multi_request( [operation, operation, operation], threads=2): self.assertEqual(incursions.data[0].faction_id, 500019) count += 1 # Check we made 3 requests self.assertEqual(count, 3) def test_esipy_backoff(self): operation = self.app.op['get_incursions']() start_calls = time.time() with httmock.HTTMock(public_incursion_server_error): incursions = self.client_no_auth.request(operation) self.assertEqual(incursions.data.error, 'broke') end_calls = time.time() # Check we retried 5 times self.assertEqual(incursions.data.count, 5) # Check that backoff slept for a sum > 2 seconds self.assertTrue(end_calls - start_calls > 2) def test_esipy_timeout(self): def send_function(*args, **kwargs): """ manually create a ConnectionError to test the retry and be sure no exception is thrown """ send_function.count += 1 raise ConnectionError send_function.count = 0 self.client_no_auth._session.send = mock.MagicMock( side_effect=send_function) operation = self.app.op['get_incursions']() with httmock.HTTMock(public_incursion): incursions = self.client_no_auth.request(operation) # there shouldn't be any exceptions self.assertEqual(incursions.status, 500) self.assertEqual(send_function.count, 5) def test_esipy_raise_on_error(self): operation = self.app.op['get_incursions']() with httmock.HTTMock(public_incursion_server_error): # try with retries with self.assertRaises(APIException): self.client_no_auth.request(operation, raise_on_error=True) # try without retries with self.assertRaises(APIException): self.client.request(operation, raise_on_error=True) # try with head with self.assertRaises(APIException): self.client_no_auth.head(operation, raise_on_error=True) def test_esipy_expired_response(self): operation = self.app.op['get_incursions'] with httmock.HTTMock(public_incursion_expired): warnings.filterwarnings('error', '.*returned expired result') with self.assertRaises(UserWarning): self.client_no_auth.request(operation()) warnings.resetwarnings() warnings.simplefilter('ignore') incursions = self.client_no_auth.request(operation()) self.assertEquals(incursions.status, 200) def test_esipy_uncached_method(self): operation = self.app.op['post_universe_ids'](names=['Foo']) self.assertEqual(self.cache._dict, {}) with httmock.HTTMock(post_universe_id): res = self.client.request(operation) self.assertEqual(res.data.characters[0].id, 123456789) self.assertEqual(self.cache._dict, {}) def test_esipy_head_request(self): operation = self.app.op['get_incursions']() with httmock.HTTMock(public_incursion): res = self.client.head(operation) self.assertIsNone(res.data) self.assertIn('Expires', res.header) def test_esipy_expired_header_etag(self): @httmock.all_requests def check_etag(url, request): self.assertEqual(request.headers.get('If-None-Match'), '"esipy_test_etag_status"') return httmock.response(headers={ 'Etag': '"esipy_test_etag_status"', 'expires': make_expire_time_str(), 'date': make_expire_time_str() }, status_code=304) operation = self.app.op['get_status']() with httmock.HTTMock(eve_status): self.assertEqual(self.cache._dict, {}) res = self.client.request(operation) self.assertNotEqual(self.cache._dict, {}) self.assertEqual(res.data.server_version, "1313143") time.sleep(2) with httmock.HTTMock(check_etag): res = self.client.request(operation) self.assertEqual(res.data.server_version, "1313143") def test_esipy_expired_header_noetag(self): def check_etag(url, request): self.assertNotIn('If-None-Match', request.headers) return httmock.response(status_code=200, content={ "players": 29597, "server_version": "1313143", "start_time": "2018-05-20T11:04:30Z" }) operation = self.app.op['get_status']() with httmock.HTTMock(eve_status_noetag): res = self.client.request(operation) self.assertEqual(res.data.server_version, "1313143") time.sleep(2) with httmock.HTTMock(check_etag): res = self.client.request(operation) self.assertEqual(res.data.server_version, "1313143") def test_esipy_non_json_response(self): operation = self.app.op['get_status']() with httmock.HTTMock(non_json_error): try: self.client.request(operation) except APIException as exc: self.assertEqual(exc.status_code, 502) self.assertEqual( exc.response, six.b('<html><body>Some HTML Errors</body></html>')) try: self.client_no_auth.request(operation) except APIException as exc: self.assertEqual(exc.status_code, 502) self.assertEqual( exc.response, six.b('<html><body>Some HTML Errors</body></html>'))
class myApp: pp = pprint.PrettyPrinter(indent=2) app = None client = None def main(self): print "setting up the app..." self.app = App.create( url= "https://esi.tech.ccp.is/latest/swagger.json?datasource=tranquility" ) print "done. \n Setting up the client." self.client = EsiClient( retry_requests= True, # set to retry on http 5xx error (default False) header={'User-Agent': 'Jimmy - api test app: [email protected]'}, raw_body_only= True, # default False, set to True to never parse response and only return raw JSON string content. ) print "done, after this it's all me and my calls." chars = ["alderith", "gruxella", "lord grapefruit", "druzidelcastro"] try: # self.get_id_for_users(chars) # self.get_group_item_ids() self.get_orders_for_region(10000002) except Exception as err: self.pp.pprint(err) # self.get_user_info() # try: # self.original_example() # except Exception as err: # print "Error:: " # print err # e = sys.exc_info()[0] # self.pp.pprint(e) # self.original_example() # print "hello world" def get_orders_for_region(self, reg_id): count = 0 query = self.app.op['get_markets_region_id_orders']( order_type="all", region_id=reg_id, ) response = self.client.request(query) # self.pp.pprint(response.raw) print "Attempting to parse json" data = json.loads(response.raw) print "done parsing json" print len(data) # self.pp.pprint(data) # return queries = [] print "this dataset expires:" print response.header['Expires'] print "we need this many pages:" size = response.header['X-Pages'][0] print size if (size > 1): for page in range(1, size + 1): # for page in range(1,3): queries.append(self.app.op['get_markets_region_id_orders']( order_type="all", region_id=reg_id, page=page, )) print "doing long request to CCP..." # results = self.client.multi_request(queries,None,None,size) results = self.client.multi_request(queries) print "done with long request" for result in results: # self.pp.pprint(result[1].header) # self.pp.pprint(result[1].data) data = json.loads(result[1].raw) count += len(data) print "Running count = %d" % (count) print "is buy order? %r" % (data[0]['is_buy_order']) print "remaining volume: %d" % (data[0]['volume_remain']) # self.pp.pprint(results) print "Total orders = %d" % (count) return results def get_market_groups(self): query = self.app.op['get_markets_groups']() response = self.client.request(query) # self.pp.pprint(response.data) self.pp.pprint(response.header) print len(response.data) return response def get_group_item_ids(self): marketgroup_response = self.get_market_groups() groups = [] count = 0 for group in marketgroup_response.data: count += 1 print group query = self.app.op['get_markets_groups_market_group_id']( market_group_id=group) response = self.client.request(query) groups.append(response.data) if count == 25: break # self.pp.pprint(response.data) self.pp.pprint(groups) def get_id_for_users(self, *user_array): users = [] query = self.app.op['post_universe_ids'](names=user_array[0]) response = self.client.request(query) for char in response.data['characters']: print "%s id = %d" % (char['name'], char['id']) def get_user_info(self): query = self.app.op['post_universe_ids']( names=["Gruxella", "Alderith"]) # query = self.app.op['get_characters_character_id']( # character_id=712133937 # ) # query = self.app.op['get_characters_names']( # character_ids=[712133937] # ) response = self.client.request(query) print response.data def original_example(self): market_order_operation = self.app.op['get_markets_region_id_orders']( region_id=10000002, type_id=34, order_type='all', ) # do the request print "did I get here? 2" response = self.client.request(market_order_operation) # use it: response.data contains the parsed result of the request. print response.data[0].price # to get the headers objects, you can get the header attribute print response.header