class ConfluenceFuturesAPI(ConfluenceAPI): def __init__(self, username, password, uri_base, user_agent=ConfluenceAPI.DEFAULT_USER_AGENT, executor=None, max_workers=10): """ Initialize the async concurrent.futures API object. :param username: Your Confluence username. :param password: Your Confluence password. :param uri_base: The base url for your Confluence wiki (e.g. myorg.atlassian.com/wiki) :param user_agent: (Optional): The user-agent you wish to send on requests to the API. DEFAULT: PythonConfluenceAPI. :param executor: (Optional): The concurrent.futures executor to power the API calls. Default: None, create a new ThreadPoolExecutor. :param max_workers: (Optional): If the executor is not specified and the default ThreadPoolExecutor is spawned, this specifies the number of worker threads to create. """ super(ConfluenceFuturesAPI, self).__init__(username, password, uri_base, user_agent) self.executor = executor self.max_workers = max_workers def _start_http_session(self): """ Start a new requests HTTP session, clearing cookies and session data. :return: None """ api_logger.debug("Starting new HTTP session...") self.session = FuturesSession(executor=self.executor, max_workers=self.max_workers) self.session.headers.update({"User-Agent": self.user_agent}) if self.username and self.password: api_logger.debug("Requests will use authorization.") self.session.auth = HTTPBasicAuth(self.username, self.password) def _service_request(self, request_type, sub_uri, params=None, callback=None, raise_for_status=True, raw=False, **kwargs): """ Base method for handling HTTP requests via the current requests session. :param request_type: The request type as a string (e.g. "POST", "GET", "PUT", etc.) :param sub_uri: The REST end point (sub-uri) to communicate with. :param params: (Optional) HTTP Request parameters. Default: none :param callback: (Optional) A callback function to be excuted on the resulting requests response. This synchronous implementation will return the results of the callback. Default: None. This method returns either the decoded JSON or the raw request content. :param raise_for_status: (Optional) When set True, we raise requests.HTTPError on 4xx or 5xx status. When set False, non-2xx/3xx status code is ignored. Default: True :param raw: (Optional) If no callback is set, return the raw content from the request if this is set True. If False, the method attempts to parse the request as JSON data and return the resutls. Default: False :param kwargs: Additional parameters to pass to the session request call. :return: The concurrent.futures object that holds the future for the API method call. """ api_logger.debug("Sending request: {} ({})".format( sub_uri, request_type)) if not self.session: self._start_http_session() uri = urljoin(self.uri_base, sub_uri) if params: kwargs.update(params=params) if callback: def base_callback(_, response): if raise_for_status: response.raise_for_status() return callback(response) else: def base_callback(_, response): if raise_for_status: response.raise_for_status() return response.content if raw else json.loads(response.text) response_future = self.session.request( request_type, uri, background_callback=base_callback, **kwargs) return response_future
class ConfluenceFuturesAPI(ConfluenceAPI): def __init__(self, username, password, uri_base, user_agent=ConfluenceAPI.DEFAULT_USER_AGENT, executor=None, max_workers=10): """ Initialize the async concurrent.futures API object. :param username: Your Confluence username. :param password: Your Confluence password. :param uri_base: The base url for your Confluence wiki (e.g. myorg.atlassian.com/wiki) :param user_agent: (Optional): The user-agent you wish to send on requests to the API. DEFAULT: PythonConfluenceAPI. :param executor: (Optional): The concurrent.futures executor to power the API calls. Default: None, create a new ThreadPoolExecutor. :param max_workers: (Optional): If the executor is not specified and the default ThreadPoolExecutor is spawned, this specifies the number of worker threads to create. """ super(ConfluenceFuturesAPI, self).__init__(username, password, uri_base, user_agent) self.executor = executor self.max_workers = max_workers def _start_http_session(self): """ Start a new requests HTTP session, clearing cookies and session data. :return: None """ api_logger.debug("Starting new HTTP session...") self.session = FuturesSession(executor=self.executor, max_workers=self.max_workers) self.session.headers.update({"User-Agent": self.user_agent}) if self.username and self.password: api_logger.debug("Requests will use authorization.") self.session.auth = HTTPBasicAuth(self.username, self.password) def _service_request(self, request_type, sub_uri, params=None, callback=None, raise_for_status=True, raw=False, **kwargs): """ Base method for handling HTTP requests via the current requests session. :param request_type: The request type as a string (e.g. "POST", "GET", "PUT", etc.) :param sub_uri: The REST end point (sub-uri) to communicate with. :param params: (Optional) HTTP Request parameters. Default: none :param callback: (Optional) A callback function to be excuted on the resulting requests response. This synchronous implementation will return the results of the callback. Default: None. This method returns either the decoded JSON or the raw request content. :param raise_for_status: (Optional) When set True, we raise requests.HTTPError on 4xx or 5xx status. When set False, non-2xx/3xx status code is ignored. Default: True :param raw: (Optional) If no callback is set, return the raw content from the request if this is set True. If False, the method attempts to parse the request as JSON data and return the resutls. Default: False :param kwargs: Additional parameters to pass to the session request call. :return: The concurrent.futures object that holds the future for the API method call. """ api_logger.debug("Sending request: {} ({})".format(sub_uri, request_type)) if not self.session: self._start_http_session() uri = urljoin(self.uri_base, sub_uri) if params: kwargs.update(params=params) if callback: def base_callback(_, response): if raise_for_status: response.raise_for_status() return callback(response) else: def base_callback(_, response): if raise_for_status: response.raise_for_status() return response.content if raw else json.loads(response.text) response_future = self.session.request(request_type, uri, background_callback=base_callback, **kwargs) return response_future
class THttpClient(TTransportBase): '''Http implementation of TTransport base.''' def __init__(self, uri_or_host, port=None, path=None, customThrift=False, request='httplib', http2=False, proxy_host=None, proxy_port=None, proxy_auth=None): '''THttpClient supports two different types constructor parameters. THttpClient(host, port, path) - deprecated THttpClient(uri) Only the second supports https. ''' if port is not None: warnings.warn( 'Please use the THttpClient("http://host:port/path") syntax', DeprecationWarning, stacklevel=2 ) self.host = uri_or_host self.port = port assert path self.path = path self.scheme = 'http' else: parsed = urllib.parse.urlparse(uri_or_host) self.scheme = parsed.scheme assert self.scheme in ('http', 'https') if self.scheme == 'http': self.port = parsed.port or http_client.HTTP_PORT elif self.scheme == 'https': self.port = parsed.port or http_client.HTTPS_PORT self.host = parsed.hostname self.path = parsed.path if parsed.query: self.path += '?%s' % parsed.query proxy = None self.request = request self.http2 = http2 self.realhost = proxy_host self.realport = proxy_port self.proxy_auth = proxy_auth self.__wbuf = BytesIO() if self.scheme == 'https' and self.using_proxy() and self.proxy_auth: self.proxy_headers = {'Proxy-Authorization': self.proxy_auth} else: self.proxy_headers = None self.url = '%s://%s:%s%s' % (self.scheme, self.host, self.port, self.path) if customThrift: if self.request == 'hyper': if self.http2: self.__http = hyper.HTTP20Connection(self.host, self.port, proxy_host=self.realhost, proxy_port=self.realport, proxy_headers=self.proxy_headers) else: self.__http = hyper.HTTPConnection(self.host, self.port, proxy_host=self.realhost, proxy_port=self.realport, proxy_headers=self.proxy_headers) elif self.request == 'httpx': if self.http2: self.__http = httpx.AsyncClient(base_url='%s://%s' % (self.scheme, self.host), http2=self.http2) else: self.__http = httpx.Client(base_url='%s://%s' % (self.scheme, self.host)) elif self.request == 'requests': self.__http = requests.Session() if self.using_proxy(): self.__http.proxies = urllib.request.getproxies() elif self.request == 'requests-futures': self.__http = FuturesSession() if self.using_proxy(): self.__http.proxies = urllib.request.getproxies() elif self.request == 'httplib2': self.__http = httplib2.Http() else: if self.scheme == 'http': self.__http = http_client.HTTPConnection(self.host, self.port) elif self.scheme == 'https': self.__http = http_client.HTTPSConnection(self.host, self.port) if self.using_proxy(): self.__http.set_tunnel(self.realhost, self.realport, self.proxy_headers) else: self.__http = None self.__async_loop = asyncio.get_event_loop() if self.request == 'httpx' and self.http2 else None self.__http_response = None self.__response_data = None self.__last_read = 0 self.__timeout = None self.__custom_headers = None self.__time = time.time() self.__custom_thrift = customThrift self.__loop = 0 @staticmethod def basic_proxy_auth_header(proxy): if proxy is None or not proxy.username: return None ap = '%s:%s' % (urllib.parse.unquote(proxy.username), urllib.parse.unquote(proxy.password)) cr = base64.b64encode(ap).strip() return 'Basic ' + cr def using_proxy(self): return self.realhost is not None def open(self): if self.request == 'hyper': if self.http2: self.__http = hyper.HTTP20Connection(self.host, self.port, proxy_host=self.realhost, proxy_port=self.realport, proxy_headers=self.proxy_headers) else: self.__http = hyper.HTTPConnection(self.host, self.port, proxy_host=self.realhost, proxy_port=self.realport, proxy_headers=self.proxy_headers) elif self.request == 'httpx': if self.http2: self.__http = httpx.AsyncClient(base_url='%s://%s' % (self.scheme, self.host), http2=self.http2) else: self.__http = httpx.Client(base_url='%s://%s' % (self.scheme, self.host)) elif self.request == 'requests': self.__http = requests.Session() if self.using_proxy(): self.__http.proxies = urllib.request.getproxies() elif self.request == 'requests-futures': self.__http = FuturesSession() if self.using_proxy(): self.__http.proxies = urllib.request.getproxies() elif self.request == 'httplib2': self.__http = httplib2.Http() else: if self.scheme == 'http': self.__http = http_client.HTTPConnection(self.host, self.port) elif self.scheme == 'https': self.__http = http_client.HTTPSConnection(self.host, self.port) if self.using_proxy(): self.__http.set_tunnel(self.realhost, self.realport, self.proxy_headers) def close(self): if self.request != 'httpx': self.__http.close() self.__http = None self.reset() def reset(self): self.__http_response = None self.__response_data = None self.__last_read = 0 def getHeaders(self): return self.headers def isOpen(self): return self.__http is not None def setTimeout(self, ms): if not hasattr(socket, 'getdefaulttimeout'): raise NotImplementedError if ms is None: self.__timeout = None else: self.__timeout = ms / 1000.0 def setCustomHeaders(self, headers): self.__custom_headers = headers def read(self, sz): if self.request in ['httpx', 'httplib2', 'requests', 'requests-futures']: max_sz = self.__last_read + sz min_sz = self.__last_read self.__last_read = max_sz content = self.__response_data[min_sz:max_sz] else: content = self.__http_response.read(sz) return content def write(self, buf): self.__wbuf.write(buf) def __withTimeout(f): def _f(*args, **kwargs): orig_timeout = socket.getdefaulttimeout() socket.setdefaulttimeout(args[0].__timeout) try: result = f(*args, **kwargs) finally: socket.setdefaulttimeout(orig_timeout) return result return _f async def httpx_flush(self, data, headers): # Building httpx request request = self.__http.build_request('POST', self.path, data=data, headers=headers) # Sending httpx request self.__http_response = await self.__http.send(request) self.code = self.__http_response.status_code self.message = self.__http_response.reason_phrase self.headers = self.__http_response.headers self.__response_data = self.__http_response.read() self.__last_read = 0 def flush(self): if self.request == 'httplib': # Only when using httplib request if self.__custom_thrift: if self.__loop <= 2: if self.isOpen(): self.close() self.open(); self.__loop += 1 elif time.time() - self.__time > 90: self.close(); self.open(); self.__time = time.time() else: if self.isOpen(): self.close() self.open() else: self.reset() # Pull data out of buffer data = self.__wbuf.getvalue() self.__wbuf = BytesIO() if not self.__custom_headers or 'User-Agent' not in self.__custom_headers: user_agent = 'Python/THttpClient' script = os.path.basename(sys.argv[0]) if script: user_agent = '%s (%s)' % (user_agent, urllib.parse.quote(script)) else: user_agent = None if self.request == 'hyper': headers = {'Content-Type': 'application/x-thrift', 'Content-Length': str(len(data)), 'User-Agent': user_agent} if self.__custom_headers: headers.update(self.__custom_headers) # Sending request with payload request = self.__http.request('POST', self.path, data, headers) # Get reply to flush the request self.__http_response = self.__http.get_response(request) self.code = self.__http_response.status self.message = self.__http_response.reason self.headers = self.__http_response.headers elif self.request == 'httpx': headers = {'Content-Type': 'application/x-thrift', 'Content-Length': str(len(data)), 'User-Agent': user_agent} if self.__custom_headers: headers.update(self.__custom_headers) if self.http2: self.__async_loop.run_until_complete(self.httpx_flush(data, headers)) else: # Building httpx request request = self.__http.build_request('POST', self.path, data=data, headers=headers) # Sending httpx request self.__http_response = self.__http.send(request) self.__response_data = self.__http_response.read() self.__last_read = 0 self.code = self.__http_response.status_code self.message = self.__http_response.reason_phrase self.headers = self.__http_response.headers elif self.request == 'httplib2': headers = {'Content-Type': 'application/x-thrift', 'Content-Length': str(len(data)), 'User-Agent': user_agent} if self.__custom_headers: headers.update(self.__custom_headers) # Sending and get reply to request self.__http_response, self.__response_data = self.__http.request(self.url, 'POST', headers=headers, body=data) self.__last_read = 0 self.code = self.__http_response.status self.message = self.__http_response.reason self.headers = self.__http_response elif self.request == 'requests': headers = {'Content-Type': 'application/x-thrift', 'Content-Length': str(len(data)), 'User-Agent': user_agent} if self.__custom_headers: headers.update(self.__custom_headers) # Sending and get reply to request self.__http_response = self.__http.request('POST', self.url, data=data, headers=headers) self.__response_data = self.__http_response.content self.__last_read = 0 self.code = self.__http_response.status_code self.message = self.__http_response.reason self.headers = self.__http_response.headers elif self.request == 'requests-futures': headers = {'Content-Type': 'application/x-thrift', 'Content-Length': str(len(data)), 'User-Agent': user_agent} if self.__custom_headers: headers.update(self.__custom_headers) # Sending request with payload request = self.__http.request('POST', self.url, data=data, headers=headers) # Get reply to flush the request self.__http_response = request.result() self.__response_data = self.__http_response.content self.__last_read = 0 self.code = self.__http_response.status_code self.message = self.__http_response.reason self.headers = self.__http_response.headers else: # HTTP request if self.using_proxy() and self.scheme == 'http': # need full URL of real host for HTTP proxy here (HTTPS uses CONNECT tunnel) self.__http.putrequest('POST', 'http://%s:%s%s' % (self.realhost, self.realport, self.path)) else: self.__http.putrequest('POST', self.path) # Write headers self.__http.putheader('Content-Type', 'application/x-thrift') self.__http.putheader('Content-Length', str(len(data))) if not self.__custom_headers or 'User-Agent' not in self.__custom_headers: self.__http.putheader('User-Agent', user_agent) if self.__custom_headers: for key, val in six.iteritems(self.__custom_headers): self.__http.putheader(key, val) self.__http.endheaders() # Write payload self.__http.send(data) # Get reply to flush the request self.__http_response = self.__http.getresponse() self.code = self.__http_response.status self.message = self.__http_response.reason self.headers = self.__http_response.msg # Decorate if we know how to timeout if hasattr(socket, 'getdefaulttimeout'): flush = __withTimeout(flush)
class HackerOneClient(object): BASE_URL = "https://api.hackerone.com/v1" REQUEST_HEADERS = { "User-Agent": "HackerOne Python Client v" + __version__, } def __init__(self, identifier, token): self.identifier = identifier self.token = token self._init_session() def _init_session(self): self.s = FuturesSession() self.s.headers.update(self.REQUEST_HEADERS) self.s.auth = HTTPBasicAuth(self.identifier, self.token) def make_request(self, url, params=None, data=None, method=None): if method is None: method = "GET" if not url.startswith("http"): url = self.BASE_URL + url if isinstance(params, dict): params = encode_params(params) return self.s.request(method, url, params=params, data=data) def request_json(self, url, params=None, data=None, method=None): r = self.make_request(url, params, data, method).result() r.raise_for_status() return r.json() def request_object(self, url, params=None, data=None, method=None): data = self.request_json(url, params, data, method)["data"] # If we're fetching a single object make sure that consumers # know this is the canonical version data["_canonical"] = True return hydrate_object(data, self) def request_paginated_objects(self, url, params=None, yield_pages=False): future = self.make_request(url, params) res_iter = self._request_paginated_inner(url, future, yield_pages) return LazyListing(res_iter) def _request_paginated_inner(self, url, future, yield_pages): while url: resp = future.result() resp.raise_for_status() parsed = resp.json() url = parsed["links"].get('next') if url: future = self.make_request(url) hydrated_objs = hydrate_objects(parsed["data"], self) if yield_pages: yield hydrated_objs else: for obj in hydrated_objs: yield obj def get_resource(self, rsrc_type, obj_id): return rsrc_type.get(self, obj_id) def find_resources(self, rsrc_type, sort=None, yield_pages=False, **kwargs): """Find instances of `rsrc_type` that match the filter in `**kwargs`""" return rsrc_type.find(self, sort=sort, yield_pages=yield_pages, **kwargs)
class DataProcessing: """Process the Bigquery Data.""" def __init__(self, s3_client=None): """Initialize DataProcessing object.""" self.data = None self.cache = CacheDict(max_len=50000) self.pkg_counter = Counter() self.s3_client = s3_client self.req_session = FuturesSession(session=Session()) def async_fetch(self, url, payload={}, headers={"Accept": "application/json"}, method='GET', others=None): """Fetch urls asynchronously.""" if url in self.cache: self.responses.append(self.cache[url]) else: self.process_queue.append( (others, url, self.req_session.request(method, url))) def is_fetch_done(self, callback=lambda x: x): """Check whether all the requests are processed or not.""" _flag = True for resp in self.process_queue: _flag = False others, url, req_obj = resp logger.info("other:{}, url:{}, req_obj:{}".format(others, url, req_obj)) if url in self.cache: req_obj.cancel() self.process_queue.remove(resp) self.responses.append(self.cache[url]) elif req_obj.done(): req_obj.cancel() self.process_queue.remove(resp) self.cache[url] = (others, callback(req_obj)) self.responses.append((others, callback(req_obj))) return _flag def update_s3_bucket(self, data, bucket_name, filename='collated.json'): """Upload s3 bucket.""" if self.s3_client is None: # creat s3 client if not exists. self.s3_client = AmazonS3( bucket_name=bucket_name, aws_access_key_id=os.getenv('AWS_S3_ACCESS_KEY_ID'), aws_secret_access_key=os.getenv('AWS_S3_SECRET_ACCESS_KEY'), ) self.s3_client.connect() if not self.s3_client.is_connected(): raise ValueError("Unable to connect to s3.") json_data = dict() if self.s3_client.object_exists(filename): logger.info("{} exists, updating it.".format(filename)) json_data = self.s3_client.read_json_file(filename) if not json_data: raise ValueError("Unable to get the json data path:{}/{}" .format(bucket_name, filename)) json_data.update(data) self.s3_client.write_json_file(filename, json_data) logger.info("Updated file Succefully!")
def searchTickets(request, params): session = FuturesSession() async_list = [] tic = [] headers = { 'user-agent': 'yatse/0.0.1', 'api-key': settings.API_KEY, 'api-user': request.user.username } for Srv in Server.objects.all(): url = '%s/yatse/' % Srv.url # , hooks={'response': do_something} req = session.request('SEARCH', url, data=json.dumps(params), headers=headers) setattr(req, 'serverName', Srv.name) setattr(req, 'serverID', Srv.id) setattr(req, 'serverShortName', Srv.short) async_list.append(req) for req in async_list: try: result = req.result() if result.status_code != 200: messages.add_message( request, messages.ERROR, _(u'%s respoded width: %s' % (req.serverName, result.status_code))) else: data = json.loads(result.content) for date in data: date['YATSServer'] = req.serverShortName date['serverID'] = req.serverID date['c_date'] = dateutil.parser.parse(date['c_date']) date['last_action_date'] = dateutil.parser.parse( date['last_action_date']) if is_naive(date['last_action_date']): date['last_action_date'] = make_aware( date['last_action_date']) if 'close_date' in date and date['close_date']: date['close_date'] = dateutil.parser.parse( date['close_date']) date['is_late'] = 0 if 'daedline' in date and date['daedline']: date['daedline'] = dateutil.parser.parse( date['daedline']) if date['daedline'] < datetime.date.today(): date['is_late'] = 2 if date['daedline'] < datetime.date.today( ) + datetime.timedelta(days=7): date['is_late'] = 1 tic = tic + data except: messages.add_message( request, messages.ERROR, _(u'YATS nicht erreichbar: %s' % req.serverName)) return tic
class AsyncConnection(AbstractConnection): def __init__(self, *, base_url, disable_ssl_certificate, token_manager, retries, max_requests_workers=6, proxy_url=None): super().__init__(base_url=base_url, disable_ssl_certificate=disable_ssl_certificate, token_manager=token_manager, retries=retries) executor = cf.ThreadPoolExecutor(max_workers=max_requests_workers) adapter_kwargs = { 'pool_connections': max_requests_workers, 'pool_maxsize': max_requests_workers, 'max_retries': self._retries, 'pool_block': True } self._asession = FuturesSession(executor=executor) self._asession.mount('https://', HTTPAdapter(**adapter_kwargs)) self._asession.mount('http://', HTTPAdapter(**adapter_kwargs)) if proxy_url is not None: self._asession.proxies = { 'http': proxy_url, 'https': proxy_url, } self._access_token_lock = Lock() self._max_requests_workers = max_requests_workers @property def executor(self): return self._asession.executor @property def max_request_workers(self): return self._max_requests_workers def _add_authorization_maybe(self, headers: dict, url: str): with self._access_token_lock: super()._add_authorization_maybe(headers, url) def post(self, path, headers=None, callback=None, data=None, timeout=30.0): url = urljoin(self._base_url, path) params = { 'method': 'POST', 'url': url, 'headers': headers, 'data': data, 'verify': (not self._disable_ssl_certificate), 'timeout': timeout } return self._send_request(params, on_finish_callback=callback) def put(self, path, headers=None, callback=None, files=None, timeout=30.0): url = urljoin(self._base_url, self._encode_spaces(path)) params = { 'method': 'PUT', 'url': url, 'headers': headers, 'files': files, 'verify': (not self._disable_ssl_certificate), 'timeout': timeout } return self._send_request(params=params, on_finish_callback=callback) def _send_request(self, params, on_finish_callback): params['headers'] = params['headers'] or {} self._add_authorization_maybe(params['headers'], params['url']) self._add_user_agent(params['headers']) try: token = params['headers']['Authorization'].split( 'Bearer')[1].strip() except KeyError: token = None def extended_callback(response, *args, **kwargs): if response.status_code == 401: LOGGER.debug('Got a 401 status') skip = self._skip_token_renewal(params['url']) if not skip: with self._access_token_lock: # block concurrent send requests renewed = (token != self._token_manager.token.access_token) if renewed: LOGGER.debug('Token already renewed') else: self._renew_token() if on_finish_callback: on_finish_callback(response) c_params = params c_params['hooks'] = {'response': extended_callback} LOGGER.debug('Making request {} to {}'.format(params['method'], params['url'])) return self._asession.request(**c_params)
from requests_html import HTML from requests_futures.sessions import FuturesSession BASE = 'http://theremin.music.uiowa.edu/' # Downloaded from `MISPiano.html` with open('mispiano.html') as f: root = HTML(html=f.read()) mfs = [h for h in root.links if 'Piano.ff' in h] nmfs = len(mfs) print(f'total: {nmfs} files') session = FuturesSession(max_workers=10) ars = [session.request('get', BASE + h) for h in mfs] for i, ar in enumerate(ars): r = ar.result() fname = r.url.split('/')[-1] if r.status_code != 200: print(f'Failed to download {fname}') continue with open(fname, 'wb') as f: f.write(r.content) print(f'[{i+1}/{nmfs}] {fname} done')
class Client(object): def __init__(self, url, key=None, secret=None, verbose=False, do_async=False, max_workers=8, session=None, response_hook=None, **kwargs): """ Creates a client to connect to the HTTP bridge services :param url: The URL to connect to to access the Crossbar :param do_async: Run request in async mode :param key: The key for the API calls :param secret: The secret for the API calls :param verbose: True if you want debug messages printed :param kwargs: Extra kwargs passed to requests.request e.g proxies, auth, verify etc. :return: Nothing """ assert url is not None self.url = url self.do_async = do_async self.key = key self.secret = secret self.verbose = verbose self.sequence = 1 self.kwargs = kwargs if self.do_async is True: self.session = FuturesSession(max_workers=max_workers, session=session) self.session.hooks['response'] = response_hook def get_url(self, ext_url): url = self.url if ext_url not in self.url: url = '{}{}'.format(self.url, ext_url) return url def publish(self, topic, *args, **kwargs): """ Publishes the request to the bridge service :param topic: The topic to publish to :param args: The arguments :param kwargs: The key/word arguments :return: The ID of the publish """ assert topic is not None ext_url = kwargs.pop('transport_path', 'publish') response_hook = kwargs.pop('response_hook', None) params = {"topic": topic, "args": args, "kwargs": kwargs} response = self._make_api_call("POST", self.get_url(ext_url), json_params=params, response_hook=response_hook) if self.do_async is True: return response return response["id"] def call(self, procedure, *args, **kwargs): """ Calls a procedure from the bridge service :param topic: The topic to publish to :param args: The arguments :param kwargs: The key/word arguments :return: The response from calling the procedure """ assert procedure is not None ext_url = kwargs.pop('transport_path', 'call') response_hook = kwargs.pop('response_hook', None) params = {"procedure": procedure, "args": args, "kwargs": kwargs} response = self._make_api_call("POST", self.get_url(ext_url), json_params=params, response_hook=response_hook) if self.do_async is True: return response value = None if "args" in response and len(response["args"]) > 0: value = response["args"][0] if "error" in response: error = response["error"] if "wamp.error.no_such_procedure" in error: raise ClientNoCalleeRegistered(value) else: raise ClientCallRuntimeError(value) return value def _compute_signature(self, body): """ Computes the signature. Described at: http://crossbar.io/docs/HTTP-Bridge-Services-Caller/ Reference code is at: https://github.com/crossbario/crossbar/blob/master/crossbar/adapter/rest/common.py :return: (signature, none, timestamp) """ timestamp = datetime.datetime.utcnow().strftime( "%Y-%m-%dT%H:%M:%S.%fZ") nonce = randint(0, 2**53) # Compute signature: HMAC[SHA256]_{secret} (key | timestamp | seq | nonce | body) => signature hm = hmac.new(self.secret, None, hashlib.sha256) hm.update(self.key) hm.update(timestamp) hm.update(str(self.sequence)) hm.update(str(nonce)) hm.update(body) signature = base64.urlsafe_b64encode(hm.digest()) return signature, nonce, timestamp def _make_api_call(self, method, url, json_params=None, response_hook=None): """ Performs the REST API Call :param method: HTTP Method :param url: The URL :param json_params: The parameters intended to be JSON serialized :return: """ if self.verbose is True: print("\ncrossbarhttp: Request: %s %s" % (method, url)) if json_params is not None and self.verbose is True: print("crossbarhttp: Params: " + json.dumps(json_params)) if self.key is not None and self.secret is not None and json_params is not None: signature, nonce, timestamp = self._compute_signature( json.dumps(json_params)) params = urlencode({ "timestamp": timestamp, "seq": str(self.sequence), "nonce": nonce, "signature": signature, "key": self.key }) if self.verbose is True: print("crossbarhttp: Signature Params: " + params) url += "?" + params # TODO: I can't figure out what this is. Guessing it is a number you increment on every call self.sequence += 1 try: if self.do_async is True: return self.session.request(method, url=url, json=json_params, hooks={'response': response_hook}, **self.kwargs) response = requests.request(method, url=url, json=json_params, **self.kwargs) if response.status_code == 200: data = response.json() if self.verbose is True: print("crossbarhttp: Response: " + str(data)) return data elif response.status_code == 400: raise ClientMissingParams(str(response.text)) elif response.status_code == 401: raise ClientSignatureError(str(response.text)) else: raise ClientBadUrl(str(response.text)) except requests.exceptions.RequestException as e: raise ClientBadHost(str(e))
from polyswarmd.utils.response import success, failure, install_error_handlers logger = logging.getLogger(__name__) cache: Cache = Cache(config={"CACHE_TYPE": "simple", "CACHE_DEFAULT_TIMEOUT": 30}) # Set up our app object app = Flask(__name__) app.url_map.strict_slashes = False _config = PolySwarmd.auto() app.config['POLYSWARMD'] = _config # Setting this value works even when Content-Length is omitted, we must have it app.config['MAX_CONTENT_LENGTH'] = _config.artifact.max_size * _config.artifact.limit session = FuturesSession(executor=ThreadPoolExecutor(4), adapter_kwargs={'max_retries': 2}) session.request = functools.partial(session.request, timeout=10) app.config['REQUESTS_SESSION'] = session app.config['CHECK_BLOCK_LIMIT'] = True app.config['THREADPOOL'] = ThreadPoolExecutor() install_error_handlers(app) from polyswarmd.views.eth import misc from polyswarmd.views.artifacts import artifacts from polyswarmd.views.balances import balances from polyswarmd.views.bounties import bounties from polyswarmd.views.relay import relay from polyswarmd.views.offers import offers from polyswarmd.views.staking import staking from polyswarmd.views.event_message import init_websockets