def fetch_comments_parallelly(self, bug_ids: List[int], n_max_workers: int = 16) -> [Comment]: headers = { "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36", "Accept": "application/json" } def exception_handler(request, exception): print("ERROR: An exception occurred! {}".format(exception)) urls = list( map(lambda bug_id: "{}/{}/comment".format(self._base_url, bug_id), bug_ids)) n_workers = min(len(urls), n_max_workers) session = FuturesSession(max_workers=n_workers) futures = [session.get(u, headers=headers) for u in urls] done, incomplete = wait(futures) session.close() if len(incomplete) > 0: print("ERROR!!! Failed to process at least one request...") bug_comments = {} for f in done: r = f.result() r.raise_for_status() parsed_data = r.json() bugs_data = parsed_data["bugs"] included_bug_ids = list(bugs_data.keys()) assert (len(included_bug_ids) == 1) bug_id = included_bug_ids[0] bug_comments[int(bug_id)] = bugs_data[bug_id]["comments"] return bug_comments
def get(self, url_paths, timeout=300, stream=False, use_proxy=False): print(datetime.datetime.now()) url_paths = [u.replace('://www.', '://') for u in url_paths] session = FuturesSession(max_workers=self._max_workers) if use_proxy: futures = [ session.get(url, headers=_get_headers(self._ua_generator.random), proxies=_get_proxy(random.choice( self._proxy_list)), timeout=timeout, stream=stream) for url in url_paths ] else: futures = [ session.get(url, timeout=timeout, stream=stream) for url in url_paths ] scrapers = [] for f, u in zip(futures, url_paths): try: r = f.result() s = SCRAPERS[url_path_to_dict(u)['host']](r) except Exception as e: # Tmp: add logging here of failure print("FAILURE: %s", e) s = SCRAPERS[url_path_to_dict(u)['host']](None) scrapers.append(s) session.close() print(datetime.datetime.now()) return scrapers
async def getlyrics(artistsong): percents = {" ": "+", "!": "%21", '"': "%22", "#": "%23", "$": "%24", "%": "%25", "&": "%26", "'": "%27", "(": "%28", ")": "%29", "*": "%2A", "+": "%2B", "`": "%60", ",": "%2C", "-": "%2D", ".": "%2E", "/": "%2F"} searchquery = "" for char in artistsong: if char in percents: char = percents[char] searchquery += char session = FuturesSession() future = session.get("https://google.com/search?q=" + searchquery + "+lyrics") response_one = future.result() soup = BeautifulSoup(response_one.text, 'html.parser') bouncer = "Our systems have detected unusual traffic from your computer network" if bouncer in soup.get_text(): title_ = "" artist_ = "" lyrics_ = "Google has detected us being suspicious, try again later." source_ = "" else: try: title_ = soup.find('span', class_="BNeawe tAd8D AP7Wnd").get_text() artist_ = soup.find_all('span', class_="BNeawe s3v9rd AP7Wnd")[-1].get_text() lyrics_ = soup.find_all('div', class_="BNeawe tAd8D AP7Wnd")[-1].get_text() source_ = soup.find_all('span', class_="uEec3 AP7Wnd")[-1].get_text() except AttributeError: title_, artist_, lyrics_, source_ = "", "", "Not able to find the lyrics for {}.".format(searchquery), "" session.close() return title_, artist_, lyrics_, source_
class GraphDatabase(GraphDatabase): def __init__(self, url, timeout=None, username="", password="", loop=None, verify_ssl=False, future_class=None): super().__init__(url, timeout=timeout, username=username, password=password, loop=loop, validate_cert=verify_ssl, future_class=futures.Future) self._session = FuturesSession() def session(self): raise NotImplementedError def connect(self, session=None, force_close=False, force_release=False, pool=None): """ Get a connection to the graph database. :param str session: Session id (optional). Typically a uuid :param bool force_close: force connection to close after read. :param bool force_release: If possible, force release to pool after read. :param gremlinclient.pool.Pool pool: Associated connection pool. :returns: :py:class:`gremlinclient.connection.Connection` """ return self._connect(Connection, session, force_close, force_release, pool) def _connect(self, conn_type, session, force_close, force_release, pool): future = self._future_class() resp = Response(url, self._session, self._future_class, loop=self._loop) gc = conn_type(resp, self._future_class, self._timeout, self._username, self._password, self._loop, self._validate_cert, False, pool, force_release, session) future.set_result(gc) return future def close(self): self._session.close()
class GraphDatabase(GraphDatabase): def __init__(self, url, timeout=None, username="", password="", loop=None, verify_ssl=False, future_class=None): super().__init__(url, timeout=timeout, username=username, password=password, loop=loop, validate_cert=verify_ssl, future_class=futures.Future) self._session = FuturesSession() def session(self): raise NotImplementedError def connect(self, session=None, force_close=False, force_release=False, pool=None): """ Get a connection to the graph database. :param str session: Session id (optional). Typically a uuid :param bool force_close: force connection to close after read. :param bool force_release: If possible, force release to pool after read. :param gremlinclient.pool.Pool pool: Associated connection pool. :returns: :py:class:`gremlinclient.connection.Connection` """ return self._connect( Connection, session, force_close, force_release, pool) def _connect(self, conn_type, session, force_close, force_release, pool): future = self._future_class() resp = Response(url, self._session, self._future_class, loop=self._loop) gc = conn_type(resp, self._future_class, self._timeout, self._username, self._password, self._loop, self._validate_cert, False, pool, force_release, session) future.set_result(gc) return future def close(self): self._session.close()
def _async_request(self, method_name, store_id=None, json_data=None): tasks = [] session = FuturesSession() session.headers['Content-Type'] = 'application/json' for store in self.stores: func = getattr(store, method_name) request_task = RequestTask(store, func(session, json_data=json_data, store_id=store_id), method_name) tasks.append(request_task) results = [] for task in tasks: result = self._get_result_object(task, method_name) results.append(result) print(result) session.close() return results
class Tracker: def __init__(self, app_id: str) -> None: self._app_id = app_id self._session = FuturesSession() def send_async(self, teacher: Teacher) -> Future: payload = { "v": 1, "tid": "UA-2241989-17", "cid": 555, "t": "pageview", "dh": self._app_id, "dp": teacher.id, "dt": teacher.name, } return self._session.post("http://www.google-analytics.com/collect", payload) def close(self) -> None: self._session.close()
def makeAsyncLoop(links): # Распредилитель изображений video_capture = cv2.VideoCapture(0) sess = FuturesSession() counter = 0 scale = 1 print(str(links) + ' - links are looped') while True: if len(links) == 0: return 0 ret, frame = video_capture.read() # Чтение кадра с камеры small_frame = cv2.resize(frame, (0, 0), fx=scale, fy=scale) # При небходимости, уменьшение разрешения кадра img = pickle.dumps(small_frame) # Сериализация изображения # try: sess.post(links[counter] + 'handle', data=img) # Попытка отправить следующий кадр # except: # links.remove(links[counter]) # Удалить ссылку из списка если она не доступна counter += 1 if counter > len(links) - 1: counter = 0 sess.close() sess = FuturesSession()
async def getlyrics(artistsong): percents = {" ": "+", "!": "%21", '"': "%22", "#": "%23", "$": "%24", "%": "%25", "&": "%26", "'": "%27", "(": "%28", ")": "%29", "*": "%2A", "+": "%2B", "`": "%60", ",": "%2C", "-": "%2D", ".": "%2E", "/": "%2F"} searchquery = "" for char in artistsong: if char in percents.keys(): char = percents[char] searchquery += char session = FuturesSession() future = session.get("https://google.com/search?q=" + searchquery + "+lyrics") response_one = future.result() soup = BeautifulSoup(response_one.text, 'html.parser') try: title_ = soup.find('span', class_="BNeawe tAd8D AP7Wnd").get_text() artist_ = soup.find_all('span', class_="BNeawe s3v9rd AP7Wnd")[-1].get_text() lyrics_ = soup.find_all('div', class_="BNeawe tAd8D AP7Wnd")[-1].get_text() source_ = soup.find_all('span', class_="uEec3 AP7Wnd")[-1].get_text() except AttributeError: title_, artist_, lyrics_, source_ = "Not Found: {}".format(artistsong), "Not Found: {}".format(artistsong), \ "Not Found: {}".format(artistsong), "Not Found: {}".format(artistsong) session.close() return title_, artist_, lyrics_, source_
class TelegramNotifier(_MetricMonitorCallback): def __init__(self, bot_id: str, chat_id: str, metric_name: str, delta, max_workers: int = 1): self.bot_id = bot_id self.chat_id = chat_id self.metric_name = metric_name self.session = FuturesSession(max_workers=max_workers) self.futures = [] self.delta = delta self.previous_metric = np.inf def on_epoch_end(self, epoch: int, logs: ModelHistory) -> None: def _response_hook(resp, *args, **kwargs): if resp.status_code != 200: logging.warning('Failed to deliver Telegram message with ' + f'error code {resp.status_code}') current_metric = self.get_metric('patience', self.metric_name, logs) current_delta = self.previous_metric - current_metric if current_delta >= self.delta: msg = '{} improved from%20{}%20to%20{}'.format( self.metric_name, self.previous_metric, current_metric) msg_url = 'https://api.telegram.org/bot{}/sendMessage?chat_id={}&text={}'.format( self.bot_id, self.chat_id, msg) future = self.session.get(msg_url, hooks={'response': _response_hook}) self.futures.append(future) self.previous_metric = current_metric def on_train_end(self) -> None: self.session.close()
class BaseConnection(object): """Base Connection Class.""" def __init__(self, debug=False, method='GET', proxy_host=None, timeout=20, proxy_port=80, parallel=None, escape_xml=False, **kwargs): if debug: set_stream_logger() self.response = None self.request = None self.verb = None self.config = None self.debug = debug self.method = method self.timeout = timeout self.proxy_host = proxy_host self.proxy_port = proxy_port self.escape_xml = escape_xml self.datetime_nodes = [] self._list_nodes = [] self.proxies = dict() if self.proxy_host: proxy = 'http://%s:%s' % (self.proxy_host, self.proxy_port) self.proxies = {'http': proxy, 'https': proxy} self.session = FuturesSession() self.session.mount('http://', HTTPAdapter(max_retries=3)) self.session.mount('https://', HTTPAdapter(max_retries=3)) self.parallel = parallel self.base_list_nodes = [] self.datetime_nodes = [] self._reset() def debug_callback(self, debug_type, debug_message): log.debug('type: ' + str(debug_type) + ' message' + str(debug_message)) def v(self, *args, **kwargs): return getValue(self.response.dict(), *args, **kwargs) def getNodeText(self, nodelist): return getNodeTextUtils(nodelist) def _reset(self): self.response = None self.request = None self.verb = None self._list_nodes = [] self._request_id = None self._request_dict = {} self._time = time.time() self._response_content = None self._response_dom = None self._response_obj = None self._response_soup = None self._response_dict = None self._response_error = None self._resp_body_errors = [] self._resp_body_warnings = [] self._resp_codes = [] def _add_prefix(self, nodes, verb): if verb: for i, v in enumerate(nodes): if not nodes[i].startswith(verb.lower()): nodes[i] = "%sresponse.%s" % (verb.lower(), nodes[i].lower()) def execute(self, verb, data=None, list_nodes=[], verb_attrs=None, files=None): "Executes the HTTP request." log.debug('execute: verb=%s data=%s' % (verb, data)) self._reset() self._list_nodes += list_nodes self._add_prefix(self._list_nodes, verb) if hasattr(self, 'base_list_nodes'): self._list_nodes += self.base_list_nodes self.build_request(verb, data, verb_attrs, files) self.execute_request() if hasattr(self.response, 'content'): self.process_response() self.error_check() log.debug('total time=%s' % (time.time() - self._time)) return self.response def build_request(self, verb, data, verb_attrs, files=None): self.verb = verb self._request_dict = data self._request_id = uuid.uuid4() url = self.build_request_url(verb) headers = self.build_request_headers(verb) headers.update({ 'User-Agent': UserAgent, 'X-EBAY-SDK-REQUEST-ID': str(self._request_id) }) # if we are adding files, we ensure there is no Content-Type header already defined # otherwise Request will use the existing one which is likely not to be multipart/form-data # data must also be a dict so we make it so if needed requestData = self.build_request_data(verb, data, verb_attrs) if files: del (headers['Content-Type']) if isinstance(requestData, str): # pylint: disable-msg=E0602 requestData = {'XMLPayload': requestData} request = Request( self.method, url, data=smart_encode_request_data(requestData), headers=headers, files=files, ) self.request = request.prepare() def build_request_headers(self, verb): return {} def build_request_data(self, verb, data, verb_attrs): return "" def build_request_url(self, verb): url = "%s://%s%s" % (HTTP_SSL[self.config.get('https', False)], self.config.get('domain'), self.config.get('uri')) return url def execute_request(self): log.debug("REQUEST (%s): %s %s" % (self._request_id, self.request.method, self.request.url)) log.debug('headers=%s' % self.request.headers) log.debug('body=%s' % self.request.body) if self.parallel: self.parallel._add_request(self) return None self.response = self.session.send(self.request, verify=True, proxies=self.proxies, timeout=self.timeout, allow_redirects=True) log.debug('RESPONSE (%s):' % self._request_id) log.debug('elapsed time=%s' % self.response.elapsed) log.debug('status code=%s' % self.response.status_code) log.debug('headers=%s' % self.response.headers) log.debug('content=%s' % self.response.text) def process_response(self, parse_response=True): """Post processing of the response""" self.response = Response(self.response, verb=self.verb, list_nodes=self._list_nodes, datetime_nodes=self.datetime_nodes, parse_response=parse_response) self.session.close() # set for backward compatibility self._response_content = self.response.content if self.response.status_code != 200: self._response_error = self.response.reason def error_check(self): estr = self.error() if estr and self.config.get('errors', True): log.error(estr) raise ConnectionError(estr, self.response) def response_codes(self): return self._resp_codes def response_status(self): "Retuns the HTTP response status string." return self.response.reason def response_code(self): "Returns the HTTP response status code." return self.response.status_code def response_content(self): return self.response.content def response_soup(self): "Returns a BeautifulSoup object of the response." if not self._response_soup: try: from bs4 import BeautifulStoneSoup except ImportError: from BeautifulSoup import BeautifulStoneSoup log.warn( 'DeprecationWarning: BeautifulSoup 3 or earlier is deprecated; install bs4 instead\n' ) self._response_soup = BeautifulStoneSoup( smart_decode(self.response_content)) return self._response_soup def response_obj(self): log.warn('response_obj() DEPRECATED, use response.reply instead') return self.response.reply def response_dom(self): """ Deprecated: use self.response.dom() instead Returns the response DOM (xml.dom.minidom). """ log.warn('response_dom() DEPRECATED, use response.dom instead') if not self._response_dom: dom = None content = None try: if self.response.content: regex = re.compile(b'xmlns="[^"]+"') content = regex.sub(b'', self.response.content) else: content = "<%sResponse></%sResponse>" % (self.verb, self.verb) dom = parseString(content) self._response_dom = dom.getElementsByTagName(self.verb + 'Response')[0] except ExpatError as e: raise ConnectionResponseError( "Invalid Verb: %s (%s)" % (self.verb, e), self.response) except IndexError: self._response_dom = dom return self._response_dom def response_dict(self): "Returns the response dictionary." log.warn( 'response_dict() DEPRECATED, use response.dict() or response.reply instead' ) return self.response.reply def response_json(self): "Returns the response JSON." log.warn('response_json() DEPRECATED, use response.json() instead') return self.response.json() def _get_resp_body_errors(self): """Parses the response content to pull errors. Child classes should override this method based on what the errors in the XML response body look like. They can choose to look at the 'ack', 'Errors', 'errorMessage' or whatever other fields the service returns. the implementation below is the original code that was part of error() """ if self._resp_body_errors and len(self._resp_body_errors) > 0: return self._resp_body_errors errors = [] if self.verb is None: return errors dom = self.response.dom() if dom is None: return errors return [] def error(self): "Builds and returns the api error message." error_array = [] if self._response_error: error_array.append(self._response_error) error_array.extend(self._get_resp_body_errors()) if len(error_array) > 0: # Force all errors to be unicode in a proper way error_array = [smart_decode(smart_encode(e)) for e in error_array] error_string = u"{verb}: {message}".format( verb=self.verb, message=u", ".join(error_array)) return error_string return None def opendoc(self): webbrowser.open(self.config.get('doc_url'))
class HTTPDriver(BaseDriver): """HTTPDriver The :class:`HTTPDriver` class reads SBP messages from an HTTP service for a device and writes out to a stream. This driver is like a file-handle with read and writes over two separately HTTP connections, but can also be enabled and disabled by its consumer. Parameters ---------- device_uid : uid Device unique id url : str HTTP endpoint retries : tuple Configure connect and read retry count. Defaults to (MAX_CONNECT_RETRIES, MAX_READ_RETRIES). timeout : tuple Configure connect and read timeouts. Defaults to (DEFAULT_CONNECT_TIMEOUT, DEFAULT_READ_TIMEOUT). """ def __init__( self, device_uid=None, url="https://broker.staging.skylark.swiftnav.com", retries=DEFAULT_RETRIES, timeout=DEFAULT_TIMEOUT, ): self._retry = Retry(connect=DEFAULT_RETRIES[0], read=DEFAULT_RETRIES[1], redirect=MAX_REDIRECTS, status_forcelist=[500], backoff_factor=DEFAULT_BACKOFF_FACTOR) self.url = url self.read_session = requests.Session() self.read_session.mount( "http://", HTTPAdapter(pool_connections=DEFAULT_POOLSIZE, pool_maxsize=DEFAULT_POOLSIZE, pool_block=DEFAULT_POOLBLOCK, max_retries=self._retry)) self.read_session.mount( "https://", HTTPAdapter(pool_connections=DEFAULT_POOLSIZE, pool_maxsize=DEFAULT_POOLSIZE, pool_block=DEFAULT_POOLBLOCK, max_retries=self._retry)) self.write_session = None self.device_uid = device_uid self.timeout = timeout self.read_response = None self.write_response = None self.source = None def flush(self): """File-flush wrapper (noop). """ pass def close(self): """File-handle close wrapper (noop). """ try: self.read_close() self.write_close() except: pass @property def write_ok(self): """ Are we connected for writes? """ # Note that self.write_response is either None or a Response # object, which cast to False for 4xx and 5xx HTTP codes. return bool(self.write_response) def connect_write(self, source, whitelist, device_uid=None, pragma=None): """Initialize a streaming write HTTP response. Manually connects the underlying file-handle. In the event of a network disconnection, use to manually reinitiate an HTTP session. Parameters ---------- source : sbp.client.handler.Handler Iterable source of SBP messages. whitelist : [int] Whitelist of messages to write """ header_device_uid = device_uid or self.device_uid headers = { 'Device-Uid': header_device_uid, 'Content-Type': BROKER_SBP_TYPE, 'Pragma': pragma } if not pragma: del headers['Pragma'] try: self.executor = ThreadPoolExecutor(max_workers=DEFAULT_POOLSIZE) self.write_session = FuturesSession(executor=self.executor) self.write_session.mount( "http://", HTTPAdapter(pool_connections=DEFAULT_POOLSIZE, pool_maxsize=DEFAULT_POOLSIZE, pool_block=DEFAULT_POOLBLOCK, max_retries=self._retry)) self.write_session.mount( "https://", HTTPAdapter(pool_connections=DEFAULT_POOLSIZE, pool_maxsize=DEFAULT_POOLSIZE, pool_block=DEFAULT_POOLBLOCK, max_retries=self._retry)) self.source = source.filter(whitelist) gen = (msg.pack() for msg, _ in self.source) self.write_session.put(self.url, data=gen, headers=headers) self.write_response = True except requests.exceptions.ConnectionError: msg = "Client connection error to %s with [PUT] headers %s" \ % (self.url, headers) warnings.warn(msg) except requests.exceptions.ConnectTimeout: msg = "Client connection timeout to %s with [PUT] headers %s" \ % (self.url, headers) warnings.warn(msg) except requests.exceptions.RetryError: msg = "Client retry error to %s with [PUT] headers %s" \ % (self.url, headers) warnings.warn(msg) except requests.exceptions.ReadTimeout: msg = "Client read timeout to %s with [PUT] headers %s" \ % (self.url, headers) warnings.warn(msg) return self.write_ok def write(self, data): """Write wrapper (noop). Actual stream is initiated by the write connection. Parameters ---------- data : object Data to write. """ pass def write_close(self): """File-handle close wrapper (noop). """ try: self.write_session.close() self.executor.shutdown(wait=False) self.source.breakiter() self.source = None self.executor = None self.write_session = None except: pass @property def read_ok(self): """ Are we connected for reads? """ return bool(self.read_response) def connect_read(self, device_uid=None, pragma=None): """Initialize a streaming read/write HTTP response. Manually connects the underlying file-handle. In the event of a network disconnection, use to manually reinitiate an HTTP session. """ header_device_uid = device_uid or self.device_uid headers = { 'Device-Uid': header_device_uid, 'Accept': BROKER_SBP_TYPE, 'Pragma': pragma } if not pragma: del headers['Pragma'] try: self.read_response = self.read_session.get(self.url, stream=True, headers=headers, timeout=self.timeout) except requests.exceptions.ConnectionError: msg = "Client connection error to %s with [GET] headers %s" \ % (self.url, headers) warnings.warn(msg) except requests.exceptions.ConnectTimeout: msg = "Client connection timeout to %s with [GET] headers %s" \ % (self.url, headers) warnings.warn(msg) except requests.exceptions.RetryError: msg = "Client retry error to %s with [GET] headers %s" \ % (self.url, headers) warnings.warn(msg) except requests.exceptions.ReadTimeout: msg = "Client read timeout to %s with [GET] headers %s" \ % (self.url, headers) warnings.warn(msg) return self.read_ok def read(self, size): """Read wrapper. If the client connection is closed or some other exception is thrown, raises an IOError. Parameters ---------- size : int Size to read (in bytes). Returns ---------- bytearray, or None """ if self.read_response is None or not self.device_uid: raise ValueError("Invalid/insufficient HTTP request parameters!") elif not self.read_ok or self.read_response.raw.closed: raise IOError("HTTP read closed?!") try: return self.read_response.raw.read(size) except: raise IOError("HTTP read error!") def read_close(self): """File-handle close wrapper (noop). """ try: self.read_response.close() self.read_response = None except: pass
class HTTPDriver(BaseDriver): """HTTPDriver The :class:`HTTPDriver` class reads SBP messages from an HTTP service for a device and writes out to a stream. This driver is like a file-handle with read and writes over two separately HTTP connections, but can also be enabled and disabled by its consumer. Parameters ---------- device_uid : uid Device unique id url : str HTTP endpoint retries : tuple Configure connect and read retry count. Defaults to (MAX_CONNECT_RETRIES, MAX_READ_RETRIES). timeout : tuple Configure connect and read timeouts. Defaults to (DEFAULT_CONNECT_TIMEOUT, DEFAULT_READ_TIMEOUT). """ def __init__(self, device_uid=None, url="https://broker.staging.skylark.swiftnav.com", retries=DEFAULT_RETRIES, timeout=DEFAULT_TIMEOUT,): self._retry = Retry(connect=DEFAULT_RETRIES[0], read=DEFAULT_RETRIES[1], redirect=MAX_REDIRECTS, status_forcelist=[500], backoff_factor=DEFAULT_BACKOFF_FACTOR) self.url = url self.read_session = requests.Session() self.read_session.mount("http://", HTTPAdapter(pool_connections=DEFAULT_POOLSIZE, pool_maxsize=DEFAULT_POOLSIZE, pool_block=DEFAULT_POOLBLOCK, max_retries=self._retry)) self.read_session.mount("https://", HTTPAdapter(pool_connections=DEFAULT_POOLSIZE, pool_maxsize=DEFAULT_POOLSIZE, pool_block=DEFAULT_POOLBLOCK, max_retries=self._retry)) self.write_session = None self.device_uid = device_uid self.timeout = timeout self.read_response = None self.write_response = None self.source = None def flush(self): """File-flush wrapper (noop). """ pass def close(self): """File-handle close wrapper (noop). """ try: self.read_close() self.write_close() except: pass @property def write_ok(self): """ Are we connected for writes? """ # Note that self.write_response is either None or a Response # object, which cast to False for 4xx and 5xx HTTP codes. return bool(self.write_response) def connect_write(self, source, whitelist, device_uid=None, pragma=None): """Initialize a streaming write HTTP response. Manually connects the underlying file-handle. In the event of a network disconnection, use to manually reinitiate an HTTP session. Parameters ---------- source : sbp.client.handler.Handler Iterable source of SBP messages. whitelist : [int] Whitelist of messages to write """ header_device_uid = device_uid or self.device_uid headers = {'Device-Uid': header_device_uid, 'Content-Type': BROKER_SBP_TYPE, 'Pragma': pragma} if not pragma: del headers['Pragma'] try: self.executor = ThreadPoolExecutor(max_workers=DEFAULT_POOLSIZE) self.write_session = FuturesSession(executor=self.executor) self.write_session.mount("http://", HTTPAdapter(pool_connections=DEFAULT_POOLSIZE, pool_maxsize=DEFAULT_POOLSIZE, pool_block=DEFAULT_POOLBLOCK, max_retries=self._retry)) self.write_session.mount("https://", HTTPAdapter(pool_connections=DEFAULT_POOLSIZE, pool_maxsize=DEFAULT_POOLSIZE, pool_block=DEFAULT_POOLBLOCK, max_retries=self._retry)) self.source = source.filter(whitelist) gen = (msg.pack() for msg, _ in self.source) self.write_session.put(self.url, data=gen, headers=headers) self.write_response = True except requests.exceptions.ConnectionError as err: msg = "Client connection error to %s with [PUT] headers %s: msg=%s" \ % (self.url, headers, err.message) warnings.warn(msg) except requests.exceptions.ConnectTimeout as err: msg = "Client connection timeout to %s with [PUT] headers %s: msg=%s" \ % (self.url, headers, err.message) warnings.warn(msg) except requests.exceptions.RetryError: msg = "Client retry error to %s with [PUT] headers %s: msg=%s" \ % (self.url, headers, err.message) warnings.warn(msg) except requests.exceptions.ReadTimeout: msg = "Client read timeout to %s with [PUT] headers %s: msg=%s" \ % (self.url, headers, err.message) warnings.warn(msg) return self.write_ok def write(self, data): """Write wrapper (noop). Actual stream is initiated by the write connection. Parameters ---------- data : object Data to write. """ pass def write_close(self): """File-handle close wrapper (noop). """ try: self.write_session.close() self.executor.shutdown(wait=False) self.source.breakiter() self.source = None self.executor = None self.write_session = None except: pass @property def read_ok(self): """ Are we connected for reads? """ return bool(self.read_response) def connect_read(self, device_uid=None, pragma=None): """Initialize a streaming read/write HTTP response. Manually connects the underlying file-handle. In the event of a network disconnection, use to manually reinitiate an HTTP session. """ header_device_uid = device_uid or self.device_uid headers = {'Device-Uid': header_device_uid, 'Accept': BROKER_SBP_TYPE, 'Pragma': pragma} if not pragma: del headers['Pragma'] try: self.read_response = self.read_session.get(self.url, stream=True, headers=headers, timeout=self.timeout) except requests.exceptions.ConnectionError as err: msg = "Client connection error to %s with [GET] headers %s: msg=%s" \ % (self.url, headers, err.message) warnings.warn(msg) except requests.exceptions.ConnectTimeout as err: msg = "Client connection timeout to %s with [GET] headers %s: msg=%s" \ % (self.url, headers, err.message) warnings.warn(msg) except requests.exceptions.RetryError: msg = "Client retry error to %s with [GET] headers %s: msg=%s" \ % (self.url, headers, err.message) warnings.warn(msg) except requests.exceptions.ReadTimeout: msg = "Client read timeout to %s with [GET] headers %s: msg=%s" \ % (self.url, headers, err.message) warnings.warn(msg) return self.read_ok def read(self, size): """Read wrapper. If the client connection is closed or some other exception is thrown, raises an IOError. Parameters ---------- size : int Size to read (in bytes). Returns ---------- bytearray, or None """ if self.read_response is None or not self.device_uid: raise ValueError("Invalid/insufficient HTTP request parameters!") elif not self.read_ok or self.read_response.raw.closed: raise IOError("HTTP read closed?!") try: return self.read_response.raw.read(size) except: raise IOError("HTTP read error!") def read_close(self): """File-handle close wrapper (noop). """ try: self.read_response.close() self.read_response = None except: pass
class MozDefMessage(object): # Supported message types MSGTYPE_NONE = 0 MSGTYPE_EVENT = 1 MSGTYPE_COMPLIANCE = 2 MSGTYPE_VULNERABILITY = 3 MSGTYPE_ASSETHINT = 4 MSGTYPE_RRA = 5 def __init__(self, url): """This class is the new base class for MozDef messages. All other classes besides MozDefMsg derive from this class or from classes derived from this class (like MozDevEvent). This class shouldn't be used directly and the derived classes should be used instead. Note the very similar name between this class and the MozDefMsg class but the differing purposes between the two classes (see the MozDefMsg docstring) """ self._msgtype = self.MSGTYPE_NONE self.log = {} self._sendlog = {} self._httpsession = Session() self._httpsession.trust_env = False self._httpsession.hooks['response'].append(self._httpsession_cb) self._url = url self.hostname = socket.getfqdn() # This is due to some systems incorrectly # setting the hostname field to localhost.localdomain # so, we add logic to use a different 'hostname' method # if that's the case if self.hostname == 'localhost.localdomain': self.hostname = socket.gethostname() # Set some default options self._send_to_syslog = False self._send_to_sqs = False self._syslog_only = False self._fire_and_forget = False self._verify_certificate = False self._verify_path = None def __del__(self): ''' Close out any Sessions we started. ''' self._httpsession.close() def validate(self): return True def validate_log(self): return True def set_verify(self, f): self._verify_certificate = f def set_verify_path(self, p): self._verify_path = p def set_fire_and_forget(self, f): self._fire_and_forget = f def set_sqs_queue_name(self, f): self._sqs_queue_name = f def set_sqs_aws_account_id(self, f): self._sqs_aws_account_id = f def set_sqs_region(self, f): self._sqs_region = f def set_send_to_sqs(self, f): self._send_to_sqs = f def set_send_to_syslog(self, f, only_syslog=False): self._send_to_syslog = f self._syslog_only = only_syslog def syslog_convert(self): raise MozDefError('message type does not support syslog conversion') def construct(self): raise MozDefError('subclass of MozDefMessage must override construct()') def _httpsession_cb(self, session, response): if response.result().status_code != 200: if not self._fire_and_forget: raise MozDefError('POST failed with code %r' % \ response.result().status_code) def send_syslog(self): raise MozDefError('message type does not support syslog submission') def send(self): if not self.validate(): raise MozDefError('message failed validation') self.construct() if not self.validate_log(): raise MozDefError('message failed post construct validation') if self._send_to_syslog: self.send_syslog() if self._syslog_only: return if self._send_to_sqs: self.send_sqs() return vflag = self._verify_certificate if vflag: if self._verify_path != None: vflag = self._verify_path buf = json.dumps(self._sendlog, sort_keys=True, indent=4) # Compatibility notes: # When updating either path (futures_loaded or not loaded) please ensure both have the same functionality # future_loaded is used by Python 2, the non-loaded version if for Python 3 if futures_loaded: self._httpsession.post(self._url, buf, verify=vflag) else: response = self._httpsession.post(self._url, buf, verify=vflag) if response.ok == False: if not self._fire_and_forget: raise MozDefError('POST failed with code %r msg %s' % \ (response.status_code, response.text))
class ManagedPersister(Persister): def __init__(self, workflow_name: str, wf_start_time: float, service_url: str, wf_exec_id=None, context: str = None, with_validation: bool = False, db_name: str = None, bag_size: int = 1, should_send_to_file: bool = False, should_send_to_service: bool = True): super().__init__(workflow_name, wf_start_time, wf_exec_id) self.retrospective_url = urljoin(service_url, "retrospective-provenance") self.prospective_url = urljoin(service_url, "prospective-provenance") self.context = context self.with_validation = with_validation self.db_name = db_name self.requests_queue = list() self.bag_size = bag_size self.should_send_to_service = should_send_to_service self.should_send_to_file = should_send_to_file self.session = None if self.should_send_to_service: logger.debug("You are using the Service URL: " + service_url) self.session = FuturesSession() def add_request(self, persistence_request: ProvRequestObj): try: request_data = persistence_request.as_dict() if self.context: request_data["context"] = self.context self.requests_queue.append(request_data) if len(self.requests_queue) >= self.bag_size: self._flush() # if `configuration` is present this object should be persisted synchronously # if "configuration" in prov_obj: # self.__flush__(True) except Exception: logger.error("[Prov] Unexpected exception") traceback.print_exc() pass def _close(self): if self.session: logger.info( "Waiting to get response from all submitted provenance tasks..." ) while not self.session.executor._work_queue.empty(): # wait to guarantee that all provenance requests have been sent (fired) to collector service sleep(0.1) # Persist remaining tasks synchronously self._flush(all_and_wait=True) if self.session: self.session.close() def _flush(self, all_and_wait: bool = False): if len(self.requests_queue) > 0: if all_and_wait: logger.debug("Going to flush everything. Flushing " + str(len(self.requests_queue))) if self.should_send_to_file: offline_prov_log.debug(json.dumps(self.requests_queue)) if self.should_send_to_service: self._send_to_service(self.requests_queue) self.requests_queue = list() else: to_flush = self.requests_queue[:self.bag_size] del self.requests_queue[:self.bag_size] logger.debug("Going to flush a part. Flushing " + str(len(to_flush)) + " out of " + str(len(self.requests_queue))) if self.should_send_to_file: offline_prov_log.debug(json.dumps(to_flush)) if self.should_send_to_service: self._send_to_service(to_flush) def _send_to_service(self, to_flush: List[dict]): params = { "with_validation": str(self.with_validation), "db_name": self.db_name } try: logger.debug("[Prov-Persistence]" + json.dumps(to_flush)) # TODO: check whether we need this result() below r = self.session.post(self.retrospective_url, json=to_flush, params=params, verify=False).result() except ConnectionError as ex: logger.error( "[Prov][ConnectionError] There is a communication error between client and server -> " + str(ex)) r = None pass except Exception as ex: traceback.print_exc() logger.error( "[Prov] Unexpected exception while adding retrospective provenance: " + type(ex).__name__ + "->" + str(ex)) r = None pass # If requests were validated, check for errors if r and self.with_validation: self._log_validation_message(r) def persist_prospective(self, json_data: dict): try: if self.should_send_to_file: offline_prov_log.debug(json.dumps(self.requests_queue)) if self.should_send_to_service: logger.debug("[Prov-Persistence][Prospective]" + json.dumps(json_data)) try: r = self.session.post(self.prospective_url, json=json_data, params={ 'overwrite': True }, verify=False).result() if 200 <= r.status_code <= 209: logger.debug( "Prospective provenance inserted successfully.") elif r.status_code == 406: error_parsed = json.loads(r._content.decode('utf-8')) error_obj = error_parsed['error'].replace("'", '"') logger.error(error_obj) elif r.status_code == 500: r = self.session.put(self.prospective_url, json=json_data).result() try: assert 200 <= r.status_code <= 209 except AssertionError: logger.error( "Prospective provenance was not inserted correctly. Status code = " + str(r.status_code)) elif r.status_code > 300: logger.error( "Prospective provenance was not inserted correctly. Status code = " + str(r.status_code)) except ConnectionError as ex: traceback.print_exc() logger.error( "[Prov][ConnectionError] There is a communication error between client and server -> " + str(ex)) pass except Exception as ex: logger.error( "[Prov] Unexpected exception while adding prospective provenance: " + type(ex).__name__) pass except Exception as ex: logger.error("[Prov] Unexpected exception " + type(ex).__name__) traceback.print_exc() pass @staticmethod def _log_validation_message(response): error_obj = json.loads(response._content.decode('utf-8')) if len(error_obj['error']) > 0: for error_list in error_obj['error']: for error in error_list: if error['code'][0] == 'W': logger.warning('{} {}{}'.format( error['type'], error['explanation'], '\n')) else: logger.error('{} {}{}'.format(error['type'], error['explanation'], '\n'))
def run(self): settings = QSettings() pref_target_path = settings.value(Settings.SETTINGS_SAVE_PATH, Settings.DEFAULT_TARGET_PATH, type=str) pref_max_pool_cnt = settings.value(Settings.SETTINGS_MAX_POOL_CNT, Settings.DEFAULT_MAX_POOL, type=int) gallery_save_path = pref_target_path+'/'+self.gallery.path if not os.path.exists(gallery_save_path): os.makedirs(gallery_save_path) # Cloudflare Authorization self.state.emit('Authorize..') Logger.LOGGER.info("Wait for Cloudflare Authorization..") self.driver.get(URL_HIYOBI) while "Just a moment..." in self.driver.page_source: pass user_agent = self.driver.execute_script("return navigator.userAgent;") try: cookie_value = '__cfduid=' + self.driver.get_cookie('__cfduid')['value'] + \ '; cf_clearance=' + self.driver.get_cookie('cf_clearance')['value'] headers = {'User-Agent': user_agent} cookies = {'session_id': cookie_value} except TypeError: Logger.LOGGER.warning("Not apply cookies to requests") headers = None cookies = None # Fetch image data from gallery page self.state.emit('Fetch..') Logger.LOGGER.info("Connect to Gallery page..") self.driver.get(self.gallery.url) sleep(1) soup = BeautifulSoup(self.driver.page_source, "html.parser") # Start download multi-thread Logger.LOGGER.info("Download Start..") img_urls = soup.find_all('div', class_="img-url") self.total_cnt = len(img_urls) session = FuturesSession(max_workers=pref_max_pool_cnt) if headers is not None: session.headers = headers if cookies is not None: session.cookies = cookies responses = {} for url_path in img_urls: url = READER_URL+url_path.text name = url.split('/')[-1] responses[name] = session.get(url) for filename in responses: self.response_to_file(response=responses[filename].result(), name=filename, path=gallery_save_path) session.close() # Compress Zip Files self.state.emit('Compressing..') if self.gallery.original != "": zip_path = pref_target_path+'/'+self.gallery.type+'/'+self.gallery.original+'/'+self.gallery.path+'.zip' else: zip_path = pref_target_path+'/'+self.gallery.type+'/'+self.gallery.path+'.zip' try: if not os.path.exists(zip_path[:zip_path.rfind('/')]): os.makedirs(zip_path[:zip_path.rfind('/')]) FileUtil.make_zip(gallery_save_path, zip_path) shutil.rmtree(gallery_save_path) except: print(traceback.format_exc()) Logger.LOGGER.error("Compressing Process Error... pass") # Save to Firebase # TODO Enable next line on Build FirebaseClient.fbclient.insert_data(self.gallery)
class THttpClient(TTransportBase): '''Http implementation of TTransport base.''' def __init__(self, uri_or_host, port=None, path=None, customThrift=False, request='httplib', http2=False, proxy_host=None, proxy_port=None, proxy_auth=None): '''THttpClient supports two different types constructor parameters. THttpClient(host, port, path) - deprecated THttpClient(uri) Only the second supports https. ''' if port is not None: warnings.warn( 'Please use the THttpClient("http://host:port/path") syntax', DeprecationWarning, stacklevel=2 ) self.host = uri_or_host self.port = port assert path self.path = path self.scheme = 'http' else: parsed = urllib.parse.urlparse(uri_or_host) self.scheme = parsed.scheme assert self.scheme in ('http', 'https') if self.scheme == 'http': self.port = parsed.port or http_client.HTTP_PORT elif self.scheme == 'https': self.port = parsed.port or http_client.HTTPS_PORT self.host = parsed.hostname self.path = parsed.path if parsed.query: self.path += '?%s' % parsed.query proxy = None self.request = request self.http2 = http2 self.realhost = proxy_host self.realport = proxy_port self.proxy_auth = proxy_auth self.__wbuf = BytesIO() if self.scheme == 'https' and self.using_proxy() and self.proxy_auth: self.proxy_headers = {'Proxy-Authorization': self.proxy_auth} else: self.proxy_headers = None self.url = '%s://%s:%s%s' % (self.scheme, self.host, self.port, self.path) if customThrift: if self.request == 'hyper': if self.http2: self.__http = hyper.HTTP20Connection(self.host, self.port, proxy_host=self.realhost, proxy_port=self.realport, proxy_headers=self.proxy_headers) else: self.__http = hyper.HTTPConnection(self.host, self.port, proxy_host=self.realhost, proxy_port=self.realport, proxy_headers=self.proxy_headers) elif self.request == 'httpx': if self.http2: self.__http = httpx.AsyncClient(base_url='%s://%s' % (self.scheme, self.host), http2=self.http2) else: self.__http = httpx.Client(base_url='%s://%s' % (self.scheme, self.host)) elif self.request == 'requests': self.__http = requests.Session() if self.using_proxy(): self.__http.proxies = urllib.request.getproxies() elif self.request == 'requests-futures': self.__http = FuturesSession() if self.using_proxy(): self.__http.proxies = urllib.request.getproxies() elif self.request == 'httplib2': self.__http = httplib2.Http() else: if self.scheme == 'http': self.__http = http_client.HTTPConnection(self.host, self.port) elif self.scheme == 'https': self.__http = http_client.HTTPSConnection(self.host, self.port) if self.using_proxy(): self.__http.set_tunnel(self.realhost, self.realport, self.proxy_headers) else: self.__http = None self.__async_loop = asyncio.get_event_loop() if self.request == 'httpx' and self.http2 else None self.__http_response = None self.__response_data = None self.__last_read = 0 self.__timeout = None self.__custom_headers = None self.__time = time.time() self.__custom_thrift = customThrift self.__loop = 0 @staticmethod def basic_proxy_auth_header(proxy): if proxy is None or not proxy.username: return None ap = '%s:%s' % (urllib.parse.unquote(proxy.username), urllib.parse.unquote(proxy.password)) cr = base64.b64encode(ap).strip() return 'Basic ' + cr def using_proxy(self): return self.realhost is not None def open(self): if self.request == 'hyper': if self.http2: self.__http = hyper.HTTP20Connection(self.host, self.port, proxy_host=self.realhost, proxy_port=self.realport, proxy_headers=self.proxy_headers) else: self.__http = hyper.HTTPConnection(self.host, self.port, proxy_host=self.realhost, proxy_port=self.realport, proxy_headers=self.proxy_headers) elif self.request == 'httpx': if self.http2: self.__http = httpx.AsyncClient(base_url='%s://%s' % (self.scheme, self.host), http2=self.http2) else: self.__http = httpx.Client(base_url='%s://%s' % (self.scheme, self.host)) elif self.request == 'requests': self.__http = requests.Session() if self.using_proxy(): self.__http.proxies = urllib.request.getproxies() elif self.request == 'requests-futures': self.__http = FuturesSession() if self.using_proxy(): self.__http.proxies = urllib.request.getproxies() elif self.request == 'httplib2': self.__http = httplib2.Http() else: if self.scheme == 'http': self.__http = http_client.HTTPConnection(self.host, self.port) elif self.scheme == 'https': self.__http = http_client.HTTPSConnection(self.host, self.port) if self.using_proxy(): self.__http.set_tunnel(self.realhost, self.realport, self.proxy_headers) def close(self): if self.request != 'httpx': self.__http.close() self.__http = None self.reset() def reset(self): self.__http_response = None self.__response_data = None self.__last_read = 0 def getHeaders(self): return self.headers def isOpen(self): return self.__http is not None def setTimeout(self, ms): if not hasattr(socket, 'getdefaulttimeout'): raise NotImplementedError if ms is None: self.__timeout = None else: self.__timeout = ms / 1000.0 def setCustomHeaders(self, headers): self.__custom_headers = headers def read(self, sz): if self.request in ['httpx', 'httplib2', 'requests', 'requests-futures']: max_sz = self.__last_read + sz min_sz = self.__last_read self.__last_read = max_sz content = self.__response_data[min_sz:max_sz] else: content = self.__http_response.read(sz) return content def write(self, buf): self.__wbuf.write(buf) def __withTimeout(f): def _f(*args, **kwargs): orig_timeout = socket.getdefaulttimeout() socket.setdefaulttimeout(args[0].__timeout) try: result = f(*args, **kwargs) finally: socket.setdefaulttimeout(orig_timeout) return result return _f async def httpx_flush(self, data, headers): # Building httpx request request = self.__http.build_request('POST', self.path, data=data, headers=headers) # Sending httpx request self.__http_response = await self.__http.send(request) self.code = self.__http_response.status_code self.message = self.__http_response.reason_phrase self.headers = self.__http_response.headers self.__response_data = self.__http_response.read() self.__last_read = 0 def flush(self): if self.request == 'httplib': # Only when using httplib request if self.__custom_thrift: if self.__loop <= 2: if self.isOpen(): self.close() self.open(); self.__loop += 1 elif time.time() - self.__time > 90: self.close(); self.open(); self.__time = time.time() else: if self.isOpen(): self.close() self.open() else: self.reset() # Pull data out of buffer data = self.__wbuf.getvalue() self.__wbuf = BytesIO() if not self.__custom_headers or 'User-Agent' not in self.__custom_headers: user_agent = 'Python/THttpClient' script = os.path.basename(sys.argv[0]) if script: user_agent = '%s (%s)' % (user_agent, urllib.parse.quote(script)) else: user_agent = None if self.request == 'hyper': headers = {'Content-Type': 'application/x-thrift', 'Content-Length': str(len(data)), 'User-Agent': user_agent} if self.__custom_headers: headers.update(self.__custom_headers) # Sending request with payload request = self.__http.request('POST', self.path, data, headers) # Get reply to flush the request self.__http_response = self.__http.get_response(request) self.code = self.__http_response.status self.message = self.__http_response.reason self.headers = self.__http_response.headers elif self.request == 'httpx': headers = {'Content-Type': 'application/x-thrift', 'Content-Length': str(len(data)), 'User-Agent': user_agent} if self.__custom_headers: headers.update(self.__custom_headers) if self.http2: self.__async_loop.run_until_complete(self.httpx_flush(data, headers)) else: # Building httpx request request = self.__http.build_request('POST', self.path, data=data, headers=headers) # Sending httpx request self.__http_response = self.__http.send(request) self.__response_data = self.__http_response.read() self.__last_read = 0 self.code = self.__http_response.status_code self.message = self.__http_response.reason_phrase self.headers = self.__http_response.headers elif self.request == 'httplib2': headers = {'Content-Type': 'application/x-thrift', 'Content-Length': str(len(data)), 'User-Agent': user_agent} if self.__custom_headers: headers.update(self.__custom_headers) # Sending and get reply to request self.__http_response, self.__response_data = self.__http.request(self.url, 'POST', headers=headers, body=data) self.__last_read = 0 self.code = self.__http_response.status self.message = self.__http_response.reason self.headers = self.__http_response elif self.request == 'requests': headers = {'Content-Type': 'application/x-thrift', 'Content-Length': str(len(data)), 'User-Agent': user_agent} if self.__custom_headers: headers.update(self.__custom_headers) # Sending and get reply to request self.__http_response = self.__http.request('POST', self.url, data=data, headers=headers) self.__response_data = self.__http_response.content self.__last_read = 0 self.code = self.__http_response.status_code self.message = self.__http_response.reason self.headers = self.__http_response.headers elif self.request == 'requests-futures': headers = {'Content-Type': 'application/x-thrift', 'Content-Length': str(len(data)), 'User-Agent': user_agent} if self.__custom_headers: headers.update(self.__custom_headers) # Sending request with payload request = self.__http.request('POST', self.url, data=data, headers=headers) # Get reply to flush the request self.__http_response = request.result() self.__response_data = self.__http_response.content self.__last_read = 0 self.code = self.__http_response.status_code self.message = self.__http_response.reason self.headers = self.__http_response.headers else: # HTTP request if self.using_proxy() and self.scheme == 'http': # need full URL of real host for HTTP proxy here (HTTPS uses CONNECT tunnel) self.__http.putrequest('POST', 'http://%s:%s%s' % (self.realhost, self.realport, self.path)) else: self.__http.putrequest('POST', self.path) # Write headers self.__http.putheader('Content-Type', 'application/x-thrift') self.__http.putheader('Content-Length', str(len(data))) if not self.__custom_headers or 'User-Agent' not in self.__custom_headers: self.__http.putheader('User-Agent', user_agent) if self.__custom_headers: for key, val in six.iteritems(self.__custom_headers): self.__http.putheader(key, val) self.__http.endheaders() # Write payload self.__http.send(data) # Get reply to flush the request self.__http_response = self.__http.getresponse() self.code = self.__http_response.status self.message = self.__http_response.reason self.headers = self.__http_response.msg # Decorate if we know how to timeout if hasattr(socket, 'getdefaulttimeout'): flush = __withTimeout(flush)
class DatadogHTTPClient(object): """ Client that sends a batch of logs over HTTP. """ _POST = "POST" if DD_USE_COMPRESSION: _HEADERS = { "Content-type": "application/json", "Content-Encoding": "gzip" } else: _HEADERS = {"Content-type": "application/json"} def __init__(self, host, port, no_ssl, skip_ssl_validation, api_key, scrubber, timeout=10): protocol = "http" if no_ssl else "https" self._url = "{}://{}:{}/v1/input/{}".format(protocol, host, port, api_key) self._scrubber = scrubber self._timeout = timeout self._session = None self._ssl_validation = not skip_ssl_validation self._futures = [] if logger.isEnabledFor(logging.DEBUG): logger.debug( f"Initialized http client for logs intake: " f"<host: {host}, port: {port}, url: {self._url}, no_ssl: {no_ssl}, " f"skip_ssl_validation: {skip_ssl_validation}, timeout: {timeout}>" ) def _connect(self): self._session = FuturesSession(max_workers=DD_MAX_WORKERS) self._session.headers.update(self._HEADERS) def _close(self): # Resolve all the futures and log exceptions if any for future in as_completed(self._futures): try: future.result() except Exception: logger.exception("Exception while forwarding logs") self._session.close() def send(self, logs): """ Sends a batch of log, only retry on server and network errors. """ try: data = self._scrubber.scrub("[{}]".format(",".join(logs))) except ScrubbingException: raise Exception("could not scrub the payload") if DD_USE_COMPRESSION: data = compress_logs(data, DD_COMPRESSION_LEVEL) # FuturesSession returns immediately with a future object future = self._session.post(self._url, data, timeout=self._timeout, verify=self._ssl_validation) self._futures.append(future) def __enter__(self): self._connect() return self def __exit__(self, ex_type, ex_value, traceback): self._close()
small_frame = cv2.resize(frame, (256,256), 0, 0, cv2.INTER_CUBIC) _, png = cv2.imencode(".jpg", small_frame) saved_data = base64.b64encode(png).decode('ascii') payload = json.dumps({"image": saved_data}) future = session.post(url, data=payload, timeout=5, hooks = { "response": response_hook }) waiting = True if waiting: if future.done(): try: response = future.result() except: session.close() waiting = False print("Stopped again") continue # clean up and re-try waiting = False try: res = response.data['images'][0]['faces'][0]['attributes'] res.pop('age') res.pop('glasses') res.pop('gender') res.pop('lips') order = list(sorted(res.items(), key=lambda x: x[1], reverse=True)) if order[0][0] == 'white': if random.random() < .9: slap_enable = True