def request_query(self, url: str, parameters: Dict) -> Dict: session = sessions.FuturesSession(max_workers=self.__NB_WORKER) response = session.get(url, params=parameters) self.check_request_response(response) return response.result().json()
def __init__(self, name): self.name = name self.main_channel_link = None self.main_channel_html = None self.subscribers = 0 self.bad_words = 0 self.session = sessions.FuturesSession(max_workers=10)
def load(cls, session=None): """Initialize an instance using values from lbry.io.""" if not session: session = sessions.FuturesSession() return cls( session, conf.ANALYTICS_ENDPOINT, utils.deobfuscate(conf.ANALYTICS_TOKEN) )
def get_images(urls): imagesContent = [] with sessions.FuturesSession() as session: futures = [session.get(url) for url in urls] for future in as_completed(futures): response = future.result() imagesContent.append(response.content) return imagesContent
def get_html_async(urls, n_workers=10, use_cache=True): """:func:`get_html`, but async, give or take. Takes a list (or any iterable) of urls and returns a corresponding generator of htmls. The htmls have their scripts and styles removed and are stored in cache. Args: urls (iterable(str)): The urls to be accessed n_workers (:obj:`int`, optional): The number of workers. use_cache (:obj:`bool`, optional): Attempts to use the cache if True, otherwise it will fetch from sigarra Returns: An str generator """ global cache if cache == None: load_cache() urls = tuple(urls) remove_invalid_entries(urls) work_queue = [url for url in urls if url not in cache or not use_cache] if len(work_queue) > 0: with _sessions.FuturesSession(max_workers=n_workers) as session: futures = [session.get(url) for url in work_queue] for future in futures: request = future.result() if request.status_code != 200: continue # GTFO url = request.url html = request.text for match_rule, custom_treatment in _custom_treatments.items( ): # Can we apply any custom treatment to the html? if match_rule(url): cache[url] = custom_treatment(html) break else: # If not, use the default treatment cache[url] = _default_treatment(html) return (get_html(url) for url in urls)
def check_transaction(txid): """ This function checks the transactional details by quering the BlockStream TestNet API @param: txid - list - This corresponds to the transaction ids to be queried @param: results - np array - The transactional details in certain array format """ # Using the Future Session lib for doing parallel requests (HTTP - secured) session = sessions.FuturesSession(max_workers=len(txid)) futures = [session.get(i).result() for i in txid] # using map() fn for fetching the required information about txids results = np.array(list(map(json_parser, futures))) return results
def get_all_market_info(self): session = sessions.FuturesSession(session=self.s) results = {} for type_ in self.resource_id: if type_ == 'energy drink': continue results[type_] = session.get(f'http://rivalregions.com/storage/market/{self.resource_id[type_]}?{self.c}') for res in results: r = results[res].result() price, selling_amount, player_id, player_name, total_offers = self.parse_market_response(r, self.resource_id[res]) results[res] = {'price': price, 'amount': selling_amount, 'player_id': player_id, 'player_name': player_name, 'total_offers':total_offers} return results
def null_session(futures: bool = False) -> 'Union[Session, FuturesSession]': """No proxy session. Args: futures: If returns a :class:`requests_futures.FuturesSession`. Returns: Union[requests.Session, requests_futures.FuturesSession]: The session object with no proxy settings. """ if futures: session = requests_futures_sessions.FuturesSession( max_workers=DARC_CPU) else: session = requests.Session() session.headers['User-Agent'] = default_user_agent() return session
def __init__(self, id, device_name, configFile, log): print("GpuDevice start init") threading.Thread.__init__(self) self.ready = False self.id = id self.batch_size = 2 self.log = log self.cams = {} self.max_batch_size = 4 self.config_mot = { 'engine_path': 'models/yolov4_-1_3_416_416_dynamic.engine', 'max_batch_size': self.max_batch_size } # with open('config/mot.json') as config_file: # config_mot = json.load(config_file, cls=ConfigDecoder) # self.config_mot = config['mot'] self.detector = None self.server_URL = "http://localhost:8080/update?" self.session = sessions.FuturesSession(max_workers=2) self.proceedTime = 0 self.device = str(device_name) try: self.config = configFile self.cnt = 0 self.frame = [] self.img_size = self.config['img_size'] self._stopevent = threading.Event() self.ready = True # self.isRunning = False self.log.debug(device_name + " with name " + str(self.id) + " created ok id:" + str(self.device)) self.session.get(self.server_URL + "cmd=GpuStart&name=" + str(id) + "&status=OK") self.start() except: print("GpuDevice init Can not start GPU for " + str(self.id) + " ", self.device, self.config) # traceback.print_exception(*sys.exc_info()) self.session.get(self.server_URL + "cmd=GpuStart&name=" + str(id) + "&status=error&module=Gpu") print(sys.exc_info()) self.kill()
def __init__(self, targetserver, hec_token, eventgen_name=None): self._name = 'eventgen_splunk_hec_logger' self.targetserver = targetserver self.hec_token = hec_token self.host = socket.gethostname() self.pid = os.getpid() self.events = [] self.send = True self.os = platform.platform() self.system_username = commands.getoutput('whoami') self.eventgen_name = eventgen_name atexit.register(self._stopFlushTimer) self.log = logging.getLogger(self._name) self.log.setLevel(logging.DEBUG) self.log.info("SplunkHECHandler logger is initialized") try: self.ip = [ l for l in ([ ip for ip in socket.gethostbyname_ex(socket.gethostname())[2] if not ip.startswith("127.") ][:1], [[(s.connect(('8.8.8.8', 53)), s.getsockname()[0], s.close()) for s in [ socket.socket(socket.AF_INET, socket.SOCK_DGRAM) ]][0][1]]) if l ][0][0] except: self.ip = "unknown" self.session = sessions.FuturesSession(max_workers=32) if not targetserver or not hec_token: self.log.warn( "Please provide valid targetserver and hec_token in default/eventgen_engine.conf." ) self.send = False super(SplunkHECHandler, self).__init__() self.timer = self._flushAndRepeatTimer()
def get(self): session = sessions.FuturesSession(max_workers=10) httpFutures = [] serviceRefAndStopIdArray = [('B', 1693, 'B'), ('L', 286, 'L all'), ('L', 224, 'L ret')] text = '' for serviceRefAndStopId in serviceRefAndStopIdArray: serviceRef = serviceRefAndStopId[0] stopId = serviceRefAndStopId[1] #response = taoBusTimesQuery(serviceRef, stopId) query = buildTaoBusTimesQuery(serviceRef, stopId) httpFutures.append(session.get(query)) for i, future in enumerate(httpFutures): serviceRefAndStopId = serviceRefAndStopIdArray[i] serviceRef = serviceRefAndStopId[0] stopId = serviceRefAndStopId[1] stopLabel = serviceRefAndStopId[2] httpResponse = future.result() resObj = json.loads(httpResponse.text) if len(resObj['busTimes']) > 0: response = resObj['busTimes'][0] else: response = None text += '%s: ' % (stopLabel if stopLabel else serviceRef) if response: timeDatas = response['timeDatas'] for timeData in timeDatas: text += '%d ' % timeData['minutes'] serviceDisruption = response['serviceDisruption'] if serviceDisruption: text += '\n/!\ %s' % serviceDisruption else: text += 'no data!' text += '\n' return txtPagesBuilder(text)
def i2p_session(futures: bool = False) -> 'Union[Session, FuturesSession]': """I2P (.i2p) session. Args: futures: If returns a :class:`requests_futures.FuturesSession`. Returns: Union[requests.Session, requests_futures.FuturesSession]: The session object with I2P proxy settings. See Also: * :data:`darc.proxy.i2p.I2P_REQUESTS_PROXY` """ if futures: session = requests_futures_sessions.FuturesSession( max_workers=DARC_CPU) else: session = requests.Session() session.headers['User-Agent'] = default_user_agent(proxy='I2P') session.proxies.update(I2P_REQUESTS_PROXY) return session
def check_transaction(datatype, txid): """ This function checks the transactional details by quering the BlockStream TestNet API KeyWord Arguments: datatype (str): This variable defines data type of txid (str or list) txid (str/list) - This corresponds to the transaction ids to be queried Return: results (np array): The transactional details in certain array format """ if datatype == 'str': url = "https://blockstream.info/testnet/api/tx/" req = requests.get(url + txid) results = np.array(list(json_parser(req))) elif datatype == 'arr': session = sessions.FuturesSession(max_workers=len(txid)) futures = [session.get(i).result() for i in txid] results = np.array(list(map(json_parser, futures))) return results
def get_html_async(self, urls, n_workers: int = 10): """:func:`Credentials.get_html`, but async, give or take. Takes a list (or any iterable) of urls and returns a corresponding generator of htmls. The htmls have their scripts and styles removed and are stored in cache. Args: urls (iterable(str)): The urls to be accessed n_workers (:obj:`int`, optional): The number of workers. Returns: An str generator """ urls = tuple(urls) work_queue = (url for url in urls if url not in self.cache) with _sessions.FuturesSession(max_workers=n_workers) as session: session.cookies.update( self.session.cookies ) # Getting the cookies from the object session to this assynchronous session futures = [session.get(url) for url in work_queue] for future in futures: request = future.result() if request.status_code != 200: continue # GTFO url = request.url html = request.text self.cache[url] = _utils.trim_html(html) return (self.get_html(url) for url in urls)
def __init__(self, camConfig, gpuConfig, device_id, cam_id, log, vc_device="/CPU:0"): threading.Thread.__init__(self) self.log = log self.log.debug("VideoCapture start init stream object " + str(cam_id)) self.totalFrames = 0 self.vc_device = vc_device self.server_URL = "http://localhost:8080/update?" self.startTime = int(time.time()) self.pipline_str = 'rtspsrc location={} latency={} ! rtph264depay ! h264parse ! queue leaky=1 ! decodebin ! videoconvert ! appsink sync=false' self.cur_frame_cnt = 0 self.proceed_frames_cnt = 0 self.proceedTime = [0, 0] self.session = sessions.FuturesSession(max_workers=2) self.outFrame = np.array([]) self.isDrow = False self.error_counter_default = 10 self.error_counter = 10 self.clients = [] self.out_color = (252, 35, 240) self.in_color = (52, 235, 240) self.text_color = (255, 255, 0) self.intersections = {} self.save_video_res = None self.id = str(cam_id) self.device_id = device_id self.uid = np.append( np.random.choice( np.fromstring('abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789', dtype=np.uint8), 7), np.uint8(device_id)) try: self.config = camConfig print("VideoCapture stream config", self.config) self.url = self.config['url'] if 'type' in self.config: self.type = self.config['type'] else: self.type = 0 self.isFromFile = self.config['isFromFile'] self.cap = None self.buffer = 200 self.img_size = int(gpuConfig['img_size']) print("img size", self.img_size) self.cur_proceed_frame = None self.max_hum_w = int(self.img_size/2) self.GPUconfig = gpuConfig self.frame_res = (self.img_size, self.img_size) self.skip_frames = int(self.config['skip_frames']) self.batch_size = int(self.config['batch_size']) self.path_track = int(self.config['path_track']) self.save_video_flag = self.config['save_video_flag'] print("save_video_flag", self.save_video_flag, cam_id) self.display_video_flag = self.config['display_video_flag'] if(self.config['save_video_flag'] or self.config['display_video_flag']): self.isDrow = True self.save_video_res = tuple(self.config['save_video_res']) print("save video res", self.save_video_res) scale = self.img_size/416.0 self.borders = self.config['borders'] print("scale", scale, self.borders) if self.borders: for key in self.borders: self.borders[key][0] = [int(self.borders[key][0][0]*scale),int(self.borders[key][0][1]*scale)] self.borders[key][1] = [int(self.borders[key][1][0]*scale),int(self.borders[key][1][1]*scale)] self.intersections[key] = [0, 0] self.out = None print("start load encoder", os.path.join('models',self.config['encoder_filename'])) self.encoder = gdet.create_box_encoder( os.path.join('models', 'mars-small128.onnx'), batch_size=self.batch_size, device=self.vc_device) print("encider is ok") if self.save_video_flag: outFile = self.config['save_path'] +"_"+ str(self.startTime)+".avi" if outFile == '': outFile = 'video/'+str(self.id)+"_"+ str(self.startTime)+"_auto.avi" self.log.info("!!!!!!Save out video to file " + outFile) self.out = cv2.VideoWriter(outFile, cv2.VideoWriter_fourcc(*'XVID'), 5, self.save_video_res) print("VideoCapture stream start load tracker") self.tracker = Tracker(nn_matching.NearestNeighborDistanceMetric("cosine", self.config['max_cosine_distance'], None)) print("VideoCapture stream tracker ok") if not self.isFromFile: #print("start read from camera") url = self.pipline_str.format(self.url, self.buffer) print("url", self.url,"\n", url) #self.q = VideoCaptureStream(self.url, self.frame_res) self.q = VideoCaptureStream(url, self.frame_res) print("self.q", self.q) else: #print("start read from file") self.q = VideoCaptureFile(self.url, self.frame_res, self.skip_frames) self.totalFrames = self.q.totalFrames print("created stream ok") #if client is not None: # client.send_json({'OK':["startStream", self.id]}) self.session.get(self.server_URL+'cmd=startStream&name='+self.id+'&status=OK') self._stopevent = threading.Event() except: print("VideoStream err:", sys.exc_info()) # self.session.get(self.server_URL+'cmd=startStream&name='+self.id+'&status=error') self.log.debug("VideoStream Can not start Video Stream for " + camConfig) # print("VideoStream err types",type(traceback.print_exception(*sys.exc_info())), type(sys.exc_info())) self.id = None
def reload_solr_cores() -> None: for core_name in _SOLR_CORE_NAMES: sessions.FuturesSession().get( _SOLR_RELOAD_URL.format(core_name), background_callback=_core_reload_callback)
def new_instance(cls, session=None): """Initialize an instance using values from the configuration""" if not session: session = sessions.FuturesSession() return cls(session, conf.settings['ANALYTICS_ENDPOINT'], utils.deobfuscate(conf.settings['ANALYTICS_TOKEN']))
def __init__(self, configFileName): #threading.Thread.__init__(self) self.gpusActiveList = {} # running devices self.gpusList = {} # available devices self.gpusConfigList = {} # devices configs self.streamsConfigList = {} self.camsList = {} # running streams on gpus f = logging.Formatter('[L:%(lineno)d]# %(levelname)-8s [%(asctime)s] %(message)s', datefmt = '%d-%m-%Y %H:%M:%S') # Add file rotating handler, with level DEBUG fileLog = logging.handlers.RotatingFileHandler('manager.log', 'a', 1000000, 5) fileLog.setLevel(logging.INFO) fileLog.setFormatter(f) logging.getLogger().addHandler(fileLog) # physical_devices = tf.config.experimental.list_physical_devices('GPU') # tf.config.set_visible_devices(physical_devices[0], 'GPU') # tf.config.experimental.set_memory_growth(physical_devices[0], True) self.log = logging.getLogger('appManager') self._stopevent = threading.Event() self.isGPU = True self.server_URL = "http://localhost:8080/update?" # self.camIdFrame = [] self.isGpuStarted = False try: self.session = sessions.FuturesSession(max_workers=2) self.config = self.loadConfig(configFileName, 'Gpus_manager_') if self.config: self.db_path = "db.wingu.sqlite3" self.db_table_name = "stats" self.config['gpu_configs'] = {} self.config['streams_configs'] = {} print("gpus manager config", self.config) #gpus = tf.config.experimental.list_physical_devices('GPU') if self.isGPU: # print("gpus", gpus) # print("len=", len(self.config['gpus_configs_list']), len(gpus)) # self.isGPU = True nvidia_smi.nvmlInit() self.gpuInfo = {} index = 0 gpu_id = 0 #if len(gpus) <= len(self.config['gpus_configs_list']): # print("gpus_configs_list, gpus config is ok", self.config['gpus_configs_list']) # for index, gpu_id in enumerate(gpus): print("key", index, gpu_id) name = "/GPU:" + str(index) device = str(self.config['gpus_configs_list'][index]) print("gpus name", name, device) self.gpusList[name] = index print("gpus 333", self.config['gpus_configs_list'][index]) cfg = self.loadConfig(self.config['gpus_configs_list'][index], "Gpu_") print("gpus cfg", cfg) if cfg is not None: cfg['device_id'] = name cfg['fileName'] = device self.gpusConfigList[name] = cfg if device in self.config['autostart_gpus_list']: self.startGpu(cfg, name) self.gpuInfo[name] = index print("gpu", name, index, self.gpuInfo) else: name = "/CPU:0" self.gpusList[name] = 0 # Init for CPU cfg = self.loadConfig(self.config['cpu_config'], "GPU_") if cfg != None: cfg['device_id'] = name cfg['fileName'] = self.config['cpu_config'] self.gpusConfigList['cpu'] = cfg if self.config['autostart_gpus_list'] != None: self.startGpu(cfg, name) self.log.debug("GPUsmanager Active GPUs list: "+" ".join(self.gpusActiveList.keys())) for stream in self.config['streams']: stream = str (stream) cfg = self.loadConfig(stream, "Stream_") if cfg != None: self.streamsConfigList[stream] = cfg if self.config['autostart_streams']: if stream in self.config['autostart_streams']: self.startStream(stream) self.ready = True try: self.log.debug("GPUsmanager try to autostart "+ stream) time.sleep(3) except: self.log.debug("GPUsmanager exception autostart "+ stream) self.session.get(self.server_URL+"cmd=startManager&status=OK&name=Init&module=Manager") # self.start() else: self.log.error("GPUsmanager Can not load config gor GPUs Maanger") except: self.log.error("GPUsmanager Can not start gpus manager") print(sys.exc_info()) self.kill()
# encoding: utf-8 from flask import Flask, render_template import json import os import re from requests_futures import sessions import plot app = Flask(__name__) session = sessions.FuturesSession() POLL_URL = f"https://api.gh-polls.com/poll/{os.environ['POLL_ID']}/" DEFAULT_INFO_URL = "https://matplotlib.org//tutorials/introductory/customizing.html" def init_styles(styles_filename): """Load the styles dict from JSON and construct the necessary URL properties""" with open(styles_filename, 'rb') as f: styles = json.load(f) for (style, style_properties) in styles.items(): style_properties['poll_img_url'] = POLL_URL + style style_properties['info_url'] = style_properties.get( 'info_url', DEFAULT_INFO_URL) style_properties[ 'poll_vote_url'] = style_properties['poll_img_url'] + '/vote' style_properties['votes'] = 0
def session_document_loader(secure=False, **kwargs): """ Create a Requests document loader. Can be used to setup extra Requests args such as verify, cert, timeout, or others. :param secure: require all requests to use HTTPS (default: False). :param **kwargs: extra keyword args for Requests get() call. :return: the RemoteDocument loader function. """ from pyld.jsonld import ( JsonLdError, urllib_parse, parse_link_header, LINK_HEADER_REL, ) import requests if True: import requests_cache requests_cache.install_cache("demo_cache") session = requests_cache.CachedSession() if False: from requests_futures import sessions session = sessions.FuturesSession() if False: # enable http/2 from hyper.contrib import HTTP20Adapter adapter = HTTP20Adapter() adapter = requests.adapters.HTTPAdapter(pool_connections=100, pool_maxsize=100) session.mount("https://", adapter) # session.mount('https://w3id.org', adapter) # session.mount('https://ontopia-lodview.pdnd.italia.it', adapter) def loader(url): """ Retrieves JSON-LD at the given URL. :param url: the URL to retrieve. :return: the RemoteDocument. """ try: # validate URL pieces = urllib_parse.urlparse(url) if (not all([pieces.scheme, pieces.netloc]) or pieces.scheme not in ["http", "https"] or set(pieces.netloc) > set(string.ascii_letters + string.digits + "-.:")): raise JsonLdError( 'URL could not be dereferenced; only "http" and "https" ' "URLs are supported.", "jsonld.InvalidUrl", {"url": url}, code="loading document failed", ) if secure and pieces.scheme != "https": raise JsonLdError( "URL could not be dereferenced; secure mode enabled and " 'the URL\'s scheme is not "https".', "jsonld.InvalidUrl", {"url": url}, code="loading document failed", ) headers = {"Accept": "application/ld+json, application/json"} response = session.get(url, headers=headers, **kwargs) doc = { "contextUrl": None, "documentUrl": response.url, "document": response.json(), } content_type = response.headers.get("content-type") link_header = response.headers.get("link") if link_header and content_type != "application/ld+json": link_header = parse_link_header(link_header).get( LINK_HEADER_REL) # only 1 related link header permitted if isinstance(link_header, list): raise JsonLdError( "URL could not be dereferenced, it has more than one " "associated HTTP Link Header.", "jsonld.LoadDocumentError", {"url": url}, code="multiple context link headers", ) if link_header: doc["contextUrl"] = link_header["target"] return doc except JsonLdError as e: raise e except Exception as cause: raise JsonLdError( "Could not retrieve a JSON-LD document from the URL.", "jsonld.LoadDocumentError", code="loading document failed", cause=cause, ) return loader
def report_time(test): t0 = time.time() yield print("Time needed for `%s' called: %.2fs" % (test, time.time() - t0)) with report_time("serialized"): for i in range(TRIES): requests.get(URL) session = requests.Session() with report_time("Session"): for i in range(TRIES): session.get(URL) session = sessions.FuturesSession(max_workers=2) with report_time("FuturesSession w/ 2 workers"): futures = [session.get(URL) for i in range(TRIES)] for f in futures: f.result() session = sessions.FuturesSession(max_workers=TRIES) with report_time("FuturesSession w/ max workers"): futures = [session.get(URL) for i in range(TRIES)] for f in futures: f.result() async def get(url): async with aiohttp.ClientSession() as session: async with session.get(url) as response:
def get_async(*urls): session = sessions.FuturesSession() futures = map(session.get, urls) return list(x.result() for x in futures)
def _query_requests(self): session = sessions.FuturesSession(max_workers=len(self._urls_to_query)) self._requests = [ session.get(url).result() for url in self._urls_to_query ]