def download(self, item): """ Downloads a single file and returns the HTTP response. """ if self.session is None: self.session = RequestSession() try: headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.3' } url = item['url'] request = self.session.get(url, headers=headers) response = { 'url': url, 'httpcode': request.status_code, 'status': request.ok, 'content-type': request.headers.get('Content-Type') } if request.ok: response['content'] = request.content return {'item': item, 'response': response} except RequestException as e: print('[error]', e) return { 'item': item, 'response': { 'status': False, 'error': e, 'url': item['url'] } }
def get_session() -> RequestSession: """Create a session with USGS channel. TODO: Use development seed STAC instead """ url_login = '******' session = RequestSession() login_html = session.get(url_login) html = BeautifulSoup(login_html.content, "html.parser") __ncforminfo = html.find("input", { "name": "__ncforminfo" }).attrs.get("value") csrf_token = html.find("input", {"id": "csrf_token"}).attrs.get("value") auth = { "username": user['username'], "password": user['password'], "csrf_token": csrf_token, "__ncforminfo": __ncforminfo } session.post(url_login, data=auth, allow_redirects=False) return session
def init_connection_pools(self): self.request_session = RequestSession() retry = Retry(**self.settings["requests"]["retries"]) for protocol in ("http", "https"): self.request_session.mount( f"{protocol}://", HTTPAdapter(max_retries=retry, **self.settings["requests"]["pool"],), )
def __init__(self, processed_callback, multi=False, bulksize=50, stateless=True, state_id=None, verbose=False, auto_save_states=False): self.__process_callback = processed_callback self.multi = multi self.bulksize = bulksize self.stateless = stateless self.__auto_save_states = auto_save_states self.session = RequestSession() self.stats = { 'total_images': 0, 'total_duration': 0, 'average_duration': 0, 'average_queue_count': 0, 'ignored': { 'total': 0, 'files': {} }, 'downloads': { 'total_successes': 0, 'total_errors': 0, 'successes': {}, 'errors': {} }, 'uploads': { 'total_successes': 0, 'total_errors': 0, 'successes': {}, 'errors': {} } } self.__processes_queue_counts = [] self.__processes_durations = [] self.identicals = {} if not self.stateless: self.state_id = state_id if self.state_id is None: from uuid import uuid4 self.state_id = str(uuid4()) self.states = {} self.old_states = {} self.verbose = verbose
def get_flightaware_client(config: configparser): """ Set up the SOAP client for FlightAware.com :param config: :return: """ username = config["flightaware.com"]["username"] apiKey = config["flightaware.com"]["api_key"] wsdlFile = 'https://flightxml.flightaware.com/soap/FlightXML3/wsdl' session = RequestSession() session.auth = HTTPBasicAuth(username, apiKey) client = Client(wsdlFile, transport=Transport(session=session)) return client
def retrieve(url): session = RequestSession() headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.3' } request = session.get(url, headers=headers) if not request.ok: return False res = BytesIO() res.write(request.content) return res
def download(url, local_path=None): filename = os.path.basename(url) local_file_path = os.path.join(tempfile.gettempdir(), filename) if local_path is not None and os.path.isdir(local_path): local_file_path = os.path.join(local_path, filename) session = RequestSession() headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.3' } request = session.get(url, headers=headers) if not request.ok: return False with open(local_file_path, 'wb') as fh: fh.write(request.content) return local_file_path
def test_session(self, base_url="http://testserver"): session = RequestSession() session.mount(prefix=base_url, adapter=RequestsWSGIAdapter(self)) return session
def verify_resp_json(path, key=None): session = RequestSession() if key else get_session() resp = session.get(path) resp.raise_for_status() return resp.json() if not key else resp.json()[key]
def get_session(): if not getattr(thread_local, "session", None): thread_local.session = RequestSession() return thread_local.session
def webhook_wrapper(request): request.http_session = RequestSession() try: return webhook(request) except RequestInterruption: return HttpResponse(status=200)