def download(self, item):
""" Downloads a single file and returns the HTTP response. """
if self.session is None:
self.session = RequestSession()
try:
headers = {
'User-Agent':
'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.3'
}
url = item['url']
request = self.session.get(url, headers=headers)
response = {
'url': url,
'httpcode': request.status_code,
'status': request.ok,
'content-type': request.headers.get('Content-Type')
}
if request.ok:
response['content'] = request.content
return {'item': item, 'response': response}
except RequestException as e:
print('[error]', e)
return {
'item': item,
'response': {
'status': False,
'error': e,
'url': item['url']
}
}
def get_session() -> RequestSession:
"""Create a session with USGS channel.
TODO: Use development seed STAC instead
"""
url_login = '******'
session = RequestSession()
login_html = session.get(url_login)
html = BeautifulSoup(login_html.content, "html.parser")
__ncforminfo = html.find("input", {
"name": "__ncforminfo"
}).attrs.get("value")
csrf_token = html.find("input", {"id": "csrf_token"}).attrs.get("value")
auth = {
"username": user['username'],
"password": user['password'],
"csrf_token": csrf_token,
"__ncforminfo": __ncforminfo
}
session.post(url_login, data=auth, allow_redirects=False)
return session
def init_connection_pools(self):
self.request_session = RequestSession()
retry = Retry(**self.settings["requests"]["retries"])
for protocol in ("http", "https"):
self.request_session.mount(
f"{protocol}://",
HTTPAdapter(max_retries=retry, **self.settings["requests"]["pool"],),
)
def __init__(self,
processed_callback,
multi=False,
bulksize=50,
stateless=True,
state_id=None,
verbose=False,
auto_save_states=False):
self.__process_callback = processed_callback
self.multi = multi
self.bulksize = bulksize
self.stateless = stateless
self.__auto_save_states = auto_save_states
self.session = RequestSession()
self.stats = {
'total_images': 0,
'total_duration': 0,
'average_duration': 0,
'average_queue_count': 0,
'ignored': {
'total': 0,
'files': {}
},
'downloads': {
'total_successes': 0,
'total_errors': 0,
'successes': {},
'errors': {}
},
'uploads': {
'total_successes': 0,
'total_errors': 0,
'successes': {},
'errors': {}
}
}
self.__processes_queue_counts = []
self.__processes_durations = []
self.identicals = {}
if not self.stateless:
self.state_id = state_id
if self.state_id is None:
from uuid import uuid4
self.state_id = str(uuid4())
self.states = {}
self.old_states = {}
self.verbose = verbose
def get_flightaware_client(config: configparser):
"""
Set up the SOAP client for FlightAware.com
:param config:
:return:
"""
username = config["flightaware.com"]["username"]
apiKey = config["flightaware.com"]["api_key"]
wsdlFile = 'https://flightxml.flightaware.com/soap/FlightXML3/wsdl'
session = RequestSession()
session.auth = HTTPBasicAuth(username, apiKey)
client = Client(wsdlFile, transport=Transport(session=session))
return client
def retrieve(url):
session = RequestSession()
headers = {
'User-Agent':
'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.3'
}
request = session.get(url, headers=headers)
if not request.ok:
return False
res = BytesIO()
res.write(request.content)
return res
def download(url, local_path=None):
filename = os.path.basename(url)
local_file_path = os.path.join(tempfile.gettempdir(), filename)
if local_path is not None and os.path.isdir(local_path):
local_file_path = os.path.join(local_path, filename)
session = RequestSession()
headers = {
'User-Agent':
'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.3'
}
request = session.get(url, headers=headers)
if not request.ok:
return False
with open(local_file_path, 'wb') as fh:
fh.write(request.content)
return local_file_path
def test_session(self, base_url="http://testserver"):
session = RequestSession()
session.mount(prefix=base_url, adapter=RequestsWSGIAdapter(self))
return session
def verify_resp_json(path, key=None):
session = RequestSession() if key else get_session()
resp = session.get(path)
resp.raise_for_status()
return resp.json() if not key else resp.json()[key]
def get_session():
if not getattr(thread_local, "session", None):
thread_local.session = RequestSession()
return thread_local.session
def webhook_wrapper(request):
request.http_session = RequestSession()
try:
return webhook(request)
except RequestInterruption:
return HttpResponse(status=200)