def test_merge_absolute_url():
client = httpx.Client(base_url="https://www.example.com/")
request = client.build_request("GET", "http://www.example.com/")
assert request.url == "http://www.example.com/"
with pytest.warns(DeprecationWarning):
assert not request.url.is_ssl
def __init__(self):
self.http = httpx.Client(http2=True)
if not isfile(config.data_path):
exit("Please generate cookies and tokens first.")
email = sys.argv[1]
auth = ""
hangouts_token = ""
cookies = ""
with open(config.data_path, 'r') as f:
out = json.loads(f.read())
auth = out["auth"]
hangouts_token = out["keys"]["hangouts"]
cookies = out["cookies"]
client = httpx.Client(cookies=cookies, headers=config.headers)
data = is_email_google_account(client, auth, cookies, email,
hangouts_token)
geolocator = Nominatim(user_agent="nominatim")
print(f"[+] {len(data['matches'])} account found !")
for user in data["matches"]:
print("\n------------------------------\n")
gaiaID = user["personId"][0]
email = user["lookupId"]
infos = data["people"][gaiaID]
# get name
name = get_account_name(client, gaiaID)
def test_stream_response(server):
with httpx.Client() as client:
with client.stream("GET", server.url) as response:
content = response.read()
assert response.status_code == 200
assert content == b"Hello, world!"
import hashlib, httpx, ingest, json, logging, sys, time, traceback
from datetime import datetime, timezone
from sklearn.neighbors import KDTree
SN_URL = 'https://www.spotternetwork.org/feeds/reports.txt'
API_URL = 'http://localhost:8080/events'
HAZARD_MAP = [
'?', 'tornado', 'funnel cloud', 'rotating wall cloud', 'hail', 'high wind',
'flooding', 'flash flood', 'other'
]
logging.basicConfig(filename='wx-sn.log', encoding='utf-8', level=logging.INFO)
client = httpx.Client(timeout=10.0)
with open('cities.json', 'r') as file:
cities = json.loads(file.read())
cities_coords = [(x[0], x[1]) for x in cities]
cities_tree = KDTree(cities_coords)
# SN tends to have some non ascii chars in reports that should be removed
def strip_non_ascii(input):
return ''.join(char for char in input if ord(char) < 128)
def get_ts(datestring):
dt = datetime.strptime(datestring, '%Y-%m-%d %H:%M:%S %Z')
dt = dt.replace(tzinfo=timezone.utc)
return int(dt.timestamp()) * 1000
def line_to_event(line):
def test_base_url(server):
base_url = server.url
with httpx.Client(base_url=base_url) as client:
response = client.get("/")
assert response.status_code == 200
assert response.url == base_url
def test_post(server):
with httpx.Client() as client:
response = client.post(server.url, data=b"Hello, world!")
assert response.status_code == 200
assert response.reason_phrase == "OK"
def test_that_client_is_closed_after_with_block():
with httpx.Client() as client:
pass
assert client.is_closed
def test_get_invalid_url(server, url):
with httpx.Client() as client:
with pytest.raises((httpx.UnsupportedProtocol, httpx.LocalProtocolError)):
client.get(url)
def test_that_send_cause_client_to_be_not_closed():
client = httpx.Client()
client.get("http://example.com")
assert not client.is_closed
def test_that_client_is_not_closed_in_with_block():
with httpx.Client() as client:
assert not client.is_closed
def test_that_client_is_closed_by_default():
client = httpx.Client()
assert client.is_closed
def test_merge_relative_url_with_dotted_path():
client = httpx.Client(base_url="https://www.example.com/some/path")
request = client.build_request("GET", "../testing/123")
assert request.url == "https://www.example.com/some/testing/123"
def test_merge_relative_url():
client = httpx.Client(base_url="https://www.example.com/")
request = client.build_request("GET", "/testing/123")
assert request.url == "https://www.example.com/testing/123"
def test_keldoc_parse_center():
center1_url = (
"https://vaccination-covid.keldoc.com/centre-hospitalier-regional/lorient-56100/groupe-hospitalier"
"-bretagne-sud-lorient-hopital-du-scorff?specialty=144 ")
center1_data = json.loads(
Path("tests", "fixtures", "keldoc", "center1-info.json").read_text())
request = ScraperRequest(center1_url, "2020-04-04")
client = httpx.Client(transport=httpx.MockTransport(app_center1))
test_center_1 = KeldocCenter(request, client=client)
assert test_center_1.parse_resource()
# Check if parameters are parsed correctly
assert test_center_1.resource_params
res = test_center_1.resource_params
assert res["type"] == "centre-hospitalier-regional"
assert res["location"] == "lorient-56100"
assert res[
"slug"] == "groupe-hospitalier-bretagne-sud-lorient-hopital-du-scorff"
# Fetch center data (id/center specialties)
assert test_center_1.fetch_center_data()
assert test_center_1.id == center1_data["id"]
assert test_center_1.specialties == center1_data["specialties"]
# Filter center specialties
filtered_specialties = get_relevant_vaccine_specialties_id(
test_center_1.specialties)
assert filtered_specialties == [144]
# Fetch vaccine cabinets
assert not test_center_1.fetch_vaccine_cabinets()
test_center_1.vaccine_specialties = filtered_specialties
cabinets = test_center_1.fetch_vaccine_cabinets()
assert cabinets == [18780, 16913, 16910, 16571, 16579]
# Fetch motives
motives = filter_vaccine_motives(
client,
test_center_1.selected_cabinet,
test_center_1.id,
test_center_1.vaccine_specialties,
test_center_1.vaccine_cabinets,
)
assert motives == json.loads(
Path("tests", "fixtures", "keldoc",
"center1-motives.json").read_text())
# Find first availability date
fake_now = dt.datetime(2020, 4, 4, 8, 15)
with mock_datetime_now(fake_now):
date, count, appointment_schedules = test_center_1.find_first_availability(
"2020-04-04")
assert not date
test_center_1.vaccine_motives = motives
with mock_datetime_now(fake_now):
date, count, appointment_schedules = test_center_1.find_first_availability(
"2020-04-04")
tz = datetime.timezone(datetime.timedelta(seconds=7200))
assert date == datetime.datetime(2021, 4, 20, 16, 55, tzinfo=tz)
assert appointment_schedules == [
{
"name": "chronodose",
"from": "2020-04-04T08:15:00+02:00",
"to": "2020-04-05T08:14:59+02:00",
"total": 0
},
{
"name": "1_days",
"from": "2020-04-04T00:00:00+02:00",
"to": "2020-04-04T23:59:59+02:00",
"total": 0
},
{
"name": "2_days",
"from": "2020-04-04T00:00:00+02:00",
"to": "2020-04-05T23:59:59+02:00",
"total": 0
},
{
"name": "7_days",
"from": "2020-04-04T00:00:00+02:00",
"to": "2020-04-10T23:59:59+02:00",
"total": 0
},
{
"name": "28_days",
"from": "2020-04-04T00:00:00+02:00",
"to": "2020-05-01T23:59:59+02:00",
"total": 0
},
{
"name": "49_days",
"from": "2020-04-04T00:00:00+02:00",
"to": "2020-05-22T23:59:59+02:00",
"total": 0
},
]
from scraper.doctolib.doctolib_center_scrap import doctolib_urlify
from utils.vmd_config import get_conf_platform, get_conf_inputs
from utils.vmd_logger import get_logger
KELDOC_CONF = get_conf_platform("keldoc")
KELDOC_API = KELDOC_CONF.get("api", {})
SCRAPER_CONF = KELDOC_CONF.get("center_scraper", {})
BASE_URL = KELDOC_API.get("scraper")
timeout = httpx.Timeout(KELDOC_CONF.get("timeout", 25), connect=KELDOC_CONF.get("timeout", 25))
KELDOC_ENABLED = KELDOC_CONF.get("enabled", False)
KELDOC_HEADERS = {
"User-Agent": os.environ.get("KELDOC_API_KEY", ""),
}
session = httpx.Client(timeout=timeout, headers=KELDOC_HEADERS)
logger = get_logger()
def run_departement_scrap(departement: str) -> list:
logger.info(f"[Keldoc centers] Parsing pages of departement {departement} through department SEO link")
centers_departements = parse_pages_departement(departement)
if not centers_departements:
return []
return centers_departements
def send_keldoc_request(url: str) -> Optional[dict]:
try:
req = session.get(url)
req.raise_for_status()
def test_delete(server):
with httpx.Client() as client:
response = client.delete(server.url)
assert response.status_code == 200
assert response.reason_phrase == "OK"
def test_sync_proxy_close():
try:
client = httpx.Client(proxies={"https://": PROXY_URL})
client.get("http://example.com")
finally:
client.close()
def test_merge_url():
client = httpx.Client(base_url="https://www.paypal.com/")
url = client.merge_url("http://www.paypal.com")
assert url.scheme == "https"
assert url.is_ssl
def test_unsupported_proxy_scheme():
with pytest.raises(ValueError):
httpx.Client(proxies="ftp://127.0.0.1")
def test_post_json(server):
with httpx.Client() as client:
response = client.post(server.url, json={"text": "Hello, world!"})
assert response.status_code == 200
assert response.reason_phrase == "OK"
def test_for_deprecated_proxy_params(proxies, is_valid):
if not is_valid:
with pytest.raises(ValueError):
httpx.Client(proxies=proxies)
else:
httpx.Client(proxies=proxies)
def __init__(self, api_key):
self._client = httpx.Client(base_url=self.API_URL)
self._api_key = api_key
self._customer_id = None
def __init__(self):
self._http = httpx.Client(
auth=(_config.hub_access_key_id, _config.hub_access_key_secret),
verify=False,
)
self.cache = _ServerCache(self._http)
from datetime import datetime, timedelta
from dateutil.parser import isoparse
from pytz import timezone
import httpx
from scraper.keldoc.keldoc_filters import parse_keldoc_availability
from scraper.keldoc.keldoc_routes import API_KELDOC_CALENDAR, API_KELDOC_CENTER, API_KELDOC_CABINETS
from scraper.pattern.scraper_request import ScraperRequest
from scraper.pattern.center_info import get_vaccine_name, Vaccine, INTERVAL_SPLIT_DAYS, CHRONODOSES
timeout = httpx.Timeout(25.0, connect=25.0)
KELDOC_HEADERS = {
"User-Agent": os.environ.get("KELDOC_API_KEY", ""),
}
KELDOC_SLOT_LIMIT = 50
DEFAULT_CLIENT = httpx.Client(timeout=timeout, headers=KELDOC_HEADERS)
logger = logging.getLogger("scraper")
paris_tz = timezone("Europe/Paris")
class KeldocCenter:
def __init__(self, request: ScraperRequest, client: httpx.Client = None):
self.request = request
self.base_url = request.get_url()
self.client = DEFAULT_CLIENT if client is None else client
self.resource_params = None
self.id = None
self.specialties = None
self.vaccine_specialties = None
self.vaccine_cabinets = None
self.vaccine_motives = None
DOCTOLIB_HEADERS = {
'X-Covid-Tracker-Key': os.environ.get('DOCTOLIB_API_KEY', ''),
}
DEFAULT_CLIENT: httpx.Client
logger = logging.getLogger('scraper')
if os.getenv('WITH_TOR', 'no') == 'yes':
session = requests.Session()
session.proxies = { # type: ignore
'http': 'socks5://127.0.0.1:9050',
'https': 'socks5://127.0.0.1:9050',
}
DEFAULT_CLIENT = session # type: ignore
else:
DEFAULT_CLIENT = httpx.Client()
def fetch_slots(rdv_site_web, start_date):
# Fonction principale avec le comportement "de prod".
doctolib = DoctolibSlots(client=DEFAULT_CLIENT)
return doctolib.fetch(rdv_site_web, start_date)
class DoctolibSlots:
# Permet de passer un faux client HTTP,
# pour éviter de vraiment appeler Doctolib lors des tests.
def __init__(self, client: httpx.Client = None) -> None:
self._client = DEFAULT_CLIENT if client is None else client
def __init__(self, usertoken:[str, list]):
self.token = usertoken
self.http = httpx.Client(http2=True)
def login(username: str,
password: str,
country_code: str,
domain: str,
market_place_id: str,
captcha_callback=None,
otp_callback=None,
cvf_callback=None) -> Dict[str, Any]:
amazon_url = f"https://www.amazon.{domain}"
sign_in_url = amazon_url + "/ap/signin"
cvf_url = amazon_url + "/ap/cvf/verify"
default_headers = {"User-Agent": USER_AGENT}
session = httpx.Client(headers=default_headers)
while "session-token" not in session.cookies:
session.get(amazon_url)
oauth_url = build_oauth_url(country_code, domain, market_place_id)
oauth_resp = session.get(oauth_url)
oauth_soup = get_soup(oauth_resp)
login_inputs = get_inputs_from_soup(oauth_soup)
login_inputs["email"] = username
login_inputs["password"] = password
metadata = meta_audible_app(USER_AGENT, amazon_url)
login_inputs["metadata1"] = encrypt_metadata(metadata)
login_resp = session.post(sign_in_url, data=login_inputs)
login_soup = get_soup(login_resp)
# check for captcha
while check_for_captcha(login_soup):
captcha_url = extract_captcha_url(login_soup)
if captcha_callback:
guess = captcha_callback(captcha_url)
else:
guess = default_captcha_callback(captcha_url)
inputs = get_inputs_from_soup(login_soup)
inputs["guess"] = guess
inputs["use_image_captcha"] = "true"
inputs["use_audio_captcha"] = "false"
inputs["showPasswordChecked"] = "false"
inputs["email"] = username
inputs["password"] = password
login_resp = session.post(sign_in_url, data=inputs)
login_soup = get_soup(login_resp)
# check for choice mfa
# https://www.amazon.de/ap/mfa/new-otp
while check_for_choice_mfa(login_soup):
inputs = get_inputs_from_soup(login_soup)
for node in login_soup.select(
"div[data-a-input-name=otpDeviceContext]"):
# auth-TOTP, auth-SMS, auth-VOICE
if "auth-TOTP" in node["class"]:
inp_node = node.find("input")
inputs[inp_node["name"]] = inp_node["value"]
login_resp = session.post(amazon_url + "/ap/mfa", data=inputs)
login_soup = get_soup(login_resp)
# check for mfa (otp_code)
while check_for_mfa(login_soup):
if otp_callback:
otp_code = otp_callback()
else:
otp_code = default_otp_callback()
inputs = get_inputs_from_soup(login_soup)
inputs["otpCode"] = otp_code
inputs["mfaSubmit"] = "Submit"
inputs["rememberDevice"] = "false"
login_resp = session.post(sign_in_url, data=inputs)
login_soup = get_soup(login_resp)
# check for cvf
while check_for_cvf(login_soup):
if cvf_callback:
cvf_code = cvf_callback()
else:
cvf_code = default_cvf_callback()
inputs = get_inputs_from_soup(login_soup)
login_resp = session.post(cvf_url, data=inputs)
login_soup = get_soup(login_resp)
inputs = get_inputs_from_soup(login_soup)
inputs["action"] = "code"
inputs["code"] = cvf_code
login_resp = session.post(cvf_url, data=inputs)
login_soup = get_soup(login_resp)
if login_resp.status_code != 404:
raise Exception("Unable to login")
access_token = extract_token_from_url(login_resp.url)
website_cookies = extract_cookies_from_session(session)
expires = (datetime.utcnow() + timedelta(seconds=3600)).timestamp()
session.close()
return {
"access_token": access_token,
"website_cookies": website_cookies,
"expires": expires
}
def sync_client(wsgi_app):
httpx_client = httpx.Client(app=wsgi_app)
try:
yield Client(httpx_client, base_url="http://testserver/")
finally:
httpx_client.close()
def __init__(self, client: "SyncClient"):
self.client = client
self.http = httpx.Client()
