def _get_ip(self, pool, validator):
ips = []
for page in pool:
reply = requests.get(page,
timeout=5,
headers={'Accept': 'application/json'})
maybe_ip = reply.content.strip()
# Validate
try:
ipa = validator(maybe_ip)
if not ipa.is_private:
ips.append(maybe_ip)
except AddressValueError:
pass
return Counter(ips).most_common()[0][0]
def send_file(self, data_type, path, now, suffix=None):
"""
Send a file to the ON service.
Args:
data_type: type of data that is being sent.
path: local file path.
now: the time period that corresponds to the file.
"""
if suffix:
name = '{}_{}'.format(self.hostname, suffix)
else:
name = self.hostname
url = '{server}/sign/{type}/{year}/{month}/{day}/{time}/{name}'
url = url.format(server=self.proxy_uri,
type=data_type,
year=now.year,
month=now.month,
day=now.day,
time=now.time(),
name=name)
logging.info('Prepping file: {}'.format(url))
response = requests.get(url, **self.request_args)
response.raise_for_status()
result = response.json()
try:
headers = result['headers']
url = result['url']
method = result['method']
remote_path = result['path']
except KeyError:
raise requests_exceptions.RequestException(
'Parameters missing from response')
with io.open(path, mode='rb') as data:
logging.info('Sending file: {} {}'.format(method, url))
resp = requests.request(method,
url,
headers=headers,
data=data,
verify=True,
timeout=HTTP_TIMEOUT)
resp.raise_for_status()
return remote_path
def get_data(self, data_url, params=None):
"""
Retrieve data from the ON service.
Args:
data_url: resource we should fetch. This should be either
`data_type`, or a "`data_type`/`hostname`" string.
params: optional query params to send
Returns:
Python-requests response. You can do things like r.json(), or
r.text
"""
url = '{server}/get/{type}'
url = url.format(server=self.proxy_uri, type=data_url)
logging.info('Downloading data: {}'.format(url))
response = requests.get(url, params=params, **self.request_args)
response.raise_for_status()
return response
def get_data(self, data_url, params=None):
"""
Retrieve data from the ON service.
Args:
data_url: resource we should fetch. This should be either
`data_type`, or a "`data_type`/`hostname`" string.
params: optional query params to send
Returns:
Python-requests response. You can do things like r.json(), or
r.text
"""
url = '{server}/get/{type}'
url = url.format(
server=self.proxy_uri, type=data_url)
logging.info('Downloading data: {}'.format(url))
response = requests.get(url, params=params, **self.request_args)
response.raise_for_status()
return response
def send_file(self, data_type, path, now, suffix=None):
"""
Send a file to the ON service.
Args:
data_type: type of data that is being sent.
path: local file path.
now: the time period that corresponds to the file.
"""
if suffix:
name = '{}_{}'.format(self.hostname, suffix)
else:
name = self.hostname
url = '{server}/sign/{type}/{year}/{month}/{day}/{time}/{name}'
url = url.format(
server=self.proxy_uri, type=data_type, year=now.year,
month=now.month, day=now.day, time=now.time(),
name=name)
logging.info('Prepping file: {}'.format(url))
response = requests.get(url, **self.request_args)
response.raise_for_status()
result = response.json()
try:
headers = result['headers']
url = result['url']
method = result['method']
remote_path = result['path']
except KeyError:
raise requests_exceptions.RequestException(
'Parameters missing from response'
)
with io.open(path, mode='rb') as data:
logging.info('Sending file: {} {}'.format(method, url))
resp = requests.request(method, url, headers=headers, data=data,
verify=True, timeout=HTTP_TIMEOUT)
resp.raise_for_status()
return remote_path
def get(path, **kwargs):
"""requests.get wrapper"""
token = os.environ.get(BE_GITHUB_API_TOKEN)
if token:
kwargs["headers"] = {
"Authorization": "token %s" % token
}
try:
response = requests.get(path, verify=False, **kwargs)
if response.status_code == 403:
lib.echo("Patience: You can't pull more than 60 "
"presets per hour without an API token.\n"
"See https://github.com/mottosso/be/wiki"
"/advanced#extended-preset-access")
sys.exit(lib.USER_ERROR)
return response
except Exception as e:
if self.verbose:
lib.echo("ERROR: %s" % e)
else:
lib.echo("ERROR: Something went wrong. "
"See --verbose for more information")