def ec2_catalog(): import requests from cachecontrol import CacheControl from cachecontrol.caches.file_cache import FileCache import logging logger = logging.getLogger('isitfit') logger.debug("Downloading ec2 catalog (cached to local file)") # based on URL = 'http://www.ec2instances.info/instances.json' # URL = 's3://...csv' # Edit 2019-09-10 use CDN link instead of direct gitlab link # URL = 'https://gitlab.com/autofitcloud/www.ec2instances.info-ec2op/raw/master/www.ec2instances.info/t3b_smaller_familyL2.json' URL = 'https://cdn.jsdelivr.net/gh/autofitcloud/www.ec2instances.info-ec2op/www.ec2instances.info/t3b_smaller_familyL2.json' # cached https://cachecontrol.readthedocs.io/en/latest/ sess = requests.session() cached_sess = CacheControl(sess, cache=FileCache('/tmp/isitfit_ec2info.cache')) r = cached_sess.request('get', URL) # read catalog, copy from ec2op-cli/ec2op/optimizer/cwDailyMaxMaxCpu import json j = json.dumps(r.json(), indent=4, sort_keys=True) from pandas import read_json df = read_json(j, orient='split') # Edit 2019-09-13 no need to subsample the columns at this stage # df = df[['API Name', 'Linux On Demand cost']] df = df.rename(columns={'Linux On Demand cost': 'cost_hourly'}) # df = df.set_index('API Name') # need to use merge, not index return df
def handle_pre(self, context_pre): import requests from cachecontrol import CacheControl from cachecontrol.caches.file_cache import FileCache from isitfit.utils import logger logger.debug("Downloading ec2 catalog (cached to local file)") # based on URL = 'http://www.ec2instances.info/instances.json' # URL = 's3://...csv' # Edit 2019-09-10 use CDN link instead of direct gitlab link if self.allow_ec2_different_family: URL = 'https://cdn.jsdelivr.net/gh/autofitcloud/[email protected]/www.ec2instances.info/t3c_smaller_familyNone.json' else: # URL = 'https://gitlab.com/autofitcloud/www.ec2instances.info-ec2op/raw/master/www.ec2instances.info/t3b_smaller_familyL2.json' URL = 'https://cdn.jsdelivr.net/gh/autofitcloud/[email protected]/www.ec2instances.info/t3b_smaller_familyL2.json' # Update 2019-12-03: move into /tmp/isitfit/ # fc_dir = '/tmp/isitfit_ec2info.cache' from isitfit.dotMan import DotMan import os fc_dir = os.path.join(DotMan().tempdir(), 'ec2info.cache') # cached https://cachecontrol.readthedocs.io/en/latest/ sess = requests.session() cached_sess = CacheControl(sess, cache=FileCache(fc_dir)) r = cached_sess.request('get', URL) # read catalog, copy from ec2op-cli/ec2op/optimizer/cwDailyMaxMaxCpu import json j = json.dumps(r.json(), indent=4, sort_keys=True) from pandas import read_json df = read_json(j, orient='split') # Edit 2019-09-13 no need to subsample the columns at this stage # df = df[['API Name', 'Linux On Demand cost']] df = df.rename(columns={'Linux On Demand cost': 'cost_hourly'}) # df = df.set_index('API Name') # need to use merge, not index context_pre['df_cat'] = df return context_pre
class RequestsClient(HttpClient): """An implementation of HttpClient that uses Requests as its HTTP Client Attributes: timeout (int): The default timeout for all API requests. """ def __init__(self, timeout=60, cache=False, max_retries=None, retry_interval=None): """The constructor. Args: timeout (float): The default global timeout(seconds). """ self.timeout = timeout self.session = requests.session() if max_retries and retry_interval: retries = Retry(total=max_retries, backoff_factor=retry_interval) self.session.mount('http://', HTTPAdapter(max_retries=retries)) self.session.mount('https://', HTTPAdapter(max_retries=retries)) if cache: self.session = CacheControl(self.session) def execute_as_string(self, request): """Execute a given HttpRequest to get a string response back Args: request (HttpRequest): The given HttpRequest to execute. Returns: HttpResponse: The response of the HttpRequest. """ response = self.session.request(HttpMethodEnum.to_string( request.http_method), request.query_url, headers=request.headers, params=request.query_parameters, data=request.parameters, files=request.files, timeout=self.timeout) return self.convert_response(response, False) def execute_as_binary(self, request): """Execute a given HttpRequest to get a binary response back Args: request (HttpRequest): The given HttpRequest to execute. Returns: HttpResponse: The response of the HttpRequest. """ response = self.session.request(HttpMethodEnum.to_string( request.http_method), request.query_url, headers=request.headers, params=request.query_parameters, data=request.parameters, files=request.files, timeout=self.timeout) return self.convert_response(response, True) def convert_response(self, response, binary): """Converts the Response object of the HttpClient into an HttpResponse object. Args: response (dynamic): The original response object. Returns: HttpResponse: The converted HttpResponse object. """ if binary: return HttpResponse(response.status_code, response.headers, response.content) else: return HttpResponse(response.status_code, response.headers, response.text)
class Connection(object): """ Handler for connection and calls to the Open Targets Validation Platform REST API """ _AUTO_GET_TOKEN = 'auto' def __init__( self, host='https://www.targetvalidation.org', port=443, api_version='latest', auth_app_name=None, auth_secret=None, use_http2=False, ): """ Args: host (str): host serving the API port (int): port to use for connection to the API api_version (str): api version to point to, default to 'latest' auth_app_name (str): app_name if using authentication auth_secret (str): secret if using authentication use_http2 (bool): use http2 client """ self._logger = logging.getLogger(__name__) self.host = host self.port = str(port) self.api_version = api_version self.auth_app_name = auth_app_name self.auth_secret = auth_secret if self.auth_app_name and self.auth_secret: self.use_auth = True else: self.use_auth = False self.token = None self.use_http2 = use_http2 session = requests.Session() if self.use_http2: session.mount(host, HTTP20Adapter()) self.session = CacheControl(session) self._get_remote_api_specs() def _build_url(self, endpoint): return '{}:{}/api/{}{}'.format( self.host, self.port, self.api_version, endpoint, ) @staticmethod def _auto_detect_post(params): """ Determine if a post request should be made instead of a get depending on the size of the parameters in the request. Args: params (dict): params to pass in the request Returns: Boolean: True if post is needed """ if params: for k, v in params.items(): if isinstance(v, (list, tuple)): if len(v) > 3: return True return False def get(self, endpoint, params=None): """ makes a GET request Args: endpoint (str): REST API endpoint to call params (dict): request payload Returns: Response: request response """ if self._auto_detect_post(params): self._logger.debug('switching to POST due to big size of params') return self.post(endpoint, data=params) return Response( self._make_request(endpoint, params=params, method='GET')) def post(self, endpoint, data=None): """ makes a POST request Args: endpoint (str): REST API endpoint to call data (dict): request payload Returns: Response: request response """ return Response(self._make_request(endpoint, data=data, method='POST')) def _make_token_request(self, expire=60): """ Asks for a token to the API Args: expire (int): expiration time for the token Returns: response for the get token request """ return self._make_request('/public/auth/request_token', params={ 'app_name': self.auth_app_name, 'secret': self.auth_secret, 'expiry': expire }, headers={ 'Cache-Control': 'no-cache', }) def get_token(self, expire=60): """ Asks for a token to the API Args: expire (int): expiration time for the token Returns: str: the token served by the API """ response = self._make_token_request(expire) return response.json()['token'] def _make_request(self, endpoint, params=None, data=None, method=HTTPMethods.GET, headers={}, rate_limit_fail=False, **kwargs): """ Makes a request to the REST API Args: endpoint (str): endpoint of the REST API params (dict): payload for GET request data (dict): payload for POST request method (HTTPMethods): request method, either HTTPMethods.GET or HTTPMethods.POST. Defaults to HTTPMethods.GET headers (dict): HTTP headers for the request rate_limit_fail (bool): If True raise exception when usage limit is exceeded. If False wait and retry the request. Defaults to False. Keyword Args: **kwargs: forwarded to requests Returns: a response from requests """ def call(): headers['User-agent'] = 'Open Targets Python Client/%s' % str( __version__) if self.use_http2 and set(headers.keys()) & INVALID_HTTP2_HEADERS: for h in INVALID_HTTP2_HEADERS: if h in headers: del headers[h] return self.session.request(method, self._build_url(endpoint), params=params, json=data, headers=headers, **kwargs) 'order params to allow efficient caching' if params is not None: if isinstance(params, dict): params = sorted(params.items()) else: params = sorted(params) if self.use_auth and not 'request_token' in endpoint: if self.token is None: self._update_token() if self.token is not None: headers['Auth-Token'] = self.token response = None default_retry_after = 5 if not rate_limit_fail: status_code = 429 while status_code in [429, 419]: try: response = call() status_code = response.status_code if status_code == 429: retry_after = default_retry_after if 'Retry-After' in response.headers: retry_after = float( response.headers['Retry-After']) self._logger.warning( 'Maximum usage limit hit. Retrying in {} seconds'. format(retry_after)) time.sleep(retry_after) elif status_code == 419: self._update_token(force=True) headers['Auth-Token'] = self.token time.sleep(0.5) except MaxRetryError as e: self._logger.exception(e.args[0].reason) self._logger.warning( 'Problem connecting to the remote API. Retrying in {} seconds' .format(default_retry_after)) time.sleep(default_retry_after) except OSError as e: self._logger.exception(str(e)) self._logger.warning( 'Problem connecting to the remote API. Retrying in {} seconds' .format(default_retry_after)) time.sleep(default_retry_after) else: response = call() response.raise_for_status() return response def _update_token(self, force=False): """ Update token when expired """ if self.token and not force: token_valid_response = self._make_request( '/public/auth/validate_token', headers={'Auth-Token': self.token}) if token_valid_response.status_code == 200: return elif token_valid_response.status_code == 419: pass else: token_valid_response.raise_for_status() self.token = self.get_token() def _get_remote_api_specs(self): """ Fetch and parse REST API documentation """ r = self.session.get(self.host + ':' + self.port + '/api/docs/swagger.yaml') r.raise_for_status() self.swagger_yaml = r.text self.api_specs = yaml.load(self.swagger_yaml) self.endpoint_validation_data = {} for p, data in self.api_specs['paths'].items(): p = p.split('{')[0] if p[-1] == '/': p = p[:-1] self.endpoint_validation_data[p] = {} for method, method_data in data.items(): if 'parameters' in method_data: params = {} for par in method_data['parameters']: par_type = par.get('type', 'string') params[par['name']] = par_type self.endpoint_validation_data[p][method] = params remote_version = self.get('/public/utils/version').data # TODO because content type wasnt checked proerly a float # was returned instead a proper version string if str(remote_version).startswith(API_MAJOR_VERSION): self._logger.warning( 'The remote server is running the API with version {}, but the client expected this major version {}. They may not be compatible.' .format(remote_version, API_MAJOR_VERSION)) def validate_parameter(self, endpoint, filter_type, value, method=HTTPMethods.GET): """ Validate payload to send to the REST API based on info fetched from the API documentation Args: endpoint (str): endpoint of the REST API filter_type (str): the parameter sent for the request value: the value sent for the request method (HTTPMethods): request method, either HTTPMethods.GET or HTTPMethods.POST. Defaults to HTTPMethods.GET Raises AttributeError: if validation is not passed """ endpoint_data = self.endpoint_validation_data[endpoint][method] if filter_type in endpoint_data: if endpoint_data[filter_type] == 'string' and isinstance( value, str): return elif endpoint_data[filter_type] == 'boolean' and isinstance( value, bool): return elif endpoint_data[filter_type] == 'number' and isinstance( value, (int, float)): return raise AttributeError( '{}={} is not a valid parameter for endpoint {}'.format( filter_type, value, endpoint)) def api_endpoint_docs(self, endpoint): """ Returns the documentation available for a given REST API endpoint Args: endpoint (str): endpoint of the REST API Returns: dict: documentation for the endpoint parsed from YAML docs """ return self.api_specs['paths'][endpoint] def get_api_endpoints(self): """ Get a list of available endpoints Returns: list: available endpoints """ return self.api_specs['paths'].keys() def close(self): """ Close connection to the REST API """ self.session.close() def ping(self): """ Pings the API as a live check Returns: bool: True if pinging the raw response as a ``str`` if the API has a non standard name """ response = self.get('/public/utils/ping') if response.data == 'pong': return True elif response.data: return response.data return False
class Habitica: def __init__(self, api_user, api_key, aliases): s = requests.Session() s.headers["x-api-user"] = api_user s.headers["x-api-key"] = api_key self.s = CacheControl(s, cache=FileCache(str(HTTP_CACHE))) self.aliases = aliases self.cron_tz = CRON_TZ self.cron_time = CRON_TIME self.cron_file = CRON_FILE def get(self, path, **kwargs): return self.request("GET", path, **kwargs) def post(self, path, **kwargs): return self.request("POST", path, **kwargs) def request(self, method, path, **kwargs): url = API_ENDPOINT.rstrip("/") + "/" + path.lstrip("/") r = self.s.request(method, url, **kwargs) if not r.ok: ### TODO: Use requests_toolbelt.utils.dump.dump_response instead? if 400 <= r.status_code < 500: err_type = "Client" elif 500 <= r.status_code < 600: err_type = "Server" else: err_type = "Unknown" click.echo( f"{r.status_code} {err_type} Error: {r.reason} for URL: {r.url}", err=True, ) try: resp = r.json() except ValueError: click.echo(r.text, err=True) else: print_json(resp, err=True) sys.exit(1) return r.json() ### TODO: Does the API ever return non-JSON? def cron_if_needed(self): if (self.cron_file.exists() and self.cron_file.stat().st_mtime >= self.last_scheduled_cron()): return data = self.get("/user")["data"] if data["needsCron"]: self.cron() else: self.touch_cronfile(isoparse(data["lastCron"]).timestamp()) def cron(self): print_json(self.post("/cron", data="")) self.touch_cronfile() def touch_cronfile(self, ts=None): self.cron_file.parent.mkdir(parents=True, exist_ok=True) self.cron_file.touch(exist_ok=True) if ts is not None: os.utime(self.cron_file, times=(ts, ts)) def last_scheduled_cron(self): """ Calculate the *nix timestamp for the most recent `cron_time` in `cron_tz` """ now = datetime.now(self.cron_tz) if now.time() >= self.cron_time: cron_date = now.date() else: ### TODO: Handle DST changes (How does Habitica handle them?) cron_date = now.date() - timedelta(days=1) return datetime.combine( cron_date, self.cron_time.replace(tzinfo=self.cron_tz), ).timestamp() def task_up(self, tid): return TaskResponse(self.post(f"/tasks/{tid}/score/up", data="")) def task_down(self, tid): return TaskResponse(self.post(f"/tasks/{tid}/score/down", data=""))
class Session: uid = None token = None organization = None session = None netid = None _init = False _logged_in = False _BASE_URL = 'https://tv.byu.edu/' _TIMEOUT = 5 def __init__(self): self.session = CacheControl(requests.Session()) self.session.headers.update({ 'Host': 'tv.byu.edu', 'Origin': 'https://tv.byu.edu', 'Referer': 'https://tv.byu.edu/', 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 IPTV/1.0', }) def genAuth(self): if self.uid is None: self.uid = str(random.randint(1, 10000000000)) digest = hashlib.md5( b''.join(str(i).encode() for i in ( self.uid, self.netid, self.studentid)) ).hexdigest() return { 'ds': digest, 'sid': self.netid, 'uid': self.uid, } def getHeaders(self, asJson=True): headers = {} if asJson: headers.update({ 'Accept': 'application/json', 'Content-Type': 'application/json', 'X-JSON': 'true', }) if self.token is None: headers.update(self.genAuth()) else: headers['X-BEEAUTH'] = self.token return headers def fetch(self, action=None, method='POST', asJson=True, headers=None, login=True, **kwargs): if login and not self._logged_in: self.login() if action is not None: kwargs.setdefault('json', {}).update({'act': action}) h = self.getHeaders(asJson=asJson) if headers is not None: h.update(headers) r = self.session.request( method, self._BASE_URL + 'cgi-bin/remote.cgi', headers=h, timeout=self._TIMEOUT, **kwargs) if not r.ok: r.raise_for_status() if not asJson: return r json = r.json() if json['result'] != 'OK': raise IOError('Got result %s for fetch of action %s', (json['result'], action)) return json def get(self, url, **kwargs): return self.session.get(url, timeout=self._TIMEOUT, **kwargs) def set_login(self, netid, studentid): self.netid = netid self.studentid = studentid self._logged_in = False def login(self): print('Logging in as', self.netid) if not self._init: self.get(self._BASE_URL) self._init = True self.token = None self.organization = None response = self.fetch(action='verifycreds', login=False) self.organization = response['orgName'] self.token = response['token'] self._logged_in = True print('Logged in as', self.netid)
class RequestsClient(HttpClient): """An implementation of HttpClient that uses Requests as its HTTP Client Attributes: timeout (int): The default timeout for all API requests. """ def __init__(self, timeout=60, cache=False, max_retries=None, retry_interval=None): """The constructor. Args: timeout (float): The default global timeout(seconds). """ self.timeout = timeout self.session = requests.session() if max_retries and retry_interval: retries = Retry(total=max_retries, backoff_factor=retry_interval) self.session.mount('http://', HTTPAdapter(max_retries=retries)) self.session.mount('https://', HTTPAdapter(max_retries=retries)) if cache: self.session = CacheControl(self.session) def execute_as_string(self, request): """Execute a given HttpRequest to get a string response back Args: request (HttpRequest): The given HttpRequest to execute. Returns: HttpResponse: The response of the HttpRequest. """ response = self.session.request(HttpMethodEnum.to_string(request.http_method), request.query_url, headers=request.headers, params=request.query_parameters, data=request.parameters, files=request.files, timeout=self.timeout) return self.convert_response(response, False) def execute_as_binary(self, request): """Execute a given HttpRequest to get a binary response back Args: request (HttpRequest): The given HttpRequest to execute. Returns: HttpResponse: The response of the HttpRequest. """ response = self.session.request(HttpMethodEnum.to_string(request.http_method), request.query_url, headers=request.headers, params=request.query_parameters, data=request.parameters, files=request.files, timeout=self.timeout) return self.convert_response(response, True) def convert_response(self, response, binary): """Converts the Response object of the HttpClient into an HttpResponse object. Args: response (dynamic): The original response object. Returns: HttpResponse: The converted HttpResponse object. """ if binary: return HttpResponse(response.status_code, response.headers, response.content) else: return HttpResponse(response.status_code, response.headers, response.text)
class Connection(object): """ Handler for connection and calls to the Open Targets Validation Platform REST API """ def __init__(self, host='https://platform-api.opentargets.io', port=443, api_version='v3', verify=True, proxies={}, auth=None): """ Args: host (str): host serving the API port (int): port to use for connection to the API api_version (str): api version to point to, default to 'latest' verify (bool): sets SSL verification for Request session, accepts True, False or a path to a certificate auth (AuthBase): sets the custom authentication object to use for requests made to the API. Should be one of the built in options provided by the reqests package, or a subclass of requests.auth.AuthBase. """ self._logger = logging.getLogger(__name__) self.host = host self.port = str(port) self.api_version = api_version session = requests.Session() session.verify = verify session.proxies = proxies session.auth = auth retry_policies = Retry( total=10, read=10, connect=10, backoff_factor=.5, status_forcelist=(500, 502, 504), ) http_retry = HTTPAdapter(max_retries=retry_policies) session.mount(host, http_retry) self.session = CacheControl(session) self._get_remote_api_specs() def _build_url(self, endpoint): url = '{}:{}/{}{}'.format( self.host, self.port, self.api_version, endpoint, ) return url @staticmethod def _auto_detect_post(params): """ Determine if a post request should be made instead of a get depending on the size of the parameters in the request. Args: params (dict): params to pass in the request Returns: Boolean: True if post is needed """ if params: for k, v in params.items(): if isinstance(v, (list, tuple)): if len(v) > 3: return True return False def get(self, endpoint, params=None): """ makes a GET request Args: endpoint (str): REST API endpoint to call params (dict): request payload Returns: Response: request response """ if self._auto_detect_post(params): self._logger.debug('switching to POST due to big size of params') return self.post(endpoint, data=params) return Response( self._make_request(endpoint, params=params, method='GET')) def post(self, endpoint, data=None): """ makes a POST request Args: endpoint (str): REST API endpoint to call data (dict): request payload Returns: Response: request response """ return Response(self._make_request(endpoint, data=data, method='POST')) def _make_request(self, endpoint, params=None, data=None, method=HTTPMethods.GET, headers={}, rate_limit_fail=False, **kwargs): """ Makes a request to the REST API Args: endpoint (str): endpoint of the REST API params (dict): payload for GET request data (dict): payload for POST request method (HTTPMethods): request method, either HTTPMethods.GET or HTTPMethods.POST. Defaults to HTTPMethods.GET headers (dict): HTTP headers for the request rate_limit_fail (bool): If True raise exception when usage limit is exceeded. If False wait and retry the request. Defaults to False. Keyword Args: **kwargs: forwarded to requests Returns: a response from requests """ 'order params to allow efficient caching' if params: if isinstance(params, dict): params = sorted(params.items()) else: params = sorted(params) headers['User-agent'] = 'Open Targets Python Client/%s' % str( __version__) response = self.session.request(method, self._build_url(endpoint), params=params, json=data, headers=headers, **kwargs) response.raise_for_status() return response def _get_remote_api_specs(self): """ Fetch and parse REST API documentation """ r = self.session.get(self.host + ':' + self.port + '/v%s/platform/swagger' % API_MAJOR_VERSION) r.raise_for_status() self.swagger_yaml = r.text self.api_specs = yaml.load(self.swagger_yaml) self.endpoint_validation_data = {} for p, data in self.api_specs['paths'].items(): p = p.split('{')[0] if p[-1] == '/': p = p[:-1] self.endpoint_validation_data[p] = {} self.endpoint_validation_data['/platform' + p] = {} for method, method_data in data.items(): if 'parameters' in method_data: params = {} for par in method_data['parameters']: par_type = par.get('type', 'string') params[par['name']] = par_type self.endpoint_validation_data[p][method] = params self.endpoint_validation_data['/platform' + p][method] = params remote_version = self.get('/platform/public/utils/version').data # TODO because content type wasnt checked proerly a float # was returned instead a proper version string if not str(remote_version).startswith(API_MAJOR_VERSION): self._logger.warning( 'The remote server is running the API with version {}, but the client expected this major version {}. They may not be compatible.' .format(remote_version, API_MAJOR_VERSION)) def validate_parameter(self, endpoint, filter_type, value, method=HTTPMethods.GET): """ Validate payload to send to the REST API based on info fetched from the API documentation Args: endpoint (str): endpoint of the REST API filter_type (str): the parameter sent for the request value: the value sent for the request method (HTTPMethods): request method, either HTTPMethods.GET or HTTPMethods.POST. Defaults to HTTPMethods.GET Raises AttributeError: if validation is not passed """ endpoint_data = self.endpoint_validation_data[endpoint][method] if filter_type in endpoint_data: if endpoint_data[filter_type] == 'string' and isinstance( value, str): return elif endpoint_data[filter_type] == 'boolean' and isinstance( value, bool): return elif endpoint_data[filter_type] == 'number' and isinstance( value, (int, float)): return raise AttributeError( '{}={} is not a valid parameter for endpoint {}'.format( filter_type, value, endpoint)) def api_endpoint_docs(self, endpoint): """ Returns the documentation available for a given REST API endpoint Args: endpoint (str): endpoint of the REST API Returns: dict: documentation for the endpoint parsed from YAML docs """ return self.api_specs['paths'][endpoint] def get_api_endpoints(self): """ Get a list of available endpoints Returns: list: available endpoints """ return self.api_specs['paths'].keys() def close(self): """ Close connection to the REST API """ self.session.close() def ping(self): """ Pings the API as a live check Returns: bool: True if pinging the raw response as a ``str`` if the API has a non standard name """ response = self.get('/platform/public/utils/ping') if response.data == 'pong': return True elif response.data: return response.data return False