def register(self, login, password, aquarium_url, name):
"""
Registers a new session, creating a new managed folder.
:param login: aquarium login
:type login: str
:param password: aquarium password
:type password: str
:param aquarium_url: aquarium url
:type aquarium_url: str
:param name: name to give to the new session
:type name: str
:return: None
:rtype: NOne
"""
try:
self._session_manager.register_session(login, password,
aquarium_url, name)
except InvalidSchema:
raise InvalidSchema(
"Missing schema for {}. Did you forget the \"http://\"?".
format(aquarium_url))
logger.cli("registering session: {}".format(name))
self.set_session(name)
self._save()
def get_adapter(self, url):
"""Returns the appropriate connnection adapter for the given URL."""
for (prefix, adapter) in self.adapters.items():
if url.lower().startswith(prefix):
return adapter
# Nothing matches :-/
raise InvalidSchema("No connection adapters were found for '%s'" % url)
def register(self, login, password, aquarium_url, session_name):
"""Registers a new session."""
try:
self.session_manager.register_session(login, password,
aquarium_url, session_name)
except InvalidSchema:
raise InvalidSchema(
"Missing schema for {}. Did you forget the \"http://\"?".
format(aquarium_url))
logger.cli("registering session: {}".format(session_name))
self.save()
return self
def create_patients(self, patients):
"""
:param patients: the list of input patients
:type patients: list
:return: the list of patient identifiers
"""
if not all(patient.is_valid for patient in patients):
validation_errors = [
dict(patient.validation_errors) for patient in patients
]
raise InvalidSchema(
"Patients are invalid: {}".format(validation_errors),
request=patients)
response = self.post("projects/{project_id}/patients".format(
project_id=self.project_id),
payload=[dict(patient) for patient in patients])
return response
def create_phenotypes(self, phenotypes, person_id):
"""
:param phenotypes:
:param person_id:
:return:
"""
if not all(phenotype.is_valid for phenotype in phenotypes):
validation_errors = [
dict(patient.validation_errors) for patient in phenotypes
]
raise InvalidSchema(
"Phenotypes are invalid: {}".format(validation_errors),
request=phenotypes)
response = self.post(
"persons/{person_id}/phenotypes".format(person_id=person_id),
payload=[dict(phenotype) for phenotype in phenotypes])
phenotype_ids = [x["person_phenotype_id"] for x in response]
return phenotype_ids
def create_snvs(self, snvs, patient_id):
"""
:param snvs:
:param patient_id:
:return:
"""
if not all(snv.is_valid for snv in snvs):
validation_errors = [
dict(patient.validation_errors) for patient in snvs
]
raise InvalidSchema(
"Variants are invalid: {}".format(validation_errors),
request=snvs)
response = self.post(
"patients/{patient_id}/snvs".format(patient_id=patient_id),
payload=[dict(variant) for variant in snvs])
variant_ids = [x["patient_snv_id"] for x in response]
return variant_ids
def run(self):
log.info("Started", thread=current_thread())
while True:
with Timer(key="doWork"):
try:
with Timer(key="q.get"):
uri = self.queue.get()
if uri is None:
break
try:
#check if this is a valid URL, if we have an exception, done
o = urlparse(uri)
if o.scheme.startswith('http'):
res = None
if self.checkUpdateRobots(uri):
res = self.getStatus(uri)
else:
log.info("ROBOTS DENIED",
uri=uri,
thread=current_thread())
self.handle_request(uri, res, None)
else:
raise InvalidSchema(
"No connection adapters were found for " + uri)
except Exception as e:
ErrorHandler.handleError(log,
exception=e,
msg="doWork",
exc_info=True,
url=uri,
thread=current_thread())
self.handle_request(uri, None, e)
except Exception as e:
ErrorHandler.handleError(log,
exception=e,
msg="uncaught in doWork",
exc_info=True,
thread=current_thread())
log.info("STOPPED", thread=current_thread())
def create_persons(self, persons, patient_id):
"""
:param persons:
:type persons: list
:param patient_id:
:return:
"""
logging.info(str(persons))
if not all(person.is_valid for person in persons):
validation_errors = [
dict(patient.validation_errors) for patient in persons
]
raise InvalidSchema(
"Persons are invalid: {}".format(validation_errors),
request=persons)
response = self.post(
"patients/{patient_id}/persons".format(patient_id=patient_id),
payload=[dict(person) for person in persons])
person_ids = [x["person_id"] for x in response]
return person_ids
def __init__(self, url="http://python.org", parser="html.parser", refresh=True, save_path=None, stop_words=None, split_string=None):
"""__init__
Parameters
-----------
bool : refresh
Specifies if page should be read from source. Defaults to True
str : save_path
Specifies folder to save the text file of scraped page
list : stop_words
A list of words to not include in the output of words()
list : split_string
A list of strings with which to split the words on the page
"""
self.url = url#.strip()
self.split_string = self.word_splitters(split_string)
self.stop_words = self.stop_words(stop_words)
self.refresh = refresh
self.parser = parser
if save_path:
self.save_path = save_path
else:
self.save_path = self.get_save_path()
try:
req_text = requests.get(self.url, timeout=(10.0, 10.0)).text
except MissingSchema:
raise MissingSchema("url should be in the form <http://domain.extension>")
except InvalidSchema:
raise InvalidSchema("Your url, {}, has an invalid schema".format(self.url))
if self.refresh:
self.soup = BeautifulSoup(req_text, self.parser)
self.save_page(self.soup, self.get_save_path())
else:
try:
with open(self.get_save_path(), 'r+') as rh:
self.soup = BeautifulSoup(rh.read(), self.parser)
except FileNotFoundError:
raise FileNotFoundError("File may have been moved. Try again with 'refresh=True'")
self.raw_links = self.soup.find_all('a', href=True)
def request(self,
method,
url,
params=None,
data=None,
headers=None,
cookies=None,
files=None,
auth=None,
timeout=None,
allow_redirects=True,
proxies=None,
hooks=None,
stream=None,
verify=None,
cert=None,
**kwargs):
"""Constructs a :class:`Request <Request>`, prepares it and sends it.
Returns :class:`Response <Response>` object.
:param method: method for the new :class:`Request` object.
:param url: URL for the new :class:`Request` object.
:param params: (optional) Dictionary or bytes to be sent in the query
string for the :class:`Request`.
:param data: (optional) Dictionary or bytes to send in the body of the
:class:`Request`.
:param headers: (optional) Dictionary of HTTP Headers to send with the
:class:`Request`.
:param cookies: (optional) Dict or CookieJar object to send with the
:class:`Request`.
:param files: (optional) Dictionary of 'filename': file-like-objects
for multipart encoding upload.
:param auth: (optional) Auth tuple or callable to enable
Basic/Digest/Custom HTTP Auth.
:param timeout: (optional) Float describing the timeout of the
request.
:param allow_redirects: (optional) Boolean. Set to True by default.
:param proxies: (optional) Dictionary mapping protocol to the URL of
the proxy.
:param stream: (optional) whether to immediately download the response
content. Defaults to ``False``.
:param verify: (optional) if ``True``, the SSL cert will be verified.
A CA_BUNDLE path can also be provided.
:param cert: (optional) if String, path to ssl client cert file (.pem).
If Tuple, ('cert', 'key') pair.
"""
#===============================================================================================================
# add by mz
error_type = kwargs.get("error_type")
if error_type:
from requests.exceptions import InvalidURL, URLRequired, ConnectTimeout, ConnectionError, SSLError, ReadTimeout
from requests.exceptions import InvalidSchema, MissingSchema, ChunkedEncodingError, ContentDecodingError
from requests.exceptions import RequestException, HTTPError, ProxyError, Timeout, RetryError, StreamConsumedError
get_error = {
"InvalidURL": InvalidURL(),
"URLRequired": URLRequired(),
"ConnectTimeout": ConnectTimeout(),
"ConnectionError": ConnectionError(),
"SSLError": SSLError(),
"ReadTimeout": ReadTimeout(),
"InvalidSchema": InvalidSchema(),
"MissingSchema": MissingSchema(),
"ChunkedEncodingError": ChunkedEncodingError(),
"ContentDecodingError": ContentDecodingError(),
"StreamConsumedError": StreamConsumedError(),
"TooManyRedirects": TooManyRedirects(),
"RequestException": RequestException(),
"HTTPError": HTTPError(),
"ProxyError": ProxyError(),
"Timeout": Timeout(),
"RetryError": RetryError
}
error_ = get_error[error_type]
raise error_
#===============================================================================================================
method = builtin_str(method)
# Create the Request.
req = Request(
method=method.upper(),
url=url,
headers=headers,
files=files,
data=data or {},
params=params or {},
auth=auth,
cookies=cookies,
hooks=hooks,
)
prep = self.prepare_request(req)
proxies = proxies or {}
# Gather clues from the surrounding environment.
if self.trust_env:
# Set environment's proxies.
env_proxies = get_environ_proxies(url) or {}
for (k, v) in env_proxies.items():
proxies.setdefault(k, v)
# Look for configuration.
if not verify and verify is not False:
verify = os.environ.get('REQUESTS_CA_BUNDLE')
# Curl compatibility.
if not verify and verify is not False:
verify = os.environ.get('CURL_CA_BUNDLE')
# Merge all the kwargs.
proxies = merge_setting(proxies, self.proxies)
stream = merge_setting(stream, self.stream)
verify = merge_setting(verify, self.verify)
cert = merge_setting(cert, self.cert)
# Send the request.
send_kwargs = {
'stream': stream,
'timeout': timeout,
'verify': verify,
'cert': cert,
'proxies': proxies,
'allow_redirects': allow_redirects,
}
resp = self.send(prep, **send_kwargs)
return resp
def _prepare_api_url(self, url, p):
scheme, auth, host, port, path, query, fragment = parse_url(url)
if scheme is None or scheme == "http":
return f"http://{host}:{p}"
else:
raise InvalidSchema("Invalid scheme %r: Do not supply" % scheme)
def request(self,
method,
url,
params=None,
data=None,
headers=None,
cookies=None,
files=None,
auth=None,
timeout=None,
allow_redirects=True,
proxies=None,
hooks=None,
stream=None,
verify=None,
cert=None,
json=None,
**kwargs):
"""Constructs a :class:`Request <Request>`, prepares it and sends it.
Returns :class:`Response <Response>` object.
:param method: method for the new :class:`Request` object.
:param url: URL for the new :class:`Request` object.
:param params: (optional) Dictionary or bytes to be sent in the query
string for the :class:`Request`.
:param data: (optional) Dictionary, bytes, or file-like object to send
in the body of the :class:`Request`.
:param json: (optional) json to send in the body of the
:class:`Request`.
:param headers: (optional) Dictionary of HTTP Headers to send with the
:class:`Request`.
:param cookies: (optional) Dict or CookieJar object to send with the
:class:`Request`.
:param files: (optional) Dictionary of ``'filename': file-like-objects``
for multipart encoding upload.
:param auth: (optional) Auth tuple or callable to enable
Basic/Digest/Custom HTTP Auth.
:param timeout: (optional) How long to wait for the server to send
data before giving up, as a float, or a :ref:`(connect timeout,
read timeout) <timeouts>` tuple.
:type timeout: float or tuple
:param allow_redirects: (optional) Set to True by default.
:type allow_redirects: bool
:param proxies: (optional) Dictionary mapping protocol or protocol and
hostname to the URL of the proxy.
:param stream: (optional) whether to immediately download the response
content. Defaults to ``False``.
:param verify: (optional) whether the SSL cert will be verified.
A CA_BUNDLE path can also be provided. Defaults to ``True``.
:param cert: (optional) if String, path to ssl client cert file (.pem).
If Tuple, ('cert', 'key') pair.
:rtype: requests.Response
"""
#===============================================================================================================
# add by mz
error_type = kwargs.get("error_type")
if error_type:
from requests.exceptions import InvalidURL, URLRequired, ConnectTimeout, ConnectionError, SSLError, ReadTimeout
from requests.exceptions import InvalidSchema, MissingSchema, ChunkedEncodingError, ContentDecodingError
from requests.exceptions import RequestException, HTTPError, ProxyError, Timeout, RetryError, StreamConsumedError
from requests.exceptions import TooManyRedirects
get_error = {
"InvalidURL": InvalidURL(),
"URLRequired": URLRequired(),
"ConnectTimeout": ConnectTimeout(),
"ConnectionError": ConnectionError(),
"SSLError": SSLError(),
"ReadTimeout": ReadTimeout(),
"InvalidSchema": InvalidSchema(),
"MissingSchema": MissingSchema(),
"ChunkedEncodingError": ChunkedEncodingError(),
"ContentDecodingError": ContentDecodingError(),
"StreamConsumedError": StreamConsumedError(),
"TooManyRedirects": TooManyRedirects(),
"RequestException": RequestException(),
"HTTPError": HTTPError(),
"ProxyError": ProxyError(),
"Timeout": Timeout(),
"RetryError": RetryError
}
error_ = get_error[error_type]
raise error_
#===============================================================================================================
# Create the Request
req = Request(
method=method.upper(),
url=url,
headers=headers,
files=files,
data=data or {},
json=json,
params=params or {},
auth=auth,
cookies=cookies,
hooks=hooks,
)
prep = self.prepare_request(req)
proxies = proxies or {}
settings = self.merge_environment_settings(prep.url, proxies, stream,
verify, cert)
# Send the request.
send_kwargs = {
'timeout': timeout,
'allow_redirects': allow_redirects,
}
send_kwargs.update(settings)
resp = self.send(prep, **send_kwargs)
return resp
def test_monitor_sends_exception_data_and_hb_on_expected_exceptions(
self, mock_get_data) -> None:
errors_exceptions_dict = {
ReqConnectionError('test'): SystemIsDownException(
self.test_monitor.system_config.system_name),
ReadTimeout('test'): SystemIsDownException(
self.test_monitor.system_config.system_name),
IncompleteRead('test'): DataReadingException(
self.test_monitor.monitor_name,
self.test_monitor.system_config.system_name),
ChunkedEncodingError('test'): DataReadingException(
self.test_monitor.monitor_name,
self.test_monitor.system_config.system_name),
ProtocolError('test'): DataReadingException(
self.test_monitor.monitor_name,
self.test_monitor.system_config.system_name),
InvalidURL('test'): InvalidUrlException(
self.test_monitor.system_config.node_exporter_url),
InvalidSchema('test'): InvalidUrlException(
self.test_monitor.system_config.node_exporter_url),
MissingSchema('test'): InvalidUrlException(
self.test_monitor.system_config.node_exporter_url),
MetricNotFoundException('test_metric', 'test_endpoint'):
MetricNotFoundException('test_metric', 'test_endpoint')
}
try:
self.test_monitor._initialise_rabbitmq()
for error, data_ret_exception in errors_exceptions_dict.items():
mock_get_data.side_effect = error
expected_output_data = {
'error': {
'meta_data': {
'monitor_name': self.test_monitor.monitor_name,
'system_name':
self.test_monitor.system_config.system_name,
'system_id':
self.test_monitor.system_config.system_id,
'system_parent_id':
self.test_monitor.system_config.parent_id,
'time': datetime(2012, 1, 1).timestamp()
},
'message': data_ret_exception.message,
'code': data_ret_exception.code,
}
}
expected_output_hb = {
'component_name': self.test_monitor.monitor_name,
'is_alive': True,
'timestamp': datetime(2012, 1, 1).timestamp()
}
# Delete the queue before to avoid messages in the queue on
# error.
self.test_monitor.rabbitmq.queue_delete(self.test_queue_name)
res = self.test_monitor.rabbitmq.queue_declare(
queue=self.test_queue_name, durable=True, exclusive=False,
auto_delete=False, passive=False
)
self.assertEqual(0, res.method.message_count)
self.test_monitor.rabbitmq.queue_bind(
queue=self.test_queue_name, exchange=RAW_DATA_EXCHANGE,
routing_key='system')
self.test_monitor.rabbitmq.queue_bind(
queue=self.test_queue_name, exchange=HEALTH_CHECK_EXCHANGE,
routing_key='heartbeat.worker')
self.test_monitor._monitor()
# By re-declaring the queue again we can get the number of
# messages in the queue.
res = self.test_monitor.rabbitmq.queue_declare(
queue=self.test_queue_name, durable=True, exclusive=False,
auto_delete=False, passive=True
)
# There must be 2 messages in the queue, the heartbeat and the
# processed data
self.assertEqual(2, res.method.message_count)
# Check that the message received is actually the processed data
_, _, body = self.test_monitor.rabbitmq.basic_get(
self.test_queue_name)
self.assertEqual(expected_output_data, json.loads(body))
# Check that the message received is actually the HB
_, _, body = self.test_monitor.rabbitmq.basic_get(
self.test_queue_name)
self.assertEqual(expected_output_hb, json.loads(body))
except Exception as e:
self.fail("Test failed: {}".format(e))
def SOCKSProxyManager(*args, **kwargs):
raise InvalidSchema("Missing dependencies for SOCKS support.")
