def _download_bulk(r):
# Figure out the passed credentials, if any. Two possibilities:
# (1) User and password, given explicitly for the base URLs (or an
# explicity given `eida_token` key per URL).
# (2) A global EIDA_TOKEN key. It will be used for all services that
# don't have explicit credentials and also support the `/auth` route.
credentials = r["credentials"].get(urlparse(r["endpoint"]).netloc, {})
c = client.Client(r["endpoint"],
debug=r["debug"],
timeout=r["timeout"],
**credentials)
if not credentials and "EIDA_TOKEN" in r["credentials"] and \
c._has_eida_auth:
c.set_eida_token(r["credentials"]["EIDA_TOKEN"])
if r["data_type"] == "waveform":
fct = c.get_waveforms_bulk
service = c.services["dataselect"]
elif r["data_type"] == "station":
fct = c.get_stations_bulk
service = c.services["station"]
# Keep only kwargs that are supported by this particular service.
kwargs = {k: v for k, v in r["kwargs"].items() if k in service}
bulk_str = ""
for key, value in kwargs.items():
bulk_str += "%s=%s\n" % (key, str(value))
try:
return fct(bulk_str + r["bulk_str"])
except FDSNException:
return None
def _download_bulk(r):
# Figure out the passed credentials, if any. Two possibilities:
# (1) User and password, given explicitly for the base URLs (or an
# explicity given `eida_token` key per URL).
# (2) A global EIDA_TOKEN key. It will be used for all services that
# don't have explicit credentials and also support the `/auth` route.
credentials = r["credentials"].get(urlparse(r["endpoint"]).netloc, {})
c = client.Client(r["endpoint"], debug=r["debug"], timeout=r["timeout"],
**credentials)
if not credentials and "EIDA_TOKEN" in r["credentials"] and \
c._has_eida_auth:
c.set_eida_token(r["credentials"]["EIDA_TOKEN"])
if r["data_type"] == "waveform":
fct = c.get_waveforms_bulk
service = c.services["dataselect"]
elif r["data_type"] == "station":
fct = c.get_stations_bulk
service = c.services["station"]
# Keep only kwargs that are supported by this particular service.
kwargs = {k: v for k, v in r["kwargs"].items() if k in service}
bulk_str = ""
for key, value in kwargs.items():
bulk_str += "%s=%s\n" % (key, str(value))
try:
return fct(bulk_str + r["bulk_str"])
except FDSNException:
return None
def __new__(cls, root=None):
# root provided and it's no web URL
if root:
scheme = urlparse(root).scheme
if scheme in ('http', 'https'):
return super(NRL, cls).__new__(RemoteNRL)
# Check if it's really a folder on the file-system.
if not os.path.isdir(root):
msg = ("Provided path '{}' seems to be a local file path "
"but the directory does not exist.").format(root)
raise ValueError(msg)
return super(NRL, cls).__new__(LocalNRL)
# Otherwise delegate to the remote NRL client to deal with all kinds
# of remote resources (currently only HTTP).
return super(NRL, cls).__new__(RemoteNRL)
def __init__(self,
debug=False,
timeout=120,
include_providers=None,
exclude_providers=None,
credentials=None):
"""
:type routing_type: str
:param routing_type: The type of
router to initialize. For details see :func:`RoutingClient`.
:type exclude_providers: str or list of str
:param exclude_providers: Get no data from these providers. Can be
the full HTTP address or one of the shortcuts ObsPy knows about.
:type include_providers: str or list of str
:param include_providers: Get data only from these providers. Can be
the full HTTP address of one of the shortcuts ObsPy knows about.
:type credentials: dict
:param credentials: Credentials for the individual data centers as a
dictionary that maps base url of FDSN web service to either
username/password or EIDA token, e.g.
``credentials={
'geofon.gfz-potsdam.de': {'eida_token': 'my_token_file.txt'},
'service.iris.edu': {'user': '******', 'password': '******'}
'EIDA_TOKEN': '/path/to/token.txt'
}``
The root level ``'EIDA_TOKEN'`` will be applied to all data centers
that claim to support the ``/auth`` route and don't have data
center specific credentials.
You can also use a URL mapping as for the normal FDSN client
instead of the URL.
"""
HTTPClient.__init__(self, debug=debug, timeout=timeout)
self.include_providers = include_providers
self.exclude_providers = exclude_providers
# Parse credentials.
self.credentials = {}
for key, value in (credentials or {}).items():
if key == "EIDA_TOKEN":
self.credentials[key] = value
# Map, if necessary.
if key in URL_MAPPINGS:
key = URL_MAPPINGS[key]
# Make sure urlparse works correctly.
if not key.startswith("http"):
key = "http://" + key
# Only use the location.
self.credentials[urlparse(key).netloc] = value
def __new__(cls, root=None):
# root provided and it's no web URL
if root:
scheme = urlparse(root).scheme
if scheme in ('http', 'https'):
return super(NRL, cls).__new__(RemoteNRL)
# Check if it's really a folder on the file-system.
if not os.path.isdir(root):
msg = ("Provided path '{}' seems to be a local file path "
"but the directory does not exist.").format(
root)
raise ValueError(msg)
return super(NRL, cls).__new__(LocalNRL)
# Otherwise delegate to the remote NRL client to deal with all kinds
# of remote resources (currently only HTTP).
return super(NRL, cls).__new__(RemoteNRL)
def __init__(self, debug=False, timeout=120, include_providers=None,
exclude_providers=None, credentials=None):
"""
:type routing_type: str
:param routing_type: The type of
router to initialize. For details see :func:`RoutingClient`.
:type exclude_providers: str or list of str
:param exclude_providers: Get no data from these providers. Can be
the full HTTP address or one of the shortcuts ObsPy knows about.
:type include_providers: str or list of str
:param include_providers: Get data only from these providers. Can be
the full HTTP address of one of the shortcuts ObsPy knows about.
:type credentials: dict
:param credentials: Credentials for the individual data centers as a
dictionary that maps base url of FDSN web service to either
username/password or EIDA token, e.g.
``credentials={
'geofon.gfz-potsdam.de': {'eida_token': 'my_token_file.txt'},
'service.iris.edu': {'user': '******', 'password': '******'}
'EIDA_TOKEN': '/path/to/token.txt'
}``
The root level ``'EIDA_TOKEN'`` will be applied to all data centers
that claim to support the ``/auth`` route and don't have data
center specific credentials.
You can also use a URL mapping as for the normal FDSN client
instead of the URL.
"""
HTTPClient.__init__(self, debug=debug, timeout=timeout)
self.include_providers = include_providers
self.exclude_providers = exclude_providers
# Parse credentials.
self.credentials = {}
for key, value in (credentials or {}).items():
if key == "EIDA_TOKEN":
self.credentials[key] = value
# Map, if necessary.
if key in URL_MAPPINGS:
key = URL_MAPPINGS[key]
# Make sure urlparse works correctly.
if not key.startswith("http"):
key = "http://" + key
# Only use the location.
self.credentials[urlparse(key).netloc] = value
def _strip_protocol(url):
url = urlparse(url)
return url.netloc + url.path
def _create_report(ttrs, timetaken, log, server, hostname, sorted_tests,
ci_url=None, pr_url=None, import_failures=None):
"""
If `server` is specified without URL scheme, 'https://' will be used as a
default.
"""
# import additional libraries here to speed up normal tests
from future import standard_library
with standard_library.hooks():
import urllib.parse
import codecs
from xml.etree import ElementTree
from xml.sax.saxutils import escape
if import_failures is None:
import_failures = {}
timestamp = int(time.time())
result = {'timestamp': timestamp}
result['slowest_tests'] = [("%0.3fs" % dt, "%s" % desc)
for (desc, dt) in sorted_tests[:20]]
result['timetaken'] = timetaken
if log:
try:
data = codecs.open(log, 'r', encoding='UTF-8').read()
result['install_log'] = escape(data)
except Exception:
print("Cannot open log file %s" % log)
# get ObsPy module versions
result['obspy'] = {}
tests = 0
errors = 0
failures = 0
skipped = 0
try:
installed = get_git_version()
except Exception:
installed = ''
result['obspy']['installed'] = installed
for module in sorted(ALL_MODULES):
result['obspy'][module] = {}
result['obspy'][module]['installed'] = installed
# add a failed-to-import test module to report with an error
if module in import_failures:
result['obspy'][module]['timetaken'] = 0
result['obspy'][module]['tested'] = True
result['obspy'][module]['tests'] = 1
# can't say how many tests would have been in that suite so just
# leave 0
result['obspy'][module]['skipped'] = 0
result['obspy'][module]['failures'] = {}
result['obspy'][module]['errors'] = {
'f%s' % (errors): import_failures[module]}
tests += 1
errors += 1
continue
if module not in ttrs:
continue
# test results
ttr = ttrs[module]
result['obspy'][module]['timetaken'] = ttr.__dict__['timetaken']
result['obspy'][module]['tested'] = True
result['obspy'][module]['tests'] = ttr.testsRun
skipped += len(ttr.skipped)
result['obspy'][module]['skipped'] = len(ttr.skipped)
tests += ttr.testsRun
# depending on module type either use failure (network related modules)
# or errors (all others)
result['obspy'][module]['errors'] = {}
result['obspy'][module]['failures'] = {}
if module in NETWORK_MODULES:
for _, text in ttr.errors:
result['obspy'][module]['failures']['f%s' % (failures)] = text
failures += 1
for _, text in ttr.failures:
result['obspy'][module]['failures']['f%s' % (failures)] = text
failures += 1
else:
for _, text in ttr.errors:
result['obspy'][module]['errors']['f%s' % (errors)] = text
errors += 1
for _, text in ttr.failures:
result['obspy'][module]['errors']['f%s' % (errors)] = text
errors += 1
# get dependencies
result['dependencies'] = {}
for module in DEPENDENCIES:
if module == "pep8-naming":
module_ = "pep8ext_naming"
else:
module_ = module
temp = module_.split('.')
try:
mod = __import__(module_,
fromlist=[native_str(temp[1:])])
except ImportError:
version_ = '---'
else:
try:
version_ = mod.__version__
except AttributeError:
version_ = '???'
result['dependencies'][module] = version_
# get system / environment settings
result['platform'] = {}
for func in ['system', 'release', 'version', 'machine',
'processor', 'python_version', 'python_implementation',
'python_compiler', 'architecture']:
try:
temp = getattr(platform, func)()
if isinstance(temp, tuple):
temp = temp[0]
result['platform'][func] = temp
except Exception:
result['platform'][func] = ''
# set node name to hostname if set
result['platform']['node'] = hostname
# post only the first part of the node name (only applies to MacOS X)
try:
result['platform']['node'] = result['platform']['node'].split('.')[0]
except Exception:
pass
# test results
result['tests'] = tests
result['errors'] = errors
result['failures'] = failures
result['skipped'] = skipped
# try to append info on skipped tests:
result['skipped_tests_details'] = []
try:
for module, testresult_ in ttrs.items():
if testresult_.skipped:
for skipped_test, skip_message in testresult_.skipped:
result['skipped_tests_details'].append(
(module, skipped_test.__module__,
skipped_test.__class__.__name__,
skipped_test._testMethodName, skip_message))
except Exception:
exc_type, exc_value, exc_tb = sys.exc_info()
print("\n".join(traceback.format_exception(exc_type, exc_value,
exc_tb)))
result['skipped_tests_details'] = []
if ci_url is not None:
result['ciurl'] = ci_url
if pr_url is not None:
result['prurl'] = pr_url
# generate XML document
def _dict2xml(doc, result):
for key, value in result.items():
key = key.split('(')[0].strip()
if isinstance(value, dict):
child = ElementTree.SubElement(doc, key)
_dict2xml(child, value)
elif value is not None:
if isinstance(value, (str, native_str)):
ElementTree.SubElement(doc, key).text = value
elif isinstance(value, (str, native_str)):
ElementTree.SubElement(doc, key).text = str(value, 'utf-8')
else:
ElementTree.SubElement(doc, key).text = str(value)
else:
ElementTree.SubElement(doc, key)
root = ElementTree.Element("report")
_dict2xml(root, result)
xml_doc = ElementTree.tostring(root)
print()
# send result to report server
params = urllib.parse.urlencode({
'timestamp': timestamp,
'system': result['platform']['system'],
'python_version': result['platform']['python_version'],
'architecture': result['platform']['architecture'],
'tests': tests,
'failures': failures,
'errors': errors,
'modules': len(ttrs) + len(import_failures),
'xml': xml_doc
})
headers = {"Content-type": "application/x-www-form-urlencoded",
"Accept": "text/plain"}
url = server
if not urlparse(url).scheme:
url = "https://" + url
response = requests.post(url=url, headers=headers,
data=params.encode('UTF-8'))
# get the response
if response.status_code == 200:
report_url = response.json().get('url', server)
print('Your test results have been reported and are available at: '
'{}\nThank you!'.format(report_url))
# handle errors
else:
print("Error: Could not sent a test report to %s." % (server))
print(response.reason)
def _createReport(ttrs, timetaken, log, server, hostname):
# import additional libraries here to speed up normal tests
from obspy.core import compatibility
from xml.sax.saxutils import escape
import codecs
from xml.etree import ElementTree as etree
timestamp = int(time.time())
result = {'timestamp': timestamp}
result['timetaken'] = timetaken
if log:
try:
data = codecs.open(log, 'r', encoding='UTF-8').read()
result['install_log'] = escape(data)
except:
print(("Cannot open log file %s" % log))
# get ObsPy module versions
result['obspy'] = {}
tests = 0
errors = 0
failures = 0
skipped = 0
try:
installed = get_git_version()
except:
installed = ''
result['obspy']['installed'] = installed
for module in sorted(ALL_MODULES):
result['obspy'][module] = {}
if module not in ttrs:
continue
result['obspy'][module]['installed'] = installed
# test results
ttr = ttrs[module]
result['obspy'][module]['timetaken'] = ttr.__dict__['timetaken']
result['obspy'][module]['tested'] = True
result['obspy'][module]['tests'] = ttr.testsRun
# skipped is not supported for Python < 2.7
try:
skipped += len(ttr.skipped)
result['obspy'][module]['skipped'] = len(ttr.skipped)
except AttributeError:
skipped = ''
result['obspy'][module]['skipped'] = ''
tests += ttr.testsRun
# depending on module type either use failure (network related modules)
# or errors (all others)
result['obspy'][module]['errors'] = {}
result['obspy'][module]['failures'] = {}
if module in NETWORK_MODULES:
for _, text in ttr.errors:
result['obspy'][module]['failures']['f%s' % (failures)] = text
failures += 1
for _, text in ttr.failures:
result['obspy'][module]['failures']['f%s' % (failures)] = text
failures += 1
else:
for _, text in ttr.errors:
result['obspy'][module]['errors']['f%s' % (errors)] = text
errors += 1
for _, text in ttr.failures:
result['obspy'][module]['errors']['f%s' % (errors)] = text
errors += 1
# get dependencies
result['dependencies'] = {}
for module in DEPENDENCIES:
temp = module.split('.')
try:
mod = __import__(module,
fromlist=[native_str(temp[1:])])
if module == '_omnipy':
result['dependencies'][module] = mod.coreVersion()
else:
result['dependencies'][module] = mod.__version__
except ImportError:
result['dependencies'][module] = ''
# get system / environment settings
result['platform'] = {}
for func in ['system', 'release', 'version', 'machine',
'processor', 'python_version', 'python_implementation',
'python_compiler', 'architecture']:
try:
temp = getattr(platform, func)()
if isinstance(temp, tuple):
temp = temp[0]
result['platform'][func] = temp
except:
result['platform'][func] = ''
# set node name to hostname if set
result['platform']['node'] = hostname
# post only the first part of the node name (only applies to MacOS X)
try:
result['platform']['node'] = result['platform']['node'].split('.')[0]
except:
pass
# test results
result['tests'] = tests
result['errors'] = errors
result['failures'] = failures
result['skipped'] = skipped
# generate XML document
def _dict2xml(doc, result):
for key, value in result.items():
key = key.split('(')[0].strip()
if isinstance(value, dict):
child = etree.SubElement(doc, key)
_dict2xml(child, value)
elif value is not None:
if isinstance(value, (str, native_str)):
etree.SubElement(doc, key).text = value
elif isinstance(value, (str, native_str)):
etree.SubElement(doc, key).text = str(value, 'utf-8')
else:
etree.SubElement(doc, key).text = str(value)
else:
etree.SubElement(doc, key)
root = etree.Element("report")
_dict2xml(root, result)
xml_doc = etree.tostring(root)
print()
# send result to report server
params = compatibility.urlencode({
'timestamp': timestamp,
'system': result['platform']['system'],
'python_version': result['platform']['python_version'],
'architecture': result['platform']['architecture'],
'tests': tests,
'failures': failures,
'errors': errors,
'modules': len(ttrs),
'xml': xml_doc
})
headers = {"Content-type": "application/x-www-form-urlencoded",
"Accept": "text/plain"}
conn = compatibility.HTTPConnection(server)
conn.request("POST", "/", params, headers)
# get the response
response = conn.getresponse()
# handle redirect
if response.status == 301:
o = compatibility.urlparse(response.msg['location'])
conn = compatibility.HTTPConnection(o.netloc)
conn.request("POST", o.path, params, headers)
# get the response
response = conn.getresponse()
# handle errors
if response.status == 200:
print(("Test report has been sent to %s. Thank you!" % (server)))
else:
print(("Error: Could not sent a test report to %s." % (server)))
print((response.reason))
def _strip_protocol(url):
url = urlparse(url)
return url.netloc + url.path
