def _download_bulk(r): # Figure out the passed credentials, if any. Two possibilities: # (1) User and password, given explicitly for the base URLs (or an # explicity given `eida_token` key per URL). # (2) A global EIDA_TOKEN key. It will be used for all services that # don't have explicit credentials and also support the `/auth` route. credentials = r["credentials"].get(urlparse(r["endpoint"]).netloc, {}) c = client.Client(r["endpoint"], debug=r["debug"], timeout=r["timeout"], **credentials) if not credentials and "EIDA_TOKEN" in r["credentials"] and \ c._has_eida_auth: c.set_eida_token(r["credentials"]["EIDA_TOKEN"]) if r["data_type"] == "waveform": fct = c.get_waveforms_bulk service = c.services["dataselect"] elif r["data_type"] == "station": fct = c.get_stations_bulk service = c.services["station"] # Keep only kwargs that are supported by this particular service. kwargs = {k: v for k, v in r["kwargs"].items() if k in service} bulk_str = "" for key, value in kwargs.items(): bulk_str += "%s=%s\n" % (key, str(value)) try: return fct(bulk_str + r["bulk_str"]) except FDSNException: return None
def _download_bulk(r): # Figure out the passed credentials, if any. Two possibilities: # (1) User and password, given explicitly for the base URLs (or an # explicity given `eida_token` key per URL). # (2) A global EIDA_TOKEN key. It will be used for all services that # don't have explicit credentials and also support the `/auth` route. credentials = r["credentials"].get(urlparse(r["endpoint"]).netloc, {}) c = client.Client(r["endpoint"], debug=r["debug"], timeout=r["timeout"], **credentials) if not credentials and "EIDA_TOKEN" in r["credentials"] and \ c._has_eida_auth: c.set_eida_token(r["credentials"]["EIDA_TOKEN"]) if r["data_type"] == "waveform": fct = c.get_waveforms_bulk service = c.services["dataselect"] elif r["data_type"] == "station": fct = c.get_stations_bulk service = c.services["station"] # Keep only kwargs that are supported by this particular service. kwargs = {k: v for k, v in r["kwargs"].items() if k in service} bulk_str = "" for key, value in kwargs.items(): bulk_str += "%s=%s\n" % (key, str(value)) try: return fct(bulk_str + r["bulk_str"]) except FDSNException: return None
def __new__(cls, root=None): # root provided and it's no web URL if root: scheme = urlparse(root).scheme if scheme in ('http', 'https'): return super(NRL, cls).__new__(RemoteNRL) # Check if it's really a folder on the file-system. if not os.path.isdir(root): msg = ("Provided path '{}' seems to be a local file path " "but the directory does not exist.").format(root) raise ValueError(msg) return super(NRL, cls).__new__(LocalNRL) # Otherwise delegate to the remote NRL client to deal with all kinds # of remote resources (currently only HTTP). return super(NRL, cls).__new__(RemoteNRL)
def __init__(self, debug=False, timeout=120, include_providers=None, exclude_providers=None, credentials=None): """ :type routing_type: str :param routing_type: The type of router to initialize. For details see :func:`RoutingClient`. :type exclude_providers: str or list of str :param exclude_providers: Get no data from these providers. Can be the full HTTP address or one of the shortcuts ObsPy knows about. :type include_providers: str or list of str :param include_providers: Get data only from these providers. Can be the full HTTP address of one of the shortcuts ObsPy knows about. :type credentials: dict :param credentials: Credentials for the individual data centers as a dictionary that maps base url of FDSN web service to either username/password or EIDA token, e.g. ``credentials={ 'geofon.gfz-potsdam.de': {'eida_token': 'my_token_file.txt'}, 'service.iris.edu': {'user': '******', 'password': '******'} 'EIDA_TOKEN': '/path/to/token.txt' }`` The root level ``'EIDA_TOKEN'`` will be applied to all data centers that claim to support the ``/auth`` route and don't have data center specific credentials. You can also use a URL mapping as for the normal FDSN client instead of the URL. """ HTTPClient.__init__(self, debug=debug, timeout=timeout) self.include_providers = include_providers self.exclude_providers = exclude_providers # Parse credentials. self.credentials = {} for key, value in (credentials or {}).items(): if key == "EIDA_TOKEN": self.credentials[key] = value # Map, if necessary. if key in URL_MAPPINGS: key = URL_MAPPINGS[key] # Make sure urlparse works correctly. if not key.startswith("http"): key = "http://" + key # Only use the location. self.credentials[urlparse(key).netloc] = value
def __new__(cls, root=None): # root provided and it's no web URL if root: scheme = urlparse(root).scheme if scheme in ('http', 'https'): return super(NRL, cls).__new__(RemoteNRL) # Check if it's really a folder on the file-system. if not os.path.isdir(root): msg = ("Provided path '{}' seems to be a local file path " "but the directory does not exist.").format( root) raise ValueError(msg) return super(NRL, cls).__new__(LocalNRL) # Otherwise delegate to the remote NRL client to deal with all kinds # of remote resources (currently only HTTP). return super(NRL, cls).__new__(RemoteNRL)
def __init__(self, debug=False, timeout=120, include_providers=None, exclude_providers=None, credentials=None): """ :type routing_type: str :param routing_type: The type of router to initialize. For details see :func:`RoutingClient`. :type exclude_providers: str or list of str :param exclude_providers: Get no data from these providers. Can be the full HTTP address or one of the shortcuts ObsPy knows about. :type include_providers: str or list of str :param include_providers: Get data only from these providers. Can be the full HTTP address of one of the shortcuts ObsPy knows about. :type credentials: dict :param credentials: Credentials for the individual data centers as a dictionary that maps base url of FDSN web service to either username/password or EIDA token, e.g. ``credentials={ 'geofon.gfz-potsdam.de': {'eida_token': 'my_token_file.txt'}, 'service.iris.edu': {'user': '******', 'password': '******'} 'EIDA_TOKEN': '/path/to/token.txt' }`` The root level ``'EIDA_TOKEN'`` will be applied to all data centers that claim to support the ``/auth`` route and don't have data center specific credentials. You can also use a URL mapping as for the normal FDSN client instead of the URL. """ HTTPClient.__init__(self, debug=debug, timeout=timeout) self.include_providers = include_providers self.exclude_providers = exclude_providers # Parse credentials. self.credentials = {} for key, value in (credentials or {}).items(): if key == "EIDA_TOKEN": self.credentials[key] = value # Map, if necessary. if key in URL_MAPPINGS: key = URL_MAPPINGS[key] # Make sure urlparse works correctly. if not key.startswith("http"): key = "http://" + key # Only use the location. self.credentials[urlparse(key).netloc] = value
def _strip_protocol(url): url = urlparse(url) return url.netloc + url.path
def _create_report(ttrs, timetaken, log, server, hostname, sorted_tests, ci_url=None, pr_url=None, import_failures=None): """ If `server` is specified without URL scheme, 'https://' will be used as a default. """ # import additional libraries here to speed up normal tests from future import standard_library with standard_library.hooks(): import urllib.parse import codecs from xml.etree import ElementTree from xml.sax.saxutils import escape if import_failures is None: import_failures = {} timestamp = int(time.time()) result = {'timestamp': timestamp} result['slowest_tests'] = [("%0.3fs" % dt, "%s" % desc) for (desc, dt) in sorted_tests[:20]] result['timetaken'] = timetaken if log: try: data = codecs.open(log, 'r', encoding='UTF-8').read() result['install_log'] = escape(data) except Exception: print("Cannot open log file %s" % log) # get ObsPy module versions result['obspy'] = {} tests = 0 errors = 0 failures = 0 skipped = 0 try: installed = get_git_version() except Exception: installed = '' result['obspy']['installed'] = installed for module in sorted(ALL_MODULES): result['obspy'][module] = {} result['obspy'][module]['installed'] = installed # add a failed-to-import test module to report with an error if module in import_failures: result['obspy'][module]['timetaken'] = 0 result['obspy'][module]['tested'] = True result['obspy'][module]['tests'] = 1 # can't say how many tests would have been in that suite so just # leave 0 result['obspy'][module]['skipped'] = 0 result['obspy'][module]['failures'] = {} result['obspy'][module]['errors'] = { 'f%s' % (errors): import_failures[module]} tests += 1 errors += 1 continue if module not in ttrs: continue # test results ttr = ttrs[module] result['obspy'][module]['timetaken'] = ttr.__dict__['timetaken'] result['obspy'][module]['tested'] = True result['obspy'][module]['tests'] = ttr.testsRun skipped += len(ttr.skipped) result['obspy'][module]['skipped'] = len(ttr.skipped) tests += ttr.testsRun # depending on module type either use failure (network related modules) # or errors (all others) result['obspy'][module]['errors'] = {} result['obspy'][module]['failures'] = {} if module in NETWORK_MODULES: for _, text in ttr.errors: result['obspy'][module]['failures']['f%s' % (failures)] = text failures += 1 for _, text in ttr.failures: result['obspy'][module]['failures']['f%s' % (failures)] = text failures += 1 else: for _, text in ttr.errors: result['obspy'][module]['errors']['f%s' % (errors)] = text errors += 1 for _, text in ttr.failures: result['obspy'][module]['errors']['f%s' % (errors)] = text errors += 1 # get dependencies result['dependencies'] = {} for module in DEPENDENCIES: if module == "pep8-naming": module_ = "pep8ext_naming" else: module_ = module temp = module_.split('.') try: mod = __import__(module_, fromlist=[native_str(temp[1:])]) except ImportError: version_ = '---' else: try: version_ = mod.__version__ except AttributeError: version_ = '???' result['dependencies'][module] = version_ # get system / environment settings result['platform'] = {} for func in ['system', 'release', 'version', 'machine', 'processor', 'python_version', 'python_implementation', 'python_compiler', 'architecture']: try: temp = getattr(platform, func)() if isinstance(temp, tuple): temp = temp[0] result['platform'][func] = temp except Exception: result['platform'][func] = '' # set node name to hostname if set result['platform']['node'] = hostname # post only the first part of the node name (only applies to MacOS X) try: result['platform']['node'] = result['platform']['node'].split('.')[0] except Exception: pass # test results result['tests'] = tests result['errors'] = errors result['failures'] = failures result['skipped'] = skipped # try to append info on skipped tests: result['skipped_tests_details'] = [] try: for module, testresult_ in ttrs.items(): if testresult_.skipped: for skipped_test, skip_message in testresult_.skipped: result['skipped_tests_details'].append( (module, skipped_test.__module__, skipped_test.__class__.__name__, skipped_test._testMethodName, skip_message)) except Exception: exc_type, exc_value, exc_tb = sys.exc_info() print("\n".join(traceback.format_exception(exc_type, exc_value, exc_tb))) result['skipped_tests_details'] = [] if ci_url is not None: result['ciurl'] = ci_url if pr_url is not None: result['prurl'] = pr_url # generate XML document def _dict2xml(doc, result): for key, value in result.items(): key = key.split('(')[0].strip() if isinstance(value, dict): child = ElementTree.SubElement(doc, key) _dict2xml(child, value) elif value is not None: if isinstance(value, (str, native_str)): ElementTree.SubElement(doc, key).text = value elif isinstance(value, (str, native_str)): ElementTree.SubElement(doc, key).text = str(value, 'utf-8') else: ElementTree.SubElement(doc, key).text = str(value) else: ElementTree.SubElement(doc, key) root = ElementTree.Element("report") _dict2xml(root, result) xml_doc = ElementTree.tostring(root) print() # send result to report server params = urllib.parse.urlencode({ 'timestamp': timestamp, 'system': result['platform']['system'], 'python_version': result['platform']['python_version'], 'architecture': result['platform']['architecture'], 'tests': tests, 'failures': failures, 'errors': errors, 'modules': len(ttrs) + len(import_failures), 'xml': xml_doc }) headers = {"Content-type": "application/x-www-form-urlencoded", "Accept": "text/plain"} url = server if not urlparse(url).scheme: url = "https://" + url response = requests.post(url=url, headers=headers, data=params.encode('UTF-8')) # get the response if response.status_code == 200: report_url = response.json().get('url', server) print('Your test results have been reported and are available at: ' '{}\nThank you!'.format(report_url)) # handle errors else: print("Error: Could not sent a test report to %s." % (server)) print(response.reason)
def _createReport(ttrs, timetaken, log, server, hostname): # import additional libraries here to speed up normal tests from obspy.core import compatibility from xml.sax.saxutils import escape import codecs from xml.etree import ElementTree as etree timestamp = int(time.time()) result = {'timestamp': timestamp} result['timetaken'] = timetaken if log: try: data = codecs.open(log, 'r', encoding='UTF-8').read() result['install_log'] = escape(data) except: print(("Cannot open log file %s" % log)) # get ObsPy module versions result['obspy'] = {} tests = 0 errors = 0 failures = 0 skipped = 0 try: installed = get_git_version() except: installed = '' result['obspy']['installed'] = installed for module in sorted(ALL_MODULES): result['obspy'][module] = {} if module not in ttrs: continue result['obspy'][module]['installed'] = installed # test results ttr = ttrs[module] result['obspy'][module]['timetaken'] = ttr.__dict__['timetaken'] result['obspy'][module]['tested'] = True result['obspy'][module]['tests'] = ttr.testsRun # skipped is not supported for Python < 2.7 try: skipped += len(ttr.skipped) result['obspy'][module]['skipped'] = len(ttr.skipped) except AttributeError: skipped = '' result['obspy'][module]['skipped'] = '' tests += ttr.testsRun # depending on module type either use failure (network related modules) # or errors (all others) result['obspy'][module]['errors'] = {} result['obspy'][module]['failures'] = {} if module in NETWORK_MODULES: for _, text in ttr.errors: result['obspy'][module]['failures']['f%s' % (failures)] = text failures += 1 for _, text in ttr.failures: result['obspy'][module]['failures']['f%s' % (failures)] = text failures += 1 else: for _, text in ttr.errors: result['obspy'][module]['errors']['f%s' % (errors)] = text errors += 1 for _, text in ttr.failures: result['obspy'][module]['errors']['f%s' % (errors)] = text errors += 1 # get dependencies result['dependencies'] = {} for module in DEPENDENCIES: temp = module.split('.') try: mod = __import__(module, fromlist=[native_str(temp[1:])]) if module == '_omnipy': result['dependencies'][module] = mod.coreVersion() else: result['dependencies'][module] = mod.__version__ except ImportError: result['dependencies'][module] = '' # get system / environment settings result['platform'] = {} for func in ['system', 'release', 'version', 'machine', 'processor', 'python_version', 'python_implementation', 'python_compiler', 'architecture']: try: temp = getattr(platform, func)() if isinstance(temp, tuple): temp = temp[0] result['platform'][func] = temp except: result['platform'][func] = '' # set node name to hostname if set result['platform']['node'] = hostname # post only the first part of the node name (only applies to MacOS X) try: result['platform']['node'] = result['platform']['node'].split('.')[0] except: pass # test results result['tests'] = tests result['errors'] = errors result['failures'] = failures result['skipped'] = skipped # generate XML document def _dict2xml(doc, result): for key, value in result.items(): key = key.split('(')[0].strip() if isinstance(value, dict): child = etree.SubElement(doc, key) _dict2xml(child, value) elif value is not None: if isinstance(value, (str, native_str)): etree.SubElement(doc, key).text = value elif isinstance(value, (str, native_str)): etree.SubElement(doc, key).text = str(value, 'utf-8') else: etree.SubElement(doc, key).text = str(value) else: etree.SubElement(doc, key) root = etree.Element("report") _dict2xml(root, result) xml_doc = etree.tostring(root) print() # send result to report server params = compatibility.urlencode({ 'timestamp': timestamp, 'system': result['platform']['system'], 'python_version': result['platform']['python_version'], 'architecture': result['platform']['architecture'], 'tests': tests, 'failures': failures, 'errors': errors, 'modules': len(ttrs), 'xml': xml_doc }) headers = {"Content-type": "application/x-www-form-urlencoded", "Accept": "text/plain"} conn = compatibility.HTTPConnection(server) conn.request("POST", "/", params, headers) # get the response response = conn.getresponse() # handle redirect if response.status == 301: o = compatibility.urlparse(response.msg['location']) conn = compatibility.HTTPConnection(o.netloc) conn.request("POST", o.path, params, headers) # get the response response = conn.getresponse() # handle errors if response.status == 200: print(("Test report has been sent to %s. Thank you!" % (server))) else: print(("Error: Could not sent a test report to %s." % (server))) print((response.reason))
def _strip_protocol(url): url = urlparse(url) return url.netloc + url.path