def test_urllib_imports_cm(self):
with standard_library.hooks():
import urllib
import urllib.parse
import urllib.request
import urllib.robotparser
import urllib.error
import urllib.response
self.assertTrue(True)
def test_urllib_imports_cm(self):
with standard_library.hooks():
import urllib
import urllib.parse
import urllib.request
import urllib.robotparser
import urllib.error
import urllib.response
self.assertTrue(True)
def download_setuptools(version=DEFAULT_VERSION,
download_base=DEFAULT_URL,
to_dir=os.curdir,
delay=15):
"""Download setuptools from a specified location and return its filename
`version` should be a valid setuptools version number that is available
as an egg for download under the `download_base` URL (which should end
with a '/'). `to_dir` is the directory where the egg will be downloaded.
`delay` is the number of seconds to pause before an actual download attempt.
"""
#import urllib.request, urllib.error, urllib.parse, shutil
from future.standard_library import hooks
with hooks():
from urllib.parse import urlparse, urlencode
from urllib.request import urlopen, Request
from urllib.error import HTTPError
import shutil
egg_name = "setuptools-%s-py%s.egg" % (version, sys.version[:3])
url = download_base + egg_name
saveto = os.path.join(to_dir, egg_name)
src = dst = None
if not os.path.exists(saveto): # Avoid repeated downloads
try:
from distutils import log
if delay:
log.warn(
"""
---------------------------------------------------------------------------
This script requires setuptools version %s to run (even to display
help). I will attempt to download it for you (from
%s), but
you may need to enable firewall access for this script first.
I will start the download in %d seconds.
(Note: if this machine does not have network access, please obtain the file
%s
and place it in this directory before rerunning this script.)
---------------------------------------------------------------------------""",
version, download_base, delay, url)
from time import sleep
sleep(delay)
log.warn("Downloading %s", url)
src = urlopen(url)
# Read/write all in one block, so we don't create a corrupt file
# if the download is interrupted.
data = _validate_md5(egg_name, src.read())
dst = open(saveto, "wb")
dst.write(data)
finally:
if src: src.close()
if dst: dst.close()
return os.path.realpath(saveto)
def test_urllib(self):
"""
Tests that urllib isn't changed from under our feet. (This might not
even be a problem?)
"""
from future import standard_library
import urllib
orig_file = urllib.__file__
with standard_library.hooks():
import urllib.response
self.assertEqual(orig_file, urllib.__file__)
def test_urllib(self):
"""
Tests that urllib isn't changed from under our feet. (This might not
even be a problem?)
"""
from future import standard_library
import urllib
orig_file = urllib.__file__
with standard_library.hooks():
import urllib.response
self.assertEqual(orig_file, urllib.__file__)
def test_urllib(self):
"""
This should perhaps fail: importing urllib first means that the import hooks
won't be consulted when importing urllib.response.
"""
import urllib
print(urllib.__file__)
from future import standard_library
with standard_library.hooks():
import urllib.response
print(urllib.__file__)
print(urllib.response.__file__)
def download_setuptools(
version=DEFAULT_VERSION, download_base=DEFAULT_URL, to_dir=os.curdir,
delay = 15
):
"""Download setuptools from a specified location and return its filename
`version` should be a valid setuptools version number that is available
as an egg for download under the `download_base` URL (which should end
with a '/'). `to_dir` is the directory where the egg will be downloaded.
`delay` is the number of seconds to pause before an actual download attempt.
"""
#import urllib.request, urllib.error, urllib.parse, shutil
from future.standard_library import hooks
with hooks():
from urllib.parse import urlparse, urlencode
from urllib.request import urlopen, Request
from urllib.error import HTTPError
import shutil
egg_name = "setuptools-%s-py%s.egg" % (version,sys.version[:3])
url = download_base + egg_name
saveto = os.path.join(to_dir, egg_name)
src = dst = None
if not os.path.exists(saveto): # Avoid repeated downloads
try:
from distutils import log
if delay:
log.warn("""
---------------------------------------------------------------------------
This script requires setuptools version %s to run (even to display
help). I will attempt to download it for you (from
%s), but
you may need to enable firewall access for this script first.
I will start the download in %d seconds.
(Note: if this machine does not have network access, please obtain the file
%s
and place it in this directory before rerunning this script.)
---------------------------------------------------------------------------""",
version, download_base, delay, url
); from time import sleep; sleep(delay)
log.warn("Downloading %s", url)
src = urlopen(url)
# Read/write all in one block, so we don't create a corrupt file
# if the download is interrupted.
data = _validate_md5(egg_name, src.read())
dst = open(saveto,"wb"); dst.write(data)
finally:
if src: src.close()
if dst: dst.close()
return os.path.realpath(saveto)
def test_can_import_several(self):
"""
This test fails if e.g. future/standard_library/email/header.py contains:
from future import standard_library
standard_library.remove_hooks(keep_sys_modules=True)
"""
with standard_library.hooks():
import urllib.parse
import urllib.request
import http.server
from test import support
for m in [urllib.parse, urllib.request, http.server, support]:
self.assertTrue(m is not None)
def test_can_import_several(self):
"""
This test failed in v0.12-pre if e.g.
future/standard_library/email/header.py contained:
from future import standard_library
standard_library.remove_hooks()
"""
import future.moves.urllib.parse as urllib_parse
import future.moves.urllib.request as urllib_request
with standard_library.hooks():
import http.server
for m in [urllib_parse, urllib_request, http.server]:
self.assertTrue(m is not None)
def test_can_import_several(self):
"""
This test failed in v0.12-pre if e.g.
future/standard_library/email/header.py contained:
from future import standard_library
standard_library.remove_hooks()
"""
import future.moves.urllib.parse as urllib_parse
import future.moves.urllib.request as urllib_request
with standard_library.hooks():
import http.server
for m in [urllib_parse, urllib_request, http.server]:
self.assertTrue(m is not None)
def test_seishub(self):
"""Test the seishub method, use obspy default seishub client."""
from obspy.core.event import Catalog, Event, Origin, Pick
from obspy.core.event import WaveformStreamID
from obspy import UTCDateTime
import warnings
from future import standard_library
with standard_library.hooks():
from urllib.request import URLError
t = UTCDateTime(2009, 9, 3)
test_cat = Catalog()
test_cat.append(Event())
test_cat[0].origins.append(Origin())
test_cat[0].origins[0].time = t
test_cat[0].origins[0].latitude = 45
test_cat[0].origins[0].longitude = 45
test_cat[0].origins[0].depth = 5000
test_cat[0].\
picks.append(Pick(waveform_id=WaveformStreamID(station_code='MANZ',
channel_code='EHZ',
network_code='BW'),
phase_hint='PG', time=t + 2000))
test_cat[0].\
picks.append(Pick(waveform_id=WaveformStreamID(station_code='MANZ',
channel_code='EHN',
network_code='BW'),
phase_hint='SG', time=t + 2005))
test_cat[0].\
picks.append(Pick(waveform_id=WaveformStreamID(station_code='MANZ',
channel_code='EHE',
network_code='BW'),
phase_hint='SG', time=t + 2005.5))
test_url = 'http://teide.geophysik.uni-muenchen.de:8080'
try:
template = from_seishub(test_cat, url=test_url, lowcut=1.0,
highcut=5.0, samp_rate=20, filt_order=4,
length=3, prepick=0.5, swin='all',
process_len=300)
except URLError:
warnings.warn('Timed out connection to seishub')
if 'template' in locals():
self.assertEqual(len(template), 3)
def test_seishub(self):
"""Test the seishub method, use obspy default seishub client."""
from obspy.core.event import Catalog, Event, Origin, Pick
from obspy.core.event import WaveformStreamID
from obspy import UTCDateTime
import warnings
from future import standard_library
with standard_library.hooks():
from urllib.request import URLError
t = UTCDateTime(2009, 9, 3)
test_cat = Catalog()
test_cat.append(Event())
test_cat[0].origins.append(Origin())
test_cat[0].origins[0].time = t
test_cat[0].origins[0].latitude = 45
test_cat[0].origins[0].longitude = 45
test_cat[0].origins[0].depth = 5000
test_cat[0].\
picks.append(Pick(waveform_id=WaveformStreamID(station_code='MANZ',
channel_code='EHZ',
network_code='BW'),
phase_hint='PG', time=t + 2000))
test_cat[0].\
picks.append(Pick(waveform_id=WaveformStreamID(station_code='MANZ',
channel_code='EHN',
network_code='BW'),
phase_hint='SG', time=t + 2005))
test_cat[0].\
picks.append(Pick(waveform_id=WaveformStreamID(station_code='MANZ',
channel_code='EHE',
network_code='BW'),
phase_hint='SG', time=t + 2005.5))
test_url = 'http://teide.geophysik.uni-muenchen.de:8080'
try:
template = from_seishub(test_cat, url=test_url, lowcut=1.0,
highcut=5.0, samp_rate=20, filt_order=4,
length=3, prepick=0.5, swin='all')
except URLError:
warnings.warn('Timed out connection to seishub')
if 'template' in locals():
self.assertEqual(len(template), 3)
import sys
import websocket
import threading
import traceback
import ssl
from time import sleep
import json
import decimal
import logging
from market_maker.settings import settings
from market_maker.auth.APIKeyAuth import generate_nonce, generate_signature
from market_maker.utils.log import setup_custom_logger
from market_maker.utils.math import toNearest
from future.utils import iteritems
from future.standard_library import hooks
with hooks(): # Python 2/3 compat
from urllib.parse import urlparse, urlunparse
# Connects to BitMEX websocket for streaming realtime data.
# The Marketmaker still interacts with this as if it were a REST Endpoint, but now it can get
# much more realtime data without heavily polling the API.
#
# The Websocket offers a bunch of data as raw properties right on the object.
# On connect, it synchronously asks for a push of all this data then returns.
# Right after, the MM can start using its data. It will be updated in realtime, so the MM can
# poll as often as it wants.
class BitMEXWebsocket():
# Don't grow a table larger than this amount. Helps cap memory usage.
MAX_TABLE_LEN = 200
def _create_report(ttrs, timetaken, log, server, hostname, sorted_tests,
ci_url=None, pr_url=None, import_failures=None):
# import additional libraries here to speed up normal tests
from future import standard_library
with standard_library.hooks():
import urllib.parse
import http.client
import codecs
from xml.etree import ElementTree
from xml.sax.saxutils import escape
if import_failures is None:
import_failures = {}
timestamp = int(time.time())
result = {'timestamp': timestamp}
result['slowest_tests'] = [("%0.3fs" % dt, "%s" % desc)
for (desc, dt) in sorted_tests[:20]]
result['timetaken'] = timetaken
if log:
try:
data = codecs.open(log, 'r', encoding='UTF-8').read()
result['install_log'] = escape(data)
except Exception:
print("Cannot open log file %s" % log)
# get ObsPy module versions
result['obspy'] = {}
tests = 0
errors = 0
failures = 0
skipped = 0
try:
installed = get_git_version()
except Exception:
installed = ''
result['obspy']['installed'] = installed
for module in sorted(ALL_MODULES):
result['obspy'][module] = {}
result['obspy'][module]['installed'] = installed
# add a failed-to-import test module to report with an error
if module in import_failures:
result['obspy'][module]['timetaken'] = 0
result['obspy'][module]['tested'] = True
result['obspy'][module]['tests'] = 1
# can't say how many tests would have been in that suite so just
# leave 0
result['obspy'][module]['skipped'] = 0
result['obspy'][module]['failures'] = {}
result['obspy'][module]['errors'] = {
'f%s' % (errors): import_failures[module]}
tests += 1
errors += 1
continue
if module not in ttrs:
continue
# test results
ttr = ttrs[module]
result['obspy'][module]['timetaken'] = ttr.__dict__['timetaken']
result['obspy'][module]['tested'] = True
result['obspy'][module]['tests'] = ttr.testsRun
skipped += len(ttr.skipped)
result['obspy'][module]['skipped'] = len(ttr.skipped)
tests += ttr.testsRun
# depending on module type either use failure (network related modules)
# or errors (all others)
result['obspy'][module]['errors'] = {}
result['obspy'][module]['failures'] = {}
if module in NETWORK_MODULES:
for _, text in ttr.errors:
result['obspy'][module]['failures']['f%s' % (failures)] = text
failures += 1
for _, text in ttr.failures:
result['obspy'][module]['failures']['f%s' % (failures)] = text
failures += 1
else:
for _, text in ttr.errors:
result['obspy'][module]['errors']['f%s' % (errors)] = text
errors += 1
for _, text in ttr.failures:
result['obspy'][module]['errors']['f%s' % (errors)] = text
errors += 1
# get dependencies
result['dependencies'] = {}
for module in DEPENDENCIES:
if module == "pep8-naming":
module_ = "pep8ext_naming"
else:
module_ = module
temp = module_.split('.')
try:
mod = __import__(module_,
fromlist=[native_str(temp[1:])])
except ImportError:
version_ = '---'
else:
try:
version_ = mod.__version__
except AttributeError:
version_ = '???'
result['dependencies'][module] = version_
# get system / environment settings
result['platform'] = {}
for func in ['system', 'release', 'version', 'machine',
'processor', 'python_version', 'python_implementation',
'python_compiler', 'architecture']:
try:
temp = getattr(platform, func)()
if isinstance(temp, tuple):
temp = temp[0]
result['platform'][func] = temp
except Exception:
result['platform'][func] = ''
# set node name to hostname if set
result['platform']['node'] = hostname
# post only the first part of the node name (only applies to MacOS X)
try:
result['platform']['node'] = result['platform']['node'].split('.')[0]
except Exception:
pass
# test results
result['tests'] = tests
result['errors'] = errors
result['failures'] = failures
result['skipped'] = skipped
# try to append info on skipped tests:
result['skipped_tests_details'] = []
try:
for module, testresult_ in ttrs.items():
if testresult_.skipped:
for skipped_test, skip_message in testresult_.skipped:
result['skipped_tests_details'].append(
(module, skipped_test.__module__,
skipped_test.__class__.__name__,
skipped_test._testMethodName, skip_message))
except Exception:
exc_type, exc_value, exc_tb = sys.exc_info()
print("\n".join(traceback.format_exception(exc_type, exc_value,
exc_tb)))
result['skipped_tests_details'] = []
if ci_url is not None:
result['ciurl'] = ci_url
if pr_url is not None:
result['prurl'] = pr_url
# generate XML document
def _dict2xml(doc, result):
for key, value in result.items():
key = key.split('(')[0].strip()
if isinstance(value, dict):
child = ElementTree.SubElement(doc, key)
_dict2xml(child, value)
elif value is not None:
if isinstance(value, (str, native_str)):
ElementTree.SubElement(doc, key).text = value
elif isinstance(value, (str, native_str)):
ElementTree.SubElement(doc, key).text = str(value, 'utf-8')
else:
ElementTree.SubElement(doc, key).text = str(value)
else:
ElementTree.SubElement(doc, key)
root = ElementTree.Element("report")
_dict2xml(root, result)
xml_doc = ElementTree.tostring(root)
print()
# send result to report server
params = urllib.parse.urlencode({
'timestamp': timestamp,
'system': result['platform']['system'],
'python_version': result['platform']['python_version'],
'architecture': result['platform']['architecture'],
'tests': tests,
'failures': failures,
'errors': errors,
'modules': len(ttrs) + len(import_failures),
'xml': xml_doc
})
headers = {"Content-type": "application/x-www-form-urlencoded",
"Accept": "text/plain"}
conn = http.client.HTTPConnection(server)
conn.request("POST", "/", params, headers)
# get the response
response = conn.getresponse()
# handle redirect
if response.status == 301:
o = urllib.parse.urlparse(response.msg['location'])
conn = http.client.HTTPConnection(o.netloc)
conn.request("POST", o.path, params, headers)
# get the response
response = conn.getresponse()
# handle errors
if response.status == 200:
print("Test report has been sent to %s. Thank you!" % (server))
else:
print("Error: Could not sent a test report to %s." % (server))
print(response.reason)
conn.close()
name='Booby',
owner={
'login': '******',
'name': 'Jaime Gil de Sagredo'
})
print booby.to_json()
'{"owner": {"login": "******", "name": "Jaime Gil de Sagredo"}, "name": "Booby"}'
"""
from __future__ import print_function
import tempfile
from future.standard_library import hooks
# Import urljoin for V2 and V3 - http://python-future.org/compatible_idioms.html
with hooks():
from urllib.parse import urljoin
__author__ = 'Danny Goodall'
import requests
import requests.exceptions
import requests_cache
from booby import Model, fields
from codefurther.errors import CodeFurtherConnectionError, CodeFurtherError, CodeFurtherHTTPError, CodeFurtherReadTimeoutError
class Change(Model):
"""The Change model that describes the change of this entry since last week's chart.
This class isn't made publicly visible, so it should never really need to be initialised manually. That said,
it is initialised by passing a series of keyword arguments, like so::
import sys
import websocket
import threading
import traceback
import ssl
from time import sleep
import json
import decimal
import logging
from market_maker.settings import settings
from market_maker.auth.APIKeyAuth import generate_expires, generate_signature
from market_maker.utils.log import setup_custom_logger
from market_maker.utils.math import toNearest
from future.utils import iteritems
from future.standard_library import hooks
with hooks(): # Python 2/3 compat
from urllib.parse import urlparse, urlunparse
# Connects to BitMEX websocket for streaming realtime data.
# The Marketmaker still interacts with this as if it were a REST Endpoint, but now it can get
# much more realtime data without heavily polling the API.
#
# The Websocket offers a bunch of data as raw properties right on the object.
# On connect, it synchronously asks for a push of all this data then returns.
# Right after, the MM can start using its data. It will be updated in realtime, so the MM can
# poll as often as it wants.
class BitMEXWebsocket():
# Don't grow a table larger than this amount. Helps cap memory usage.
MAX_TABLE_LEN = 200
def _createReport(ttrs, timetaken, log, server, hostname, sorted_tests):
# import additional libraries here to speed up normal tests
from future import standard_library
with standard_library.hooks():
import urllib.parse
import http.client
from xml.sax.saxutils import escape
import codecs
from xml.etree import ElementTree as etree
timestamp = int(time.time())
result = {'timestamp': timestamp}
result['slowest_tests'] = [("%0.3fs" % dt, "%s" % desc)
for (desc, dt) in sorted_tests[:20]]
result['timetaken'] = timetaken
if log:
try:
data = codecs.open(log, 'r', encoding='UTF-8').read()
result['install_log'] = escape(data)
except:
print(("Cannot open log file %s" % log))
# get ObsPy module versions
result['obspy'] = {}
tests = 0
errors = 0
failures = 0
skipped = 0
try:
installed = get_git_version()
except:
installed = ''
result['obspy']['installed'] = installed
for module in sorted(ALL_MODULES):
result['obspy'][module] = {}
if module not in ttrs:
continue
result['obspy'][module]['installed'] = installed
# test results
ttr = ttrs[module]
result['obspy'][module]['timetaken'] = ttr.__dict__['timetaken']
result['obspy'][module]['tested'] = True
result['obspy'][module]['tests'] = ttr.testsRun
# skipped is not supported for Python < 2.7
try:
skipped += len(ttr.skipped)
result['obspy'][module]['skipped'] = len(ttr.skipped)
except AttributeError:
skipped = ''
result['obspy'][module]['skipped'] = ''
tests += ttr.testsRun
# depending on module type either use failure (network related modules)
# or errors (all others)
result['obspy'][module]['errors'] = {}
result['obspy'][module]['failures'] = {}
if module in NETWORK_MODULES:
for _, text in ttr.errors:
result['obspy'][module]['failures']['f%s' % (failures)] = text
failures += 1
for _, text in ttr.failures:
result['obspy'][module]['failures']['f%s' % (failures)] = text
failures += 1
else:
for _, text in ttr.errors:
result['obspy'][module]['errors']['f%s' % (errors)] = text
errors += 1
for _, text in ttr.failures:
result['obspy'][module]['errors']['f%s' % (errors)] = text
errors += 1
# get dependencies
result['dependencies'] = {}
for module in DEPENDENCIES:
temp = module.split('.')
try:
mod = __import__(module, fromlist=[native_str(temp[1:])])
if module == '_omnipy':
result['dependencies'][module] = mod.coreVersion()
else:
result['dependencies'][module] = mod.__version__
except ImportError:
result['dependencies'][module] = ''
# get system / environment settings
result['platform'] = {}
for func in [
'system', 'release', 'version', 'machine', 'processor',
'python_version', 'python_implementation', 'python_compiler',
'architecture'
]:
try:
temp = getattr(platform, func)()
if isinstance(temp, tuple):
temp = temp[0]
result['platform'][func] = temp
except:
result['platform'][func] = ''
# set node name to hostname if set
result['platform']['node'] = hostname
# post only the first part of the node name (only applies to MacOS X)
try:
result['platform']['node'] = result['platform']['node'].split('.')[0]
except:
pass
# test results
result['tests'] = tests
result['errors'] = errors
result['failures'] = failures
result['skipped'] = skipped
# generate XML document
def _dict2xml(doc, result):
for key, value in result.items():
key = key.split('(')[0].strip()
if isinstance(value, dict):
child = etree.SubElement(doc, key)
_dict2xml(child, value)
elif value is not None:
if isinstance(value, (str, native_str)):
etree.SubElement(doc, key).text = value
elif isinstance(value, (str, native_str)):
etree.SubElement(doc, key).text = str(value, 'utf-8')
else:
etree.SubElement(doc, key).text = str(value)
else:
etree.SubElement(doc, key)
root = etree.Element("report")
_dict2xml(root, result)
xml_doc = etree.tostring(root)
print()
# send result to report server
params = urllib.parse.urlencode({
'timestamp':
timestamp,
'system':
result['platform']['system'],
'python_version':
result['platform']['python_version'],
'architecture':
result['platform']['architecture'],
'tests':
tests,
'failures':
failures,
'errors':
errors,
'modules':
len(ttrs),
'xml':
xml_doc
})
headers = {
"Content-type": "application/x-www-form-urlencoded",
"Accept": "text/plain"
}
conn = http.client.HTTPConnection(server)
conn.request("POST", "/", params, headers)
# get the response
response = conn.getresponse()
# handle redirect
if response.status == 301:
o = urllib.parse.urlparse(response.msg['location'])
conn = http.client.HTTPConnection(o.netloc)
conn.request("POST", o.path, params, headers)
# get the response
response = conn.getresponse()
# handle errors
if response.status == 200:
print(("Test report has been sent to %s. Thank you!" % (server)))
else:
print(("Error: Could not sent a test report to %s." % (server)))
print((response.reason))
def _create_report(ttrs, timetaken, log, server, hostname, sorted_tests, ci_url=None, pr_url=None):
# import additional libraries here to speed up normal tests
from future import standard_library
with standard_library.hooks():
import urllib.parse
import http.client
import codecs
from xml.etree import ElementTree
from xml.sax.saxutils import escape
timestamp = int(time.time())
result = {"timestamp": timestamp}
result["slowest_tests"] = [("%0.3fs" % dt, "%s" % desc) for (desc, dt) in sorted_tests[:20]]
result["timetaken"] = timetaken
if log:
try:
data = codecs.open(log, "r", encoding="UTF-8").read()
result["install_log"] = escape(data)
except:
print("Cannot open log file %s" % log)
# get ObsPy module versions
result["obspy"] = {}
tests = 0
errors = 0
failures = 0
skipped = 0
try:
installed = get_git_version()
except:
installed = ""
result["obspy"]["installed"] = installed
for module in sorted(ALL_MODULES):
result["obspy"][module] = {}
if module not in ttrs:
continue
result["obspy"][module]["installed"] = installed
# test results
ttr = ttrs[module]
result["obspy"][module]["timetaken"] = ttr.__dict__["timetaken"]
result["obspy"][module]["tested"] = True
result["obspy"][module]["tests"] = ttr.testsRun
# skipped is not supported for Python < 2.7
try:
skipped += len(ttr.skipped)
result["obspy"][module]["skipped"] = len(ttr.skipped)
except AttributeError:
skipped = ""
result["obspy"][module]["skipped"] = ""
tests += ttr.testsRun
# depending on module type either use failure (network related modules)
# or errors (all others)
result["obspy"][module]["errors"] = {}
result["obspy"][module]["failures"] = {}
if module in NETWORK_MODULES:
for _, text in ttr.errors:
result["obspy"][module]["failures"]["f%s" % (failures)] = text
failures += 1
for _, text in ttr.failures:
result["obspy"][module]["failures"]["f%s" % (failures)] = text
failures += 1
else:
for _, text in ttr.errors:
result["obspy"][module]["errors"]["f%s" % (errors)] = text
errors += 1
for _, text in ttr.failures:
result["obspy"][module]["errors"]["f%s" % (errors)] = text
errors += 1
# get dependencies
result["dependencies"] = {}
for module in DEPENDENCIES:
if module == "pep8-naming":
module_ = "pep8ext_naming"
else:
module_ = module
temp = module_.split(".")
try:
mod = __import__(module_, fromlist=[native_str(temp[1:])])
except ImportError:
version_ = "---"
else:
try:
version_ = mod.__version__
except AttributeError:
version_ = "???"
result["dependencies"][module] = version_
# get system / environment settings
result["platform"] = {}
for func in [
"system",
"release",
"version",
"machine",
"processor",
"python_version",
"python_implementation",
"python_compiler",
"architecture",
]:
try:
temp = getattr(platform, func)()
if isinstance(temp, tuple):
temp = temp[0]
result["platform"][func] = temp
except:
result["platform"][func] = ""
# set node name to hostname if set
result["platform"]["node"] = hostname
# post only the first part of the node name (only applies to MacOS X)
try:
result["platform"]["node"] = result["platform"]["node"].split(".")[0]
except:
pass
# test results
result["tests"] = tests
result["errors"] = errors
result["failures"] = failures
result["skipped"] = skipped
if ci_url is not None:
result["ciurl"] = ci_url
if pr_url is not None:
result["prurl"] = pr_url
# generate XML document
def _dict2xml(doc, result):
for key, value in result.items():
key = key.split("(")[0].strip()
if isinstance(value, dict):
child = ElementTree.SubElement(doc, key)
_dict2xml(child, value)
elif value is not None:
if isinstance(value, (str, native_str)):
ElementTree.SubElement(doc, key).text = value
elif isinstance(value, (str, native_str)):
ElementTree.SubElement(doc, key).text = str(value, "utf-8")
else:
ElementTree.SubElement(doc, key).text = str(value)
else:
ElementTree.SubElement(doc, key)
root = ElementTree.Element("report")
_dict2xml(root, result)
xml_doc = ElementTree.tostring(root)
print()
# send result to report server
params = urllib.parse.urlencode(
{
"timestamp": timestamp,
"system": result["platform"]["system"],
"python_version": result["platform"]["python_version"],
"architecture": result["platform"]["architecture"],
"tests": tests,
"failures": failures,
"errors": errors,
"modules": len(ttrs),
"xml": xml_doc,
}
)
headers = {"Content-type": "application/x-www-form-urlencoded", "Accept": "text/plain"}
conn = http.client.HTTPConnection(server)
conn.request("POST", "/", params, headers)
# get the response
response = conn.getresponse()
# handle redirect
if response.status == 301:
o = urllib.parse.urlparse(response.msg["location"])
conn = http.client.HTTPConnection(o.netloc)
conn.request("POST", o.path, params, headers)
# get the response
response = conn.getresponse()
# handle errors
if response.status == 200:
print("Test report has been sent to %s. Thank you!" % (server))
else:
print("Error: Could not sent a test report to %s." % (server))
print(response.reason)
def _create_report(ttrs, timetaken, log, server, hostname, sorted_tests):
# import additional libraries here to speed up normal tests
from future import standard_library
with standard_library.hooks():
import urllib.parse
import http.client
import codecs
from xml.etree import ElementTree
from xml.sax.saxutils import escape
timestamp = int(time.time())
result = {'timestamp': timestamp}
result['slowest_tests'] = [("%0.3fs" % dt, "%s" % desc)
for (desc, dt) in sorted_tests[:20]]
result['timetaken'] = timetaken
if log:
try:
data = codecs.open(log, 'r', encoding='UTF-8').read()
result['install_log'] = escape(data)
except:
print("Cannot open log file %s" % log)
# get ObsPy module versions
result['obspy'] = {}
tests = 0
errors = 0
failures = 0
skipped = 0
try:
installed = get_git_version()
except:
installed = ''
result['obspy']['installed'] = installed
for module in sorted(ALL_MODULES):
result['obspy'][module] = {}
if module not in ttrs:
continue
result['obspy'][module]['installed'] = installed
# test results
ttr = ttrs[module]
result['obspy'][module]['timetaken'] = ttr.__dict__['timetaken']
result['obspy'][module]['tested'] = True
result['obspy'][module]['tests'] = ttr.testsRun
# skipped is not supported for Python < 2.7
try:
skipped += len(ttr.skipped)
result['obspy'][module]['skipped'] = len(ttr.skipped)
except AttributeError:
skipped = ''
result['obspy'][module]['skipped'] = ''
tests += ttr.testsRun
# depending on module type either use failure (network related modules)
# or errors (all others)
result['obspy'][module]['errors'] = {}
result['obspy'][module]['failures'] = {}
if module in NETWORK_MODULES:
for _, text in ttr.errors:
result['obspy'][module]['failures']['f%s' % (failures)] = text
failures += 1
for _, text in ttr.failures:
result['obspy'][module]['failures']['f%s' % (failures)] = text
failures += 1
else:
for _, text in ttr.errors:
result['obspy'][module]['errors']['f%s' % (errors)] = text
errors += 1
for _, text in ttr.failures:
result['obspy'][module]['errors']['f%s' % (errors)] = text
errors += 1
# get dependencies
result['dependencies'] = {}
for module in DEPENDENCIES:
temp = module.split('.')
try:
mod = __import__(module,
fromlist=[native_str(temp[1:])])
if module == '_omnipy':
result['dependencies'][module] = mod.coreVersion()
else:
result['dependencies'][module] = mod.__version__
except ImportError:
result['dependencies'][module] = ''
# get system / environment settings
result['platform'] = {}
for func in ['system', 'release', 'version', 'machine',
'processor', 'python_version', 'python_implementation',
'python_compiler', 'architecture']:
try:
temp = getattr(platform, func)()
if isinstance(temp, tuple):
temp = temp[0]
result['platform'][func] = temp
except:
result['platform'][func] = ''
# set node name to hostname if set
result['platform']['node'] = hostname
# post only the first part of the node name (only applies to MacOS X)
try:
result['platform']['node'] = result['platform']['node'].split('.')[0]
except:
pass
# test results
result['tests'] = tests
result['errors'] = errors
result['failures'] = failures
result['skipped'] = skipped
# generate XML document
def _dict2xml(doc, result):
for key, value in result.items():
key = key.split('(')[0].strip()
if isinstance(value, dict):
child = ElementTree.SubElement(doc, key)
_dict2xml(child, value)
elif value is not None:
if isinstance(value, (str, native_str)):
ElementTree.SubElement(doc, key).text = value
elif isinstance(value, (str, native_str)):
ElementTree.SubElement(doc, key).text = str(value, 'utf-8')
else:
ElementTree.SubElement(doc, key).text = str(value)
else:
ElementTree.SubElement(doc, key)
root = ElementTree.Element("report")
_dict2xml(root, result)
xml_doc = ElementTree.tostring(root)
print()
# send result to report server
params = urllib.parse.urlencode({
'timestamp': timestamp,
'system': result['platform']['system'],
'python_version': result['platform']['python_version'],
'architecture': result['platform']['architecture'],
'tests': tests,
'failures': failures,
'errors': errors,
'modules': len(ttrs),
'xml': xml_doc
})
headers = {"Content-type": "application/x-www-form-urlencoded",
"Accept": "text/plain"}
conn = http.client.HTTPConnection(server)
conn.request("POST", "/", params, headers)
# get the response
response = conn.getresponse()
# handle redirect
if response.status == 301:
o = urllib.parse.urlparse(response.msg['location'])
conn = http.client.HTTPConnection(o.netloc)
conn.request("POST", o.path, params, headers)
# get the response
response = conn.getresponse()
# handle errors
if response.status == 200:
print("Test report has been sent to %s. Thank you!" % (server))
else:
print("Error: Could not sent a test report to %s." % (server))
print(response.reason)
def _create_report(ttrs,
timetaken,
log,
server,
hostname,
sorted_tests,
ci_url=None,
pr_url=None,
import_failures=None):
# import additional libraries here to speed up normal tests
from future import standard_library
with standard_library.hooks():
import urllib.parse
import http.client
import codecs
from xml.etree import ElementTree
from xml.sax.saxutils import escape
if import_failures is None:
import_failures = {}
timestamp = int(time.time())
result = {'timestamp': timestamp}
result['slowest_tests'] = [("%0.3fs" % dt, "%s" % desc)
for (desc, dt) in sorted_tests[:20]]
result['timetaken'] = timetaken
if log:
try:
data = codecs.open(log, 'r', encoding='UTF-8').read()
result['install_log'] = escape(data)
except Exception:
print("Cannot open log file %s" % log)
# get ObsPy module versions
result['obspy'] = {}
tests = 0
errors = 0
failures = 0
skipped = 0
try:
installed = get_git_version()
except Exception:
installed = ''
result['obspy']['installed'] = installed
for module in sorted(ALL_MODULES):
result['obspy'][module] = {}
result['obspy'][module]['installed'] = installed
# add a failed-to-import test module to report with an error
if module in import_failures:
result['obspy'][module]['timetaken'] = 0
result['obspy'][module]['tested'] = True
result['obspy'][module]['tests'] = 1
# can't say how many tests would have been in that suite so just
# leave 0
result['obspy'][module]['skipped'] = 0
result['obspy'][module]['failures'] = {}
result['obspy'][module]['errors'] = {
'f%s' % (errors): import_failures[module]
}
tests += 1
errors += 1
continue
if module not in ttrs:
continue
# test results
ttr = ttrs[module]
result['obspy'][module]['timetaken'] = ttr.__dict__['timetaken']
result['obspy'][module]['tested'] = True
result['obspy'][module]['tests'] = ttr.testsRun
skipped += len(ttr.skipped)
result['obspy'][module]['skipped'] = len(ttr.skipped)
tests += ttr.testsRun
# depending on module type either use failure (network related modules)
# or errors (all others)
result['obspy'][module]['errors'] = {}
result['obspy'][module]['failures'] = {}
if module in NETWORK_MODULES:
for _, text in ttr.errors:
result['obspy'][module]['failures']['f%s' % (failures)] = text
failures += 1
for _, text in ttr.failures:
result['obspy'][module]['failures']['f%s' % (failures)] = text
failures += 1
else:
for _, text in ttr.errors:
result['obspy'][module]['errors']['f%s' % (errors)] = text
errors += 1
for _, text in ttr.failures:
result['obspy'][module]['errors']['f%s' % (errors)] = text
errors += 1
# get dependencies
result['dependencies'] = {}
for module in DEPENDENCIES:
if module == "pep8-naming":
module_ = "pep8ext_naming"
else:
module_ = module
temp = module_.split('.')
try:
mod = __import__(module_, fromlist=[native_str(temp[1:])])
except ImportError:
version_ = '---'
else:
try:
version_ = mod.__version__
except AttributeError:
version_ = '???'
result['dependencies'][module] = version_
# get system / environment settings
result['platform'] = {}
for func in [
'system', 'release', 'version', 'machine', 'processor',
'python_version', 'python_implementation', 'python_compiler',
'architecture'
]:
try:
temp = getattr(platform, func)()
if isinstance(temp, tuple):
temp = temp[0]
result['platform'][func] = temp
except Exception:
result['platform'][func] = ''
# set node name to hostname if set
result['platform']['node'] = hostname
# post only the first part of the node name (only applies to MacOS X)
try:
result['platform']['node'] = result['platform']['node'].split('.')[0]
except Exception:
pass
# test results
result['tests'] = tests
result['errors'] = errors
result['failures'] = failures
result['skipped'] = skipped
# try to append info on skipped tests:
result['skipped_tests_details'] = []
try:
for module, testresult_ in ttrs.items():
if testresult_.skipped:
for skipped_test, skip_message in testresult_.skipped:
result['skipped_tests_details'].append(
(module, skipped_test.__module__,
skipped_test.__class__.__name__,
skipped_test._testMethodName, skip_message))
except Exception:
exc_type, exc_value, exc_tb = sys.exc_info()
print("\n".join(traceback.format_exception(exc_type, exc_value,
exc_tb)))
result['skipped_tests_details'] = []
if ci_url is not None:
result['ciurl'] = ci_url
if pr_url is not None:
result['prurl'] = pr_url
# generate XML document
def _dict2xml(doc, result):
for key, value in result.items():
key = key.split('(')[0].strip()
if isinstance(value, dict):
child = ElementTree.SubElement(doc, key)
_dict2xml(child, value)
elif value is not None:
if isinstance(value, (str, native_str)):
ElementTree.SubElement(doc, key).text = value
elif isinstance(value, (str, native_str)):
ElementTree.SubElement(doc, key).text = str(value, 'utf-8')
else:
ElementTree.SubElement(doc, key).text = str(value)
else:
ElementTree.SubElement(doc, key)
root = ElementTree.Element("report")
_dict2xml(root, result)
xml_doc = ElementTree.tostring(root)
print()
# send result to report server
params = urllib.parse.urlencode({
'timestamp':
timestamp,
'system':
result['platform']['system'],
'python_version':
result['platform']['python_version'],
'architecture':
result['platform']['architecture'],
'tests':
tests,
'failures':
failures,
'errors':
errors,
'modules':
len(ttrs) + len(import_failures),
'xml':
xml_doc
})
headers = {
"Content-type": "application/x-www-form-urlencoded",
"Accept": "text/plain"
}
conn = http.client.HTTPConnection(server)
conn.request("POST", "/", params, headers)
# get the response
response = conn.getresponse()
# handle redirect
if response.status == 301:
o = urllib.parse.urlparse(response.msg['location'])
conn = http.client.HTTPConnection(o.netloc)
conn.request("POST", o.path, params, headers)
# get the response
response = conn.getresponse()
# handle errors
if response.status == 200:
print("Test report has been sent to %s. Thank you!" % (server))
else:
print("Error: Could not sent a test report to %s." % (server))
print(response.reason)
conn.close()
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
The obspy.clients.seishub.client test suite.
"""
from __future__ import (absolute_import, division, print_function,
unicode_literals)
from future.builtins import * # NOQA
from future import standard_library
import unittest
with standard_library.hooks():
import urllib.request
from obspy.core import AttribDict, UTCDateTime
from obspy.clients.seishub import Client
TESTSERVER = "http://teide.geophysik.uni-muenchen.de:8080"
TESTSERVER_UNREACHABLE_MSG = "Seishub test server not reachable."
def _check_server_availability():
"""
Returns an empty string if server is reachable or failure message
otherwise.
"""
try:
code = urllib.request.urlopen(TESTSERVER, timeout=3).getcode()
assert(code == 200)
from os.path import abspath, dirname, join, exists, basename, splitext
from functools import partial
from os import mkdir
import gzip
from glob import glob
from future import standard_library
from future.utils import viewitems
from qiita_core.exceptions import QiitaEnvironmentError
from qiita_core.qiita_settings import qiita_config
from .sql_connection import SQLConnectionHandler
from .reference import Reference
from natsort import natsorted
with standard_library.hooks():
from urllib.request import urlretrieve
get_support_file = partial(join, join(dirname(abspath(__file__)),
'support_files'))
reference_base_dir = join(qiita_config.base_data_dir, "reference")
get_reference_fp = partial(join, reference_base_dir)
SETTINGS_FP = get_support_file('qiita-db-settings.sql')
LAYOUT_FP = get_support_file('qiita-db-unpatched.sql')
POPULATE_FP = get_support_file('populate_test_db.sql')
PATCHES_DIR = get_support_file('patches')
import datetime
import collections
from future.standard_library import hooks
from pycronofy import settings
from pycronofy.auth import Auth
from pycronofy.batch import BatchEntry
from pycronofy.batch import BatchResponse
from pycronofy.datetime_utils import format_event_time
from pycronofy.exceptions import PyCronofyPartialSuccessError
from pycronofy.pagination import Pages
from pycronofy.request_handler import RequestHandler
from pycronofy.validation import validate
with hooks():
from urllib.parse import urlencode
class Client(object):
"""Client for cronofy web service.
Performs authentication, and wraps API: https://docs.cronofy.com/developers/api/authorization/
"""
def __init__(self, client_id=None, client_secret=None, access_token=None, refresh_token=None, token_expiration=None, data_center=None):
"""
Example Usage:
pycronofy.Client(access_token='')
pycronofy.Client(client_id='', client_secret='')
:param string client_id: OAuth Client ID. (Optional, default None)