def test_urllib_imports_cm(self):
     with standard_library.hooks():
         import urllib
         import urllib.parse
         import urllib.request
         import urllib.robotparser
         import urllib.error
         import urllib.response
     self.assertTrue(True)
 def test_urllib_imports_cm(self):
     with standard_library.hooks():
         import urllib
         import urllib.parse
         import urllib.request
         import urllib.robotparser
         import urllib.error
         import urllib.response
     self.assertTrue(True)
Exemplo n.º 3
0
def download_setuptools(version=DEFAULT_VERSION,
                        download_base=DEFAULT_URL,
                        to_dir=os.curdir,
                        delay=15):
    """Download setuptools from a specified location and return its filename

    `version` should be a valid setuptools version number that is available
    as an egg for download under the `download_base` URL (which should end
    with a '/'). `to_dir` is the directory where the egg will be downloaded.
    `delay` is the number of seconds to pause before an actual download attempt.
    """
    #import urllib.request, urllib.error, urllib.parse, shutil
    from future.standard_library import hooks
    with hooks():
        from urllib.parse import urlparse, urlencode
        from urllib.request import urlopen, Request
        from urllib.error import HTTPError
    import shutil
    egg_name = "setuptools-%s-py%s.egg" % (version, sys.version[:3])
    url = download_base + egg_name
    saveto = os.path.join(to_dir, egg_name)
    src = dst = None
    if not os.path.exists(saveto):  # Avoid repeated downloads
        try:
            from distutils import log
            if delay:
                log.warn(
                    """
---------------------------------------------------------------------------
This script requires setuptools version %s to run (even to display
help).  I will attempt to download it for you (from
%s), but
you may need to enable firewall access for this script first.
I will start the download in %d seconds.

(Note: if this machine does not have network access, please obtain the file

   %s

and place it in this directory before rerunning this script.)
---------------------------------------------------------------------------""",
                    version, download_base, delay, url)
                from time import sleep
                sleep(delay)
            log.warn("Downloading %s", url)
            src = urlopen(url)
            # Read/write all in one block, so we don't create a corrupt file
            # if the download is interrupted.
            data = _validate_md5(egg_name, src.read())
            dst = open(saveto, "wb")
            dst.write(data)
        finally:
            if src: src.close()
            if dst: dst.close()
    return os.path.realpath(saveto)
Exemplo n.º 4
0
 def test_urllib(self):
     """
     Tests that urllib isn't changed from under our feet. (This might not
     even be a problem?)
     """
     from future import standard_library
     import urllib
     orig_file = urllib.__file__
     with standard_library.hooks():
         import urllib.response
     self.assertEqual(orig_file, urllib.__file__)
Exemplo n.º 5
0
 def test_urllib(self):
     """
     Tests that urllib isn't changed from under our feet. (This might not
     even be a problem?)
     """
     from future import standard_library
     import urllib
     orig_file = urllib.__file__
     with standard_library.hooks():
         import urllib.response
     self.assertEqual(orig_file, urllib.__file__)
Exemplo n.º 6
0
 def test_urllib(self):
     """
     This should perhaps fail: importing urllib first means that the import hooks
     won't be consulted when importing urllib.response.
     """
     import urllib
     print(urllib.__file__)
     from future import standard_library
     with standard_library.hooks():
         import urllib.response
     print(urllib.__file__)
     print(urllib.response.__file__)
Exemplo n.º 7
0
def download_setuptools(
    version=DEFAULT_VERSION, download_base=DEFAULT_URL, to_dir=os.curdir,
    delay = 15
):
    """Download setuptools from a specified location and return its filename

    `version` should be a valid setuptools version number that is available
    as an egg for download under the `download_base` URL (which should end
    with a '/'). `to_dir` is the directory where the egg will be downloaded.
    `delay` is the number of seconds to pause before an actual download attempt.
    """
    #import urllib.request, urllib.error, urllib.parse, shutil
    from future.standard_library import hooks
    with hooks():
        from urllib.parse import urlparse, urlencode
        from urllib.request import urlopen, Request
        from urllib.error import HTTPError
    import shutil
    egg_name = "setuptools-%s-py%s.egg" % (version,sys.version[:3])
    url = download_base + egg_name
    saveto = os.path.join(to_dir, egg_name)
    src = dst = None
    if not os.path.exists(saveto):  # Avoid repeated downloads
        try:
            from distutils import log
            if delay:
                log.warn("""
---------------------------------------------------------------------------
This script requires setuptools version %s to run (even to display
help).  I will attempt to download it for you (from
%s), but
you may need to enable firewall access for this script first.
I will start the download in %d seconds.

(Note: if this machine does not have network access, please obtain the file

   %s

and place it in this directory before rerunning this script.)
---------------------------------------------------------------------------""",
                    version, download_base, delay, url
                ); from time import sleep; sleep(delay)
            log.warn("Downloading %s", url)
            src = urlopen(url)
            # Read/write all in one block, so we don't create a corrupt file
            # if the download is interrupted.
            data = _validate_md5(egg_name, src.read())
            dst = open(saveto,"wb"); dst.write(data)
        finally:
            if src: src.close()
            if dst: dst.close()
    return os.path.realpath(saveto)
Exemplo n.º 8
0
    def test_can_import_several(self):
        """
        This test fails if e.g. future/standard_library/email/header.py contains:
        
            from future import standard_library
            standard_library.remove_hooks(keep_sys_modules=True)
        """

        with standard_library.hooks():
            import urllib.parse
            import urllib.request
            import http.server
            from test import support
        for m in [urllib.parse, urllib.request, http.server, support]:
            self.assertTrue(m is not None)
    def test_can_import_several(self):
        """
        This test failed in v0.12-pre if e.g.
        future/standard_library/email/header.py contained:
        
            from future import standard_library
            standard_library.remove_hooks()
        """

        import future.moves.urllib.parse as urllib_parse
        import future.moves.urllib.request as urllib_request

        with standard_library.hooks():
            import http.server
        for m in [urllib_parse, urllib_request, http.server]:
            self.assertTrue(m is not None)
    def test_can_import_several(self):
        """
        This test failed in v0.12-pre if e.g.
        future/standard_library/email/header.py contained:
        
            from future import standard_library
            standard_library.remove_hooks()
        """

        import future.moves.urllib.parse as urllib_parse
        import future.moves.urllib.request as urllib_request

        with standard_library.hooks():
            import http.server
        for m in [urllib_parse, urllib_request, http.server]:
            self.assertTrue(m is not None)
Exemplo n.º 11
0
    def test_seishub(self):
        """Test the seishub method, use obspy default seishub client."""
        from obspy.core.event import Catalog, Event, Origin, Pick
        from obspy.core.event import WaveformStreamID
        from obspy import UTCDateTime
        import warnings
        from future import standard_library
        with standard_library.hooks():
            from urllib.request import URLError


        t = UTCDateTime(2009, 9, 3)
        test_cat = Catalog()
        test_cat.append(Event())
        test_cat[0].origins.append(Origin())
        test_cat[0].origins[0].time = t
        test_cat[0].origins[0].latitude = 45
        test_cat[0].origins[0].longitude = 45
        test_cat[0].origins[0].depth = 5000
        test_cat[0].\
            picks.append(Pick(waveform_id=WaveformStreamID(station_code='MANZ',
                                                           channel_code='EHZ',
                                                            network_code='BW'),
                              phase_hint='PG', time=t + 2000))
        test_cat[0].\
            picks.append(Pick(waveform_id=WaveformStreamID(station_code='MANZ',
                                                           channel_code='EHN',
                                                           network_code='BW'),
                              phase_hint='SG', time=t + 2005))
        test_cat[0].\
            picks.append(Pick(waveform_id=WaveformStreamID(station_code='MANZ',
                                                           channel_code='EHE',
                                                           network_code='BW'),
                              phase_hint='SG', time=t + 2005.5))

        test_url = 'http://teide.geophysik.uni-muenchen.de:8080'

        try:
            template = from_seishub(test_cat, url=test_url, lowcut=1.0,
                                    highcut=5.0, samp_rate=20, filt_order=4,
                                    length=3, prepick=0.5, swin='all',
                                    process_len=300)
        except URLError:
            warnings.warn('Timed out connection to seishub')
        if 'template' in locals():
            self.assertEqual(len(template), 3)
Exemplo n.º 12
0
    def test_seishub(self):
        """Test the seishub method, use obspy default seishub client."""
        from obspy.core.event import Catalog, Event, Origin, Pick
        from obspy.core.event import WaveformStreamID
        from obspy import UTCDateTime
        import warnings
        from future import standard_library
        with standard_library.hooks():
            from urllib.request import URLError


        t = UTCDateTime(2009, 9, 3)
        test_cat = Catalog()
        test_cat.append(Event())
        test_cat[0].origins.append(Origin())
        test_cat[0].origins[0].time = t
        test_cat[0].origins[0].latitude = 45
        test_cat[0].origins[0].longitude = 45
        test_cat[0].origins[0].depth = 5000
        test_cat[0].\
            picks.append(Pick(waveform_id=WaveformStreamID(station_code='MANZ',
                                                           channel_code='EHZ',
                                                            network_code='BW'),
                              phase_hint='PG', time=t + 2000))
        test_cat[0].\
            picks.append(Pick(waveform_id=WaveformStreamID(station_code='MANZ',
                                                           channel_code='EHN',
                                                           network_code='BW'),
                              phase_hint='SG', time=t + 2005))
        test_cat[0].\
            picks.append(Pick(waveform_id=WaveformStreamID(station_code='MANZ',
                                                           channel_code='EHE',
                                                           network_code='BW'),
                              phase_hint='SG', time=t + 2005.5))

        test_url = 'http://teide.geophysik.uni-muenchen.de:8080'

        try:
            template = from_seishub(test_cat, url=test_url, lowcut=1.0,
                                    highcut=5.0, samp_rate=20, filt_order=4,
                                    length=3, prepick=0.5, swin='all')
        except URLError:
            warnings.warn('Timed out connection to seishub')
        if 'template' in locals():
            self.assertEqual(len(template), 3)
Exemplo n.º 13
0
import sys
import websocket
import threading
import traceback
import ssl
from time import sleep
import json
import decimal
import logging
from market_maker.settings import settings
from market_maker.auth.APIKeyAuth import generate_nonce, generate_signature
from market_maker.utils.log import setup_custom_logger
from market_maker.utils.math import toNearest
from future.utils import iteritems
from future.standard_library import hooks
with hooks():  # Python 2/3 compat
    from urllib.parse import urlparse, urlunparse


# Connects to BitMEX websocket for streaming realtime data.
# The Marketmaker still interacts with this as if it were a REST Endpoint, but now it can get
# much more realtime data without heavily polling the API.
#
# The Websocket offers a bunch of data as raw properties right on the object.
# On connect, it synchronously asks for a push of all this data then returns.
# Right after, the MM can start using its data. It will be updated in realtime, so the MM can
# poll as often as it wants.
class BitMEXWebsocket():

    # Don't grow a table larger than this amount. Helps cap memory usage.
    MAX_TABLE_LEN = 200
Exemplo n.º 14
0
def _create_report(ttrs, timetaken, log, server, hostname, sorted_tests,
                   ci_url=None, pr_url=None, import_failures=None):
    # import additional libraries here to speed up normal tests
    from future import standard_library
    with standard_library.hooks():
        import urllib.parse
        import http.client
    import codecs
    from xml.etree import ElementTree
    from xml.sax.saxutils import escape
    if import_failures is None:
        import_failures = {}
    timestamp = int(time.time())
    result = {'timestamp': timestamp}
    result['slowest_tests'] = [("%0.3fs" % dt, "%s" % desc)
                               for (desc, dt) in sorted_tests[:20]]
    result['timetaken'] = timetaken
    if log:
        try:
            data = codecs.open(log, 'r', encoding='UTF-8').read()
            result['install_log'] = escape(data)
        except Exception:
            print("Cannot open log file %s" % log)
    # get ObsPy module versions
    result['obspy'] = {}
    tests = 0
    errors = 0
    failures = 0
    skipped = 0
    try:
        installed = get_git_version()
    except Exception:
        installed = ''
    result['obspy']['installed'] = installed
    for module in sorted(ALL_MODULES):
        result['obspy'][module] = {}
        result['obspy'][module]['installed'] = installed
        # add a failed-to-import test module to report with an error
        if module in import_failures:
            result['obspy'][module]['timetaken'] = 0
            result['obspy'][module]['tested'] = True
            result['obspy'][module]['tests'] = 1
            # can't say how many tests would have been in that suite so just
            # leave 0
            result['obspy'][module]['skipped'] = 0
            result['obspy'][module]['failures'] = {}
            result['obspy'][module]['errors'] = {
                'f%s' % (errors): import_failures[module]}
            tests += 1
            errors += 1
            continue
        if module not in ttrs:
            continue
        # test results
        ttr = ttrs[module]
        result['obspy'][module]['timetaken'] = ttr.__dict__['timetaken']
        result['obspy'][module]['tested'] = True
        result['obspy'][module]['tests'] = ttr.testsRun
        skipped += len(ttr.skipped)
        result['obspy'][module]['skipped'] = len(ttr.skipped)
        tests += ttr.testsRun
        # depending on module type either use failure (network related modules)
        # or errors (all others)
        result['obspy'][module]['errors'] = {}
        result['obspy'][module]['failures'] = {}
        if module in NETWORK_MODULES:
            for _, text in ttr.errors:
                result['obspy'][module]['failures']['f%s' % (failures)] = text
                failures += 1
            for _, text in ttr.failures:
                result['obspy'][module]['failures']['f%s' % (failures)] = text
                failures += 1
        else:
            for _, text in ttr.errors:
                result['obspy'][module]['errors']['f%s' % (errors)] = text
                errors += 1
            for _, text in ttr.failures:
                result['obspy'][module]['errors']['f%s' % (errors)] = text
                errors += 1
    # get dependencies
    result['dependencies'] = {}
    for module in DEPENDENCIES:
        if module == "pep8-naming":
            module_ = "pep8ext_naming"
        else:
            module_ = module
        temp = module_.split('.')
        try:
            mod = __import__(module_,
                             fromlist=[native_str(temp[1:])])
        except ImportError:
            version_ = '---'
        else:
            try:
                version_ = mod.__version__
            except AttributeError:
                version_ = '???'
        result['dependencies'][module] = version_
    # get system / environment settings
    result['platform'] = {}
    for func in ['system', 'release', 'version', 'machine',
                 'processor', 'python_version', 'python_implementation',
                 'python_compiler', 'architecture']:
        try:
            temp = getattr(platform, func)()
            if isinstance(temp, tuple):
                temp = temp[0]
            result['platform'][func] = temp
        except Exception:
            result['platform'][func] = ''
    # set node name to hostname if set
    result['platform']['node'] = hostname
    # post only the first part of the node name (only applies to MacOS X)
    try:
        result['platform']['node'] = result['platform']['node'].split('.')[0]
    except Exception:
        pass
    # test results
    result['tests'] = tests
    result['errors'] = errors
    result['failures'] = failures
    result['skipped'] = skipped
    # try to append info on skipped tests:
    result['skipped_tests_details'] = []
    try:
        for module, testresult_ in ttrs.items():
            if testresult_.skipped:
                for skipped_test, skip_message in testresult_.skipped:
                    result['skipped_tests_details'].append(
                        (module, skipped_test.__module__,
                         skipped_test.__class__.__name__,
                         skipped_test._testMethodName, skip_message))
    except Exception:
        exc_type, exc_value, exc_tb = sys.exc_info()
        print("\n".join(traceback.format_exception(exc_type, exc_value,
                                                   exc_tb)))
        result['skipped_tests_details'] = []

    if ci_url is not None:
        result['ciurl'] = ci_url
    if pr_url is not None:
        result['prurl'] = pr_url

    # generate XML document
    def _dict2xml(doc, result):
        for key, value in result.items():
            key = key.split('(')[0].strip()
            if isinstance(value, dict):
                child = ElementTree.SubElement(doc, key)
                _dict2xml(child, value)
            elif value is not None:
                if isinstance(value, (str, native_str)):
                    ElementTree.SubElement(doc, key).text = value
                elif isinstance(value, (str, native_str)):
                    ElementTree.SubElement(doc, key).text = str(value, 'utf-8')
                else:
                    ElementTree.SubElement(doc, key).text = str(value)
            else:
                ElementTree.SubElement(doc, key)
    root = ElementTree.Element("report")
    _dict2xml(root, result)
    xml_doc = ElementTree.tostring(root)
    print()
    # send result to report server
    params = urllib.parse.urlencode({
        'timestamp': timestamp,
        'system': result['platform']['system'],
        'python_version': result['platform']['python_version'],
        'architecture': result['platform']['architecture'],
        'tests': tests,
        'failures': failures,
        'errors': errors,
        'modules': len(ttrs) + len(import_failures),
        'xml': xml_doc
    })
    headers = {"Content-type": "application/x-www-form-urlencoded",
               "Accept": "text/plain"}
    conn = http.client.HTTPConnection(server)
    conn.request("POST", "/", params, headers)
    # get the response
    response = conn.getresponse()
    # handle redirect
    if response.status == 301:
        o = urllib.parse.urlparse(response.msg['location'])
        conn = http.client.HTTPConnection(o.netloc)
        conn.request("POST", o.path, params, headers)
        # get the response
        response = conn.getresponse()
    # handle errors
    if response.status == 200:
        print("Test report has been sent to %s. Thank you!" % (server))
    else:
        print("Error: Could not sent a test report to %s." % (server))
        print(response.reason)
    conn.close()
Exemplo n.º 15
0
        name='Booby',
        owner={
            'login': '******',
            'name': 'Jaime Gil de Sagredo'
        })

    print booby.to_json()
    '{"owner": {"login": "******", "name": "Jaime Gil de Sagredo"}, "name": "Booby"}'
"""
from __future__ import print_function
import tempfile

from future.standard_library import hooks

# Import urljoin for V2 and V3 - http://python-future.org/compatible_idioms.html
with hooks():
    from urllib.parse import urljoin

__author__ = 'Danny Goodall'
import requests
import requests.exceptions
import requests_cache
from booby import Model, fields
from codefurther.errors import CodeFurtherConnectionError, CodeFurtherError, CodeFurtherHTTPError, CodeFurtherReadTimeoutError


class Change(Model):
    """The Change model that describes the change of this entry since last week's chart.

    This class isn't made publicly visible, so it should never really need to be initialised manually. That said,
    it is initialised by passing a series of keyword arguments, like so::
Exemplo n.º 16
0
import sys
import websocket
import threading
import traceback
import ssl
from time import sleep
import json
import decimal
import logging
from market_maker.settings import settings
from market_maker.auth.APIKeyAuth import generate_expires, generate_signature
from market_maker.utils.log import setup_custom_logger
from market_maker.utils.math import toNearest
from future.utils import iteritems
from future.standard_library import hooks
with hooks():  # Python 2/3 compat
    from urllib.parse import urlparse, urlunparse


# Connects to BitMEX websocket for streaming realtime data.
# The Marketmaker still interacts with this as if it were a REST Endpoint, but now it can get
# much more realtime data without heavily polling the API.
#
# The Websocket offers a bunch of data as raw properties right on the object.
# On connect, it synchronously asks for a push of all this data then returns.
# Right after, the MM can start using its data. It will be updated in realtime, so the MM can
# poll as often as it wants.
class BitMEXWebsocket():

    # Don't grow a table larger than this amount. Helps cap memory usage.
    MAX_TABLE_LEN = 200
Exemplo n.º 17
0
def _createReport(ttrs, timetaken, log, server, hostname, sorted_tests):
    # import additional libraries here to speed up normal tests
    from future import standard_library
    with standard_library.hooks():
        import urllib.parse
        import http.client
    from xml.sax.saxutils import escape
    import codecs
    from xml.etree import ElementTree as etree
    timestamp = int(time.time())
    result = {'timestamp': timestamp}
    result['slowest_tests'] = [("%0.3fs" % dt, "%s" % desc)
                               for (desc, dt) in sorted_tests[:20]]
    result['timetaken'] = timetaken
    if log:
        try:
            data = codecs.open(log, 'r', encoding='UTF-8').read()
            result['install_log'] = escape(data)
        except:
            print(("Cannot open log file %s" % log))
    # get ObsPy module versions
    result['obspy'] = {}
    tests = 0
    errors = 0
    failures = 0
    skipped = 0
    try:
        installed = get_git_version()
    except:
        installed = ''
    result['obspy']['installed'] = installed
    for module in sorted(ALL_MODULES):
        result['obspy'][module] = {}
        if module not in ttrs:
            continue
        result['obspy'][module]['installed'] = installed
        # test results
        ttr = ttrs[module]
        result['obspy'][module]['timetaken'] = ttr.__dict__['timetaken']
        result['obspy'][module]['tested'] = True
        result['obspy'][module]['tests'] = ttr.testsRun
        # skipped is not supported for Python < 2.7
        try:
            skipped += len(ttr.skipped)
            result['obspy'][module]['skipped'] = len(ttr.skipped)
        except AttributeError:
            skipped = ''
            result['obspy'][module]['skipped'] = ''
        tests += ttr.testsRun
        # depending on module type either use failure (network related modules)
        # or errors (all others)
        result['obspy'][module]['errors'] = {}
        result['obspy'][module]['failures'] = {}
        if module in NETWORK_MODULES:
            for _, text in ttr.errors:
                result['obspy'][module]['failures']['f%s' % (failures)] = text
                failures += 1
            for _, text in ttr.failures:
                result['obspy'][module]['failures']['f%s' % (failures)] = text
                failures += 1
        else:
            for _, text in ttr.errors:
                result['obspy'][module]['errors']['f%s' % (errors)] = text
                errors += 1
            for _, text in ttr.failures:
                result['obspy'][module]['errors']['f%s' % (errors)] = text
                errors += 1
    # get dependencies
    result['dependencies'] = {}
    for module in DEPENDENCIES:
        temp = module.split('.')
        try:
            mod = __import__(module, fromlist=[native_str(temp[1:])])
            if module == '_omnipy':
                result['dependencies'][module] = mod.coreVersion()
            else:
                result['dependencies'][module] = mod.__version__
        except ImportError:
            result['dependencies'][module] = ''
    # get system / environment settings
    result['platform'] = {}
    for func in [
            'system', 'release', 'version', 'machine', 'processor',
            'python_version', 'python_implementation', 'python_compiler',
            'architecture'
    ]:
        try:
            temp = getattr(platform, func)()
            if isinstance(temp, tuple):
                temp = temp[0]
            result['platform'][func] = temp
        except:
            result['platform'][func] = ''
    # set node name to hostname if set
    result['platform']['node'] = hostname
    # post only the first part of the node name (only applies to MacOS X)
    try:
        result['platform']['node'] = result['platform']['node'].split('.')[0]
    except:
        pass
    # test results
    result['tests'] = tests
    result['errors'] = errors
    result['failures'] = failures
    result['skipped'] = skipped

    # generate XML document
    def _dict2xml(doc, result):
        for key, value in result.items():
            key = key.split('(')[0].strip()
            if isinstance(value, dict):
                child = etree.SubElement(doc, key)
                _dict2xml(child, value)
            elif value is not None:
                if isinstance(value, (str, native_str)):
                    etree.SubElement(doc, key).text = value
                elif isinstance(value, (str, native_str)):
                    etree.SubElement(doc, key).text = str(value, 'utf-8')
                else:
                    etree.SubElement(doc, key).text = str(value)
            else:
                etree.SubElement(doc, key)

    root = etree.Element("report")
    _dict2xml(root, result)
    xml_doc = etree.tostring(root)
    print()
    # send result to report server
    params = urllib.parse.urlencode({
        'timestamp':
        timestamp,
        'system':
        result['platform']['system'],
        'python_version':
        result['platform']['python_version'],
        'architecture':
        result['platform']['architecture'],
        'tests':
        tests,
        'failures':
        failures,
        'errors':
        errors,
        'modules':
        len(ttrs),
        'xml':
        xml_doc
    })
    headers = {
        "Content-type": "application/x-www-form-urlencoded",
        "Accept": "text/plain"
    }
    conn = http.client.HTTPConnection(server)
    conn.request("POST", "/", params, headers)
    # get the response
    response = conn.getresponse()
    # handle redirect
    if response.status == 301:
        o = urllib.parse.urlparse(response.msg['location'])
        conn = http.client.HTTPConnection(o.netloc)
        conn.request("POST", o.path, params, headers)
        # get the response
        response = conn.getresponse()
    # handle errors
    if response.status == 200:
        print(("Test report has been sent to %s. Thank you!" % (server)))
    else:
        print(("Error: Could not sent a test report to %s." % (server)))
        print((response.reason))
Exemplo n.º 18
0
def _create_report(ttrs, timetaken, log, server, hostname, sorted_tests, ci_url=None, pr_url=None):
    # import additional libraries here to speed up normal tests
    from future import standard_library

    with standard_library.hooks():
        import urllib.parse
        import http.client
    import codecs
    from xml.etree import ElementTree
    from xml.sax.saxutils import escape

    timestamp = int(time.time())
    result = {"timestamp": timestamp}
    result["slowest_tests"] = [("%0.3fs" % dt, "%s" % desc) for (desc, dt) in sorted_tests[:20]]
    result["timetaken"] = timetaken
    if log:
        try:
            data = codecs.open(log, "r", encoding="UTF-8").read()
            result["install_log"] = escape(data)
        except:
            print("Cannot open log file %s" % log)
    # get ObsPy module versions
    result["obspy"] = {}
    tests = 0
    errors = 0
    failures = 0
    skipped = 0
    try:
        installed = get_git_version()
    except:
        installed = ""
    result["obspy"]["installed"] = installed
    for module in sorted(ALL_MODULES):
        result["obspy"][module] = {}
        if module not in ttrs:
            continue
        result["obspy"][module]["installed"] = installed
        # test results
        ttr = ttrs[module]
        result["obspy"][module]["timetaken"] = ttr.__dict__["timetaken"]
        result["obspy"][module]["tested"] = True
        result["obspy"][module]["tests"] = ttr.testsRun
        # skipped is not supported for Python < 2.7
        try:
            skipped += len(ttr.skipped)
            result["obspy"][module]["skipped"] = len(ttr.skipped)
        except AttributeError:
            skipped = ""
            result["obspy"][module]["skipped"] = ""
        tests += ttr.testsRun
        # depending on module type either use failure (network related modules)
        # or errors (all others)
        result["obspy"][module]["errors"] = {}
        result["obspy"][module]["failures"] = {}
        if module in NETWORK_MODULES:
            for _, text in ttr.errors:
                result["obspy"][module]["failures"]["f%s" % (failures)] = text
                failures += 1
            for _, text in ttr.failures:
                result["obspy"][module]["failures"]["f%s" % (failures)] = text
                failures += 1
        else:
            for _, text in ttr.errors:
                result["obspy"][module]["errors"]["f%s" % (errors)] = text
                errors += 1
            for _, text in ttr.failures:
                result["obspy"][module]["errors"]["f%s" % (errors)] = text
                errors += 1
    # get dependencies
    result["dependencies"] = {}
    for module in DEPENDENCIES:
        if module == "pep8-naming":
            module_ = "pep8ext_naming"
        else:
            module_ = module
        temp = module_.split(".")
        try:
            mod = __import__(module_, fromlist=[native_str(temp[1:])])
        except ImportError:
            version_ = "---"
        else:
            try:
                version_ = mod.__version__
            except AttributeError:
                version_ = "???"
        result["dependencies"][module] = version_
    # get system / environment settings
    result["platform"] = {}
    for func in [
        "system",
        "release",
        "version",
        "machine",
        "processor",
        "python_version",
        "python_implementation",
        "python_compiler",
        "architecture",
    ]:
        try:
            temp = getattr(platform, func)()
            if isinstance(temp, tuple):
                temp = temp[0]
            result["platform"][func] = temp
        except:
            result["platform"][func] = ""
    # set node name to hostname if set
    result["platform"]["node"] = hostname
    # post only the first part of the node name (only applies to MacOS X)
    try:
        result["platform"]["node"] = result["platform"]["node"].split(".")[0]
    except:
        pass
    # test results
    result["tests"] = tests
    result["errors"] = errors
    result["failures"] = failures
    result["skipped"] = skipped
    if ci_url is not None:
        result["ciurl"] = ci_url
    if pr_url is not None:
        result["prurl"] = pr_url

    # generate XML document
    def _dict2xml(doc, result):
        for key, value in result.items():
            key = key.split("(")[0].strip()
            if isinstance(value, dict):
                child = ElementTree.SubElement(doc, key)
                _dict2xml(child, value)
            elif value is not None:
                if isinstance(value, (str, native_str)):
                    ElementTree.SubElement(doc, key).text = value
                elif isinstance(value, (str, native_str)):
                    ElementTree.SubElement(doc, key).text = str(value, "utf-8")
                else:
                    ElementTree.SubElement(doc, key).text = str(value)
            else:
                ElementTree.SubElement(doc, key)

    root = ElementTree.Element("report")
    _dict2xml(root, result)
    xml_doc = ElementTree.tostring(root)
    print()
    # send result to report server
    params = urllib.parse.urlencode(
        {
            "timestamp": timestamp,
            "system": result["platform"]["system"],
            "python_version": result["platform"]["python_version"],
            "architecture": result["platform"]["architecture"],
            "tests": tests,
            "failures": failures,
            "errors": errors,
            "modules": len(ttrs),
            "xml": xml_doc,
        }
    )
    headers = {"Content-type": "application/x-www-form-urlencoded", "Accept": "text/plain"}
    conn = http.client.HTTPConnection(server)
    conn.request("POST", "/", params, headers)
    # get the response
    response = conn.getresponse()
    # handle redirect
    if response.status == 301:
        o = urllib.parse.urlparse(response.msg["location"])
        conn = http.client.HTTPConnection(o.netloc)
        conn.request("POST", o.path, params, headers)
        # get the response
        response = conn.getresponse()
    # handle errors
    if response.status == 200:
        print("Test report has been sent to %s. Thank you!" % (server))
    else:
        print("Error: Could not sent a test report to %s." % (server))
        print(response.reason)
Exemplo n.º 19
0
def _create_report(ttrs, timetaken, log, server, hostname, sorted_tests):
    # import additional libraries here to speed up normal tests
    from future import standard_library
    with standard_library.hooks():
        import urllib.parse
        import http.client
    import codecs
    from xml.etree import ElementTree
    from xml.sax.saxutils import escape
    timestamp = int(time.time())
    result = {'timestamp': timestamp}
    result['slowest_tests'] = [("%0.3fs" % dt, "%s" % desc)
                               for (desc, dt) in sorted_tests[:20]]
    result['timetaken'] = timetaken
    if log:
        try:
            data = codecs.open(log, 'r', encoding='UTF-8').read()
            result['install_log'] = escape(data)
        except:
            print("Cannot open log file %s" % log)
    # get ObsPy module versions
    result['obspy'] = {}
    tests = 0
    errors = 0
    failures = 0
    skipped = 0
    try:
        installed = get_git_version()
    except:
        installed = ''
    result['obspy']['installed'] = installed
    for module in sorted(ALL_MODULES):
        result['obspy'][module] = {}
        if module not in ttrs:
            continue
        result['obspy'][module]['installed'] = installed
        # test results
        ttr = ttrs[module]
        result['obspy'][module]['timetaken'] = ttr.__dict__['timetaken']
        result['obspy'][module]['tested'] = True
        result['obspy'][module]['tests'] = ttr.testsRun
        # skipped is not supported for Python < 2.7
        try:
            skipped += len(ttr.skipped)
            result['obspy'][module]['skipped'] = len(ttr.skipped)
        except AttributeError:
            skipped = ''
            result['obspy'][module]['skipped'] = ''
        tests += ttr.testsRun
        # depending on module type either use failure (network related modules)
        # or errors (all others)
        result['obspy'][module]['errors'] = {}
        result['obspy'][module]['failures'] = {}
        if module in NETWORK_MODULES:
            for _, text in ttr.errors:
                result['obspy'][module]['failures']['f%s' % (failures)] = text
                failures += 1
            for _, text in ttr.failures:
                result['obspy'][module]['failures']['f%s' % (failures)] = text
                failures += 1
        else:
            for _, text in ttr.errors:
                result['obspy'][module]['errors']['f%s' % (errors)] = text
                errors += 1
            for _, text in ttr.failures:
                result['obspy'][module]['errors']['f%s' % (errors)] = text
                errors += 1
    # get dependencies
    result['dependencies'] = {}
    for module in DEPENDENCIES:
        temp = module.split('.')
        try:
            mod = __import__(module,
                             fromlist=[native_str(temp[1:])])
            if module == '_omnipy':
                result['dependencies'][module] = mod.coreVersion()
            else:
                result['dependencies'][module] = mod.__version__
        except ImportError:
            result['dependencies'][module] = ''
    # get system / environment settings
    result['platform'] = {}
    for func in ['system', 'release', 'version', 'machine',
                 'processor', 'python_version', 'python_implementation',
                 'python_compiler', 'architecture']:
        try:
            temp = getattr(platform, func)()
            if isinstance(temp, tuple):
                temp = temp[0]
            result['platform'][func] = temp
        except:
            result['platform'][func] = ''
    # set node name to hostname if set
    result['platform']['node'] = hostname
    # post only the first part of the node name (only applies to MacOS X)
    try:
        result['platform']['node'] = result['platform']['node'].split('.')[0]
    except:
        pass
    # test results
    result['tests'] = tests
    result['errors'] = errors
    result['failures'] = failures
    result['skipped'] = skipped

    # generate XML document
    def _dict2xml(doc, result):
        for key, value in result.items():
            key = key.split('(')[0].strip()
            if isinstance(value, dict):
                child = ElementTree.SubElement(doc, key)
                _dict2xml(child, value)
            elif value is not None:
                if isinstance(value, (str, native_str)):
                    ElementTree.SubElement(doc, key).text = value
                elif isinstance(value, (str, native_str)):
                    ElementTree.SubElement(doc, key).text = str(value, 'utf-8')
                else:
                    ElementTree.SubElement(doc, key).text = str(value)
            else:
                ElementTree.SubElement(doc, key)
    root = ElementTree.Element("report")
    _dict2xml(root, result)
    xml_doc = ElementTree.tostring(root)
    print()
    # send result to report server
    params = urllib.parse.urlencode({
        'timestamp': timestamp,
        'system': result['platform']['system'],
        'python_version': result['platform']['python_version'],
        'architecture': result['platform']['architecture'],
        'tests': tests,
        'failures': failures,
        'errors': errors,
        'modules': len(ttrs),
        'xml': xml_doc
    })
    headers = {"Content-type": "application/x-www-form-urlencoded",
               "Accept": "text/plain"}
    conn = http.client.HTTPConnection(server)
    conn.request("POST", "/", params, headers)
    # get the response
    response = conn.getresponse()
    # handle redirect
    if response.status == 301:
        o = urllib.parse.urlparse(response.msg['location'])
        conn = http.client.HTTPConnection(o.netloc)
        conn.request("POST", o.path, params, headers)
        # get the response
        response = conn.getresponse()
    # handle errors
    if response.status == 200:
        print("Test report has been sent to %s. Thank you!" % (server))
    else:
        print("Error: Could not sent a test report to %s." % (server))
        print(response.reason)
Exemplo n.º 20
0
def _create_report(ttrs,
                   timetaken,
                   log,
                   server,
                   hostname,
                   sorted_tests,
                   ci_url=None,
                   pr_url=None,
                   import_failures=None):
    # import additional libraries here to speed up normal tests
    from future import standard_library
    with standard_library.hooks():
        import urllib.parse
        import http.client
    import codecs
    from xml.etree import ElementTree
    from xml.sax.saxutils import escape
    if import_failures is None:
        import_failures = {}
    timestamp = int(time.time())
    result = {'timestamp': timestamp}
    result['slowest_tests'] = [("%0.3fs" % dt, "%s" % desc)
                               for (desc, dt) in sorted_tests[:20]]
    result['timetaken'] = timetaken
    if log:
        try:
            data = codecs.open(log, 'r', encoding='UTF-8').read()
            result['install_log'] = escape(data)
        except Exception:
            print("Cannot open log file %s" % log)
    # get ObsPy module versions
    result['obspy'] = {}
    tests = 0
    errors = 0
    failures = 0
    skipped = 0
    try:
        installed = get_git_version()
    except Exception:
        installed = ''
    result['obspy']['installed'] = installed
    for module in sorted(ALL_MODULES):
        result['obspy'][module] = {}
        result['obspy'][module]['installed'] = installed
        # add a failed-to-import test module to report with an error
        if module in import_failures:
            result['obspy'][module]['timetaken'] = 0
            result['obspy'][module]['tested'] = True
            result['obspy'][module]['tests'] = 1
            # can't say how many tests would have been in that suite so just
            # leave 0
            result['obspy'][module]['skipped'] = 0
            result['obspy'][module]['failures'] = {}
            result['obspy'][module]['errors'] = {
                'f%s' % (errors): import_failures[module]
            }
            tests += 1
            errors += 1
            continue
        if module not in ttrs:
            continue
        # test results
        ttr = ttrs[module]
        result['obspy'][module]['timetaken'] = ttr.__dict__['timetaken']
        result['obspy'][module]['tested'] = True
        result['obspy'][module]['tests'] = ttr.testsRun
        skipped += len(ttr.skipped)
        result['obspy'][module]['skipped'] = len(ttr.skipped)
        tests += ttr.testsRun
        # depending on module type either use failure (network related modules)
        # or errors (all others)
        result['obspy'][module]['errors'] = {}
        result['obspy'][module]['failures'] = {}
        if module in NETWORK_MODULES:
            for _, text in ttr.errors:
                result['obspy'][module]['failures']['f%s' % (failures)] = text
                failures += 1
            for _, text in ttr.failures:
                result['obspy'][module]['failures']['f%s' % (failures)] = text
                failures += 1
        else:
            for _, text in ttr.errors:
                result['obspy'][module]['errors']['f%s' % (errors)] = text
                errors += 1
            for _, text in ttr.failures:
                result['obspy'][module]['errors']['f%s' % (errors)] = text
                errors += 1
    # get dependencies
    result['dependencies'] = {}
    for module in DEPENDENCIES:
        if module == "pep8-naming":
            module_ = "pep8ext_naming"
        else:
            module_ = module
        temp = module_.split('.')
        try:
            mod = __import__(module_, fromlist=[native_str(temp[1:])])
        except ImportError:
            version_ = '---'
        else:
            try:
                version_ = mod.__version__
            except AttributeError:
                version_ = '???'
        result['dependencies'][module] = version_
    # get system / environment settings
    result['platform'] = {}
    for func in [
            'system', 'release', 'version', 'machine', 'processor',
            'python_version', 'python_implementation', 'python_compiler',
            'architecture'
    ]:
        try:
            temp = getattr(platform, func)()
            if isinstance(temp, tuple):
                temp = temp[0]
            result['platform'][func] = temp
        except Exception:
            result['platform'][func] = ''
    # set node name to hostname if set
    result['platform']['node'] = hostname
    # post only the first part of the node name (only applies to MacOS X)
    try:
        result['platform']['node'] = result['platform']['node'].split('.')[0]
    except Exception:
        pass
    # test results
    result['tests'] = tests
    result['errors'] = errors
    result['failures'] = failures
    result['skipped'] = skipped
    # try to append info on skipped tests:
    result['skipped_tests_details'] = []
    try:
        for module, testresult_ in ttrs.items():
            if testresult_.skipped:
                for skipped_test, skip_message in testresult_.skipped:
                    result['skipped_tests_details'].append(
                        (module, skipped_test.__module__,
                         skipped_test.__class__.__name__,
                         skipped_test._testMethodName, skip_message))
    except Exception:
        exc_type, exc_value, exc_tb = sys.exc_info()
        print("\n".join(traceback.format_exception(exc_type, exc_value,
                                                   exc_tb)))
        result['skipped_tests_details'] = []

    if ci_url is not None:
        result['ciurl'] = ci_url
    if pr_url is not None:
        result['prurl'] = pr_url

    # generate XML document
    def _dict2xml(doc, result):
        for key, value in result.items():
            key = key.split('(')[0].strip()
            if isinstance(value, dict):
                child = ElementTree.SubElement(doc, key)
                _dict2xml(child, value)
            elif value is not None:
                if isinstance(value, (str, native_str)):
                    ElementTree.SubElement(doc, key).text = value
                elif isinstance(value, (str, native_str)):
                    ElementTree.SubElement(doc, key).text = str(value, 'utf-8')
                else:
                    ElementTree.SubElement(doc, key).text = str(value)
            else:
                ElementTree.SubElement(doc, key)

    root = ElementTree.Element("report")
    _dict2xml(root, result)
    xml_doc = ElementTree.tostring(root)
    print()
    # send result to report server
    params = urllib.parse.urlencode({
        'timestamp':
        timestamp,
        'system':
        result['platform']['system'],
        'python_version':
        result['platform']['python_version'],
        'architecture':
        result['platform']['architecture'],
        'tests':
        tests,
        'failures':
        failures,
        'errors':
        errors,
        'modules':
        len(ttrs) + len(import_failures),
        'xml':
        xml_doc
    })
    headers = {
        "Content-type": "application/x-www-form-urlencoded",
        "Accept": "text/plain"
    }
    conn = http.client.HTTPConnection(server)
    conn.request("POST", "/", params, headers)
    # get the response
    response = conn.getresponse()
    # handle redirect
    if response.status == 301:
        o = urllib.parse.urlparse(response.msg['location'])
        conn = http.client.HTTPConnection(o.netloc)
        conn.request("POST", o.path, params, headers)
        # get the response
        response = conn.getresponse()
    # handle errors
    if response.status == 200:
        print("Test report has been sent to %s. Thank you!" % (server))
    else:
        print("Error: Could not sent a test report to %s." % (server))
        print(response.reason)
    conn.close()
Exemplo n.º 21
0
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
The obspy.clients.seishub.client test suite.
"""
from __future__ import (absolute_import, division, print_function,
                        unicode_literals)
from future.builtins import *  # NOQA
from future import standard_library

import unittest

with standard_library.hooks():
    import urllib.request

from obspy.core import AttribDict, UTCDateTime
from obspy.clients.seishub import Client


TESTSERVER = "http://teide.geophysik.uni-muenchen.de:8080"
TESTSERVER_UNREACHABLE_MSG = "Seishub test server not reachable."


def _check_server_availability():
    """
    Returns an empty string if server is reachable or failure message
    otherwise.
    """
    try:
        code = urllib.request.urlopen(TESTSERVER, timeout=3).getcode()
        assert(code == 200)
Exemplo n.º 22
0
from os.path import abspath, dirname, join, exists, basename, splitext
from functools import partial
from os import mkdir
import gzip
from glob import glob

from future import standard_library
from future.utils import viewitems

from qiita_core.exceptions import QiitaEnvironmentError
from qiita_core.qiita_settings import qiita_config
from .sql_connection import SQLConnectionHandler
from .reference import Reference
from natsort import natsorted

with standard_library.hooks():
    from urllib.request import urlretrieve


get_support_file = partial(join, join(dirname(abspath(__file__)),
                                      'support_files'))
reference_base_dir = join(qiita_config.base_data_dir, "reference")
get_reference_fp = partial(join, reference_base_dir)


SETTINGS_FP = get_support_file('qiita-db-settings.sql')
LAYOUT_FP = get_support_file('qiita-db-unpatched.sql')
POPULATE_FP = get_support_file('populate_test_db.sql')
PATCHES_DIR = get_support_file('patches')

Exemplo n.º 23
0
import datetime
import collections
from future.standard_library import hooks

from pycronofy import settings
from pycronofy.auth import Auth
from pycronofy.batch import BatchEntry
from pycronofy.batch import BatchResponse
from pycronofy.datetime_utils import format_event_time
from pycronofy.exceptions import PyCronofyPartialSuccessError
from pycronofy.pagination import Pages
from pycronofy.request_handler import RequestHandler
from pycronofy.validation import validate

with hooks():
    from urllib.parse import urlencode


class Client(object):
    """Client for cronofy web service.
    Performs authentication, and wraps API: https://docs.cronofy.com/developers/api/authorization/
    """

    def __init__(self, client_id=None, client_secret=None, access_token=None, refresh_token=None, token_expiration=None, data_center=None):
        """
        Example Usage:

        pycronofy.Client(access_token='')
        pycronofy.Client(client_id='', client_secret='')

        :param string client_id: OAuth Client ID. (Optional, default None)