def _fetch(self, service, data=None, headers={}, param_list=[], **params): """ Send a HTTP request via urllib2. :type service: str :param service: Name of service :type data: str :param data: Channel list as returned by `availability` Web service :type headers: dict, optional :param headers: Additional header information for request """ headers['User-Agent'] = self.user_agent # replace special characters remoteaddr = "/".join([self.base_url.rstrip("/"), service, str(self.major_versions[service]), "query"]) options = '&'.join(param_list) if params: if options: options += '&' options += compatibility.urlencode(params) if options: remoteaddr = "%s?%s" % (remoteaddr, options) if self.debug: print(('\nRequesting %s' % (remoteaddr))) req = compatibility.Request(url=remoteaddr, data=data, headers=headers) response = compatibility.urlopen(req, timeout=self.timeout) doc = response.read() return doc
def _fetch(self, url, headers={}, **params): """ Send a HTTP request via urllib2. :type url: str :param url: Complete URL of resource :type headers: dict :param headers: Additional header information for request """ headers['User-Agent'] = self.user_agent # replace special characters remoteaddr = self.base_url + url + '?' + \ compatibility.urlencode(params) if self.debug: print(('\nRequesting %s' % (remoteaddr))) response = compatibility.urlopen(remoteaddr, timeout=self.timeout) doc = response.read() return doc
def _fetch(self, url, *args, **kwargs): # @UnusedVariable params = {} # map keywords for key, value in KEYWORDS.items(): if key in list(kwargs.keys()): kwargs[value] = kwargs[key] del kwargs[key] # check for ranges and empty values for key, value in kwargs.items(): if not value and value != 0: continue if isinstance(value, tuple) and len(value) == 2: params['min_' + str(key)] = str(value[0]) params['max_' + str(key)] = str(value[1]) elif isinstance(value, list) and len(value) == 2: params['min_' + str(key)] = str(value[0]) params['max_' + str(key)] = str(value[1]) else: params[str(key)] = str(value) # replace special characters remoteaddr = self.base_url + url + '?' + urlencode(params) if self.debug: print(('\nRequesting %s' % (remoteaddr))) # certain requests randomly fail on rare occasions, retry for _i in range(self.retries): try: response = urlopen(remoteaddr, timeout=self.timeout) doc = response.read() return doc # XXX currently there are random problems with SeisHub's internal # XXX sql database access ("cannot operate on a closed database"). # XXX this can be circumvented by issuing the same request again.. except Exception: continue response = urlopen(remoteaddr, timeout=self.timeout) doc = response.read() return doc
def _createReport(ttrs, timetaken, log, server, hostname): # import additional libraries here to speed up normal tests from obspy.core import compatibility from xml.sax.saxutils import escape import codecs from xml.etree import ElementTree as etree timestamp = int(time.time()) result = {'timestamp': timestamp} result['timetaken'] = timetaken if log: try: data = codecs.open(log, 'r', encoding='UTF-8').read() result['install_log'] = escape(data) except: print(("Cannot open log file %s" % log)) # get ObsPy module versions result['obspy'] = {} tests = 0 errors = 0 failures = 0 skipped = 0 try: installed = get_git_version() except: installed = '' result['obspy']['installed'] = installed for module in sorted(ALL_MODULES): result['obspy'][module] = {} if module not in ttrs: continue result['obspy'][module]['installed'] = installed # test results ttr = ttrs[module] result['obspy'][module]['timetaken'] = ttr.__dict__['timetaken'] result['obspy'][module]['tested'] = True result['obspy'][module]['tests'] = ttr.testsRun # skipped is not supported for Python < 2.7 try: skipped += len(ttr.skipped) result['obspy'][module]['skipped'] = len(ttr.skipped) except AttributeError: skipped = '' result['obspy'][module]['skipped'] = '' tests += ttr.testsRun # depending on module type either use failure (network related modules) # or errors (all others) result['obspy'][module]['errors'] = {} result['obspy'][module]['failures'] = {} if module in NETWORK_MODULES: for _, text in ttr.errors: result['obspy'][module]['failures']['f%s' % (failures)] = text failures += 1 for _, text in ttr.failures: result['obspy'][module]['failures']['f%s' % (failures)] = text failures += 1 else: for _, text in ttr.errors: result['obspy'][module]['errors']['f%s' % (errors)] = text errors += 1 for _, text in ttr.failures: result['obspy'][module]['errors']['f%s' % (errors)] = text errors += 1 # get dependencies result['dependencies'] = {} for module in DEPENDENCIES: temp = module.split('.') try: mod = __import__(module, fromlist=[native_str(temp[1:])]) if module == '_omnipy': result['dependencies'][module] = mod.coreVersion() else: result['dependencies'][module] = mod.__version__ except ImportError: result['dependencies'][module] = '' # get system / environment settings result['platform'] = {} for func in ['system', 'release', 'version', 'machine', 'processor', 'python_version', 'python_implementation', 'python_compiler', 'architecture']: try: temp = getattr(platform, func)() if isinstance(temp, tuple): temp = temp[0] result['platform'][func] = temp except: result['platform'][func] = '' # set node name to hostname if set result['platform']['node'] = hostname # post only the first part of the node name (only applies to MacOS X) try: result['platform']['node'] = result['platform']['node'].split('.')[0] except: pass # test results result['tests'] = tests result['errors'] = errors result['failures'] = failures result['skipped'] = skipped # generate XML document def _dict2xml(doc, result): for key, value in result.items(): key = key.split('(')[0].strip() if isinstance(value, dict): child = etree.SubElement(doc, key) _dict2xml(child, value) elif value is not None: if isinstance(value, (str, native_str)): etree.SubElement(doc, key).text = value elif isinstance(value, (str, native_str)): etree.SubElement(doc, key).text = str(value, 'utf-8') else: etree.SubElement(doc, key).text = str(value) else: etree.SubElement(doc, key) root = etree.Element("report") _dict2xml(root, result) xml_doc = etree.tostring(root) print() # send result to report server params = compatibility.urlencode({ 'timestamp': timestamp, 'system': result['platform']['system'], 'python_version': result['platform']['python_version'], 'architecture': result['platform']['architecture'], 'tests': tests, 'failures': failures, 'errors': errors, 'modules': len(ttrs), 'xml': xml_doc }) headers = {"Content-type": "application/x-www-form-urlencoded", "Accept": "text/plain"} conn = compatibility.HTTPConnection(server) conn.request("POST", "/", params, headers) # get the response response = conn.getresponse() # handle redirect if response.status == 301: o = compatibility.urlparse(response.msg['location']) conn = compatibility.HTTPConnection(o.netloc) conn.request("POST", o.path, params, headers) # get the response response = conn.getresponse() # handle errors if response.status == 200: print(("Test report has been sent to %s. Thank you!" % (server))) else: print(("Error: Could not sent a test report to %s." % (server))) print((response.reason))