def get_current_url(environ, root_only=False, strip_querystring=False, host_only=False): """A handy helper function that recreates the full URL for the current request or parts of it. Here an example: >>> from werkzeug import create_environ >>> env = create_environ("/?param=foo", "http://localhost/script") >>> get_current_url(env) 'http://localhost/script/?param=foo' >>> get_current_url(env, root_only=True) 'http://localhost/script/' >>> get_current_url(env, host_only=True) 'http://localhost/' >>> get_current_url(env, strip_querystring=True) 'http://localhost/script/' :param environ: the WSGI environment to get the current URL from. :param root_only: set `True` if you only want the root URL. :param strip_querystring: set to `True` if you don't want the querystring. :param host_only: set to `True` if the host URL should be returned. """ tmp = [environ['wsgi.url_scheme'], '://', get_host(environ)] cat = tmp.append if host_only: return ''.join(tmp) + '/' cat(urllib_quote(environ.get('SCRIPT_NAME', '').rstrip('/'))) if root_only: cat('/') else: cat(urllib_quote('/' + environ.get('PATH_INFO', '').lstrip('/'))) if not strip_querystring: qs = environ.get('QUERY_STRING') if qs: cat('?' + qs) return ''.join(tmp)
def get_emailage_url(method, url, consumer_key, consumer_secret, query=None): """Generate the oauth url for emailAge :param query: :param method: :param url: :param consumer_key: :param consumer_secret: :return: """ if not method: method = "GET" nonce, timestamp = generate_nonce_timestamp() # URL encode credential params cred_params = [('format', 'json'), ('oauth_consumer_key', consumer_key), ('oauth_nonce', nonce), ('oauth_signature_method', 'HMAC-SHA1'), ('oauth_timestamp', timestamp), ('oauth_version', '1.0')] if method == 'GET': cred_params.append(('query', query)) cred_params = urllib_urlencode(cred_params) """ivar: credential parameters required in the payload.""" query_str = cred_params sig_url = method.upper() + "&" + urllib_quote(url, "") + "&" + urllib_quote(query_str, "") sig = get_signature(consumer_secret, sig_url) """ivar: signature based on consumer secret to validate request.""" oauth_url = url + "?" + query_str + "&oauth_signature=" + urllib_quote(sig.decode(), "") return oauth_url
def search(self, word): url = "searchPoints/" + urllib_quote(word) req = API._rest_req(url) req = json.loads(req.text) ret = [Point(i) for i in req["list"]] return ret
def post_message_as_slackbot(team, token, channel, message): url = "https://{team}.slack.com/services/hooks/slackbot" url += "?token={token}&channel={channel}" url = url.format(team=team, token=token, channel=urllib_quote(channel)) res = requests.post(url, message) if res.status_code != 200: raise SlackerCliError(f"{res.content}:'{url}'")
def _quote_query(query): """Turn a dictionary into a query string in a URL, with keys in alphabetical order.""" return "&".join( "%s=%s" % (k, urllib_quote(unicode(query[k]).encode('utf-8'), safe='~')) for k in sorted(query))
def _quote_query(query): """Turn a dictionary into a query string in a URL, with keys in alphabetical order.""" return "&".join("%s=%s" % ( k, urllib_quote( unicode(query[k]).encode('utf-8'), safe='~')) for k in sorted(query))
def _api_call(url, opener, http_timeout): """ Makes a REST call against the Jenkins API. Args: url (str): The URL to get, including endpoint Returns: list: The JSON response """ parsed_url = urlparse.urlparse(url) url = '{0}://{1}{2}'.format(parsed_url.scheme, parsed_url.netloc, urllib_quote(parsed_url.path)) try: urllib2.install_opener(opener) resp = urllib2.urlopen(url, timeout=http_timeout) except (urllib2.HTTPError) as e: # Workaround for Jenkins issue with health check # which returns 500 server code if a health check fails # https://github.com/sensu-plugins/sensu-plugins-jenkins/issues/10 if e.code == 500: return json.loads(e.read()) else: collectd.error("Error making API call (%s) %s" % (e, url)) return None except (urllib2.URLError) as e: collectd.error("Error making API call (%s) %s" % (e, url)) return None try: return json.load(resp) except ValueError, e: collectd.error("Error parsing JSON for API call (%s) %s" % (e, url)) return None
def quote(value, safe='/'): """ Patched version of urllib.quote that encodes utf-8 strings before quoting """ if isinstance(value, unicode): value = value.encode('utf-8') return urllib_quote(value, safe)
def _api_call(url, type, opener, http_timeout): """ Makes a REST call against the Jenkins API. Args: url (str): The URL to get, including endpoint Returns: list: The JSON response """ parsed_url = urlparse.urlparse(url) url = '{0}://{1}{2}'.format(parsed_url.scheme, parsed_url.netloc, urllib_quote(parsed_url.path)) resp = None try: urllib2.install_opener(opener) resp = urllib2.urlopen(url, timeout=http_timeout) return load_json(resp, url) except urllib2.HTTPError as e: if e.code == 500 and type == "healthcheck": return load_json(e, url) else: collectd.error("Error making API call (%s) %s" % (e, url)) return None except urllib2.URLError as e: collectd.error("Error making API call (%s) %s" % (e, url)) return None finally: if resp: resp.close()
def post_message_as_slackbot(team, token, channel, message): url = 'https://{team}.slack.com/services/hooks/slackbot' url += '?token={token}&channel={channel}' url = url.format(team=team, token=token, channel=urllib_quote(channel)) res = requests.post(url, message) if res.status_code != 200: raise SlackerCliError("{0}:'{1}'".format(res.content, url))
def escape_shield_query(text): """Escape text to be inserted in a shield API request.""" text = urllib_quote(text, safe=' ') text = text.replace('_', '__') text = text.replace(' ', '_') text = text.replace('-', '--') return text
def _api_call(url, opener, http_timeout): """ Makes a REST call against the Jenkins API. Args: url (str): The URL to get, including endpoint Returns: list: The JSON response """ parsed_url = urlparse.urlparse(url) url = '{0}://{1}{2}'.format(parsed_url.scheme, parsed_url.netloc, urllib_quote(parsed_url.path)) try: ctx = ssl.create_default_context() ctx.check_hostname = False ctx.verify_mode = ssl.CERT_NONE urllib2.install_opener(opener) resp = urllib2.urlopen(url, timeout=http_timeout, context=ctx) except (urllib2.HTTPError, urllib2.URLError) as e: collectd.error("Error making API call (%s) %s" % (e, url)) return None try: return json.load(resp) except ValueError, e: collectd.error("Error parsing JSON for API call (%s) %s" % (e, url)) return None
def _api_call(url, type, auth_args, http_timeout): """ Makes a REST call against the Jenkins API. Args: url (str): The URL to get, including endpoint Returns: list: The JSON response """ parsed_url = urlparse.urlparse(url) url = '{0}://{1}{2}'.format(parsed_url.scheme, parsed_url.netloc, urllib_quote(parsed_url.path)) resp = None try: resp = requests.get(url, timeout=http_timeout, **auth_args) resp.raise_for_status() return load_json(resp.text, url) except HTTPError as e: if e.response.status_code == 500 and type == "healthcheck": return load_json(e.response.text, url) else: collectd.error("Error making API call (%s) %s" % (e, url)) return None except RequestException as e: collectd.error("Error making API call (%s) %s" % (e, url)) return None finally: if resp: resp.close()
def encode_for_api(self, string_to_encode): """Make sure the URI in correctly encoded. Runabove api need to encode "/" to %2F because slash are used into URI to distinct two ressources. :param string_to_encode: original string_to_encode """ return urllib_quote(string_to_encode).replace('/', '%2f')
def searchLocation(self): if self.current.city.value != "": url = "http://query.yahooapis.com/v1/public/yql?q=select+*+from+geo.places+where+text='%s'&format=json" % ( urllib_quote(self.current.city.value)) getPage(url).addCallback(self.jsonCallback).addErrback(self.error) else: self.session.open( MessageBox, _("You need to enter a valid city name before you can search for the location code." ), MessageBox.TYPE_ERROR)
def create(self, name): url = BASE_URL + '/folders' # Clean up name name = name.replace(' ', '_') name = urllib_quote(name) params = {'name' : name} headers = {'Content-Type' : 'application/vnd.mendeley-folder.1+json'} return self.parent.make_post_request(url, models.Folder, params, headers=headers)
def getWeatherData(self, degreetype, locationcode, city, weatherSearchFullName, thingSpeakChannelID, callback, callbackShowIcon, callbackAllIconsDownloaded=None, Histogram=False): self.DEBUG('getWeather().getWeatherData', '>>>') self.initialize() self.collectDataForHistogram = Histogram language = config.osd.language.value.replace('_', '-') if language == 'en-EN': language = 'en-US' elif language == 'no-NO': language = 'nn-NO' elif language == 'lt-LT': language = 'en-xl' self.city = city self.callback = callback self.callbackShowIcon = callbackShowIcon self.callbackAllIconsDownloaded = callbackAllIconsDownloaded url = 'http://weather.service.msn.com/data.aspx?src=windows&weadegreetype=%s&culture=%s&wealocations=%s' % ( degreetype, language, urllib_quote(locationcode)) if weatherSearchFullName != '': getPage(url).addCallback(self.xmlCallback).addErrback( self.xmlError) self.DEBUG('\t url_xml=', '%s' % url) #url2 = 'http://www.msn.com/weather/we-city?culture=%s&form=PRWLAS&q=%s' % (language, urllib_quote(weatherSearchFullName)) self.urlWeb = 'https://www.msn.com/%s/weather?culture=%ss&weadegreetype=%s&form=PRWLAS&q=%s' % ( language, language, degreetype, urllib_quote(weatherSearchFullName)) self.DEBUG('\t url_web="%s"' % self.urlWeb) #getPage(url2).addCallback(self.webCallback).addErrback(self.webError) if thingSpeakChannelID != '': url3 = 'https://thingspeak.com/channels/%s/feeds.xml?average=10&results=1' % thingSpeakChannelID self.DEBUG('\t url_thingSpeak=', '%s' % url3) getPage(url3).addCallback(self.thingSpeakCallback).addErrback( self.thingSpeakError)
def _download_and_verify_file_name(self, test_file_name): for file_type in FILE_TYPE_MIME_TABLE.keys(): url_encoded_file_name = urllib_quote(test_file_name) response = self.app.get( '/download/%s/%s' % (url_encoded_file_name, file_type)) eq_(response.content_type, FILE_TYPE_MIME_TABLE[file_type]) expected = ( "attachment; filename=%s.%s;filename*=utf-8''%s.%s" % (url_encoded_file_name, file_type, url_encoded_file_name, file_type)) eq_(response.content_disposition, expected)
def _make_response(content, content_type, status, file_name=None): """ Custom response function that is called by pyexcel-webio """ response = Response(content, content_type=content_type, status=status) if file_name: if PY2_VERSION and isinstance(file_name, unicode): file_name = file_name.encode('utf-8') url_encoded_file_name = urllib_quote(file_name) response.content_disposition = ( "attachment; filename=%s;filename*=utf-8''%s" % (url_encoded_file_name, url_encoded_file_name)) return response
def searchLocation(self): if self.current.city.value != "": language = config.osd.language.value.replace("_", "-") if language == "en-EN": # hack language = "en-US" url = "http://weather.service.msn.com/find.aspx?outputview=search&weasearchstr=%s&culture=%s" % ( urllib_quote(self.current.city.value), language) getPage(url).addCallback(self.xmlCallback).addErrback(self.error) else: self.session.open( MessageBox, _("You need to enter a valid city name before you can search for the location code." ), MessageBox.TYPE_ERROR)
def _get_patch(self, request, *args, **kwargs): try: resources.review_request.get_object(request, *args, **kwargs) filediff = self.get_object(request, *args, **kwargs) except ObjectDoesNotExist: return DOES_NOT_EXIST resp = HttpResponse(filediff.diff, mimetype='text/x-patch') filename = '%s.patch' % urllib_quote(filediff.source_file) resp['Content-Disposition'] = 'inline; filename=%s' % filename set_last_modified(resp, filediff.diffset.timestamp) return resp
def _make_response(content, content_type, status, file_name=None): """ Custom response function that is called by pyexcel-webio """ response = Response(content, content_type=content_type, status=status) if file_name: if PY2_VERSION and isinstance(file_name, unicode): file_name = file_name.encode('utf-8') url_encoded_file_name = urllib_quote(file_name) response.content_disposition = ( "attachment; filename=%s;filename*=utf-8''%s" % (url_encoded_file_name, url_encoded_file_name) ) return response
def get_emailage_url(method, url, consumer_key, consumer_secret, query=None): """Generate the oauth url for emailAge :param query: :param method: :param url: :param consumer_key: :param consumer_secret: :return: """ if not method: method = "GET" nonce, timestamp = generate_nonce_timestamp() # URL encode credential params cred_params = [('format', 'json'), ('oauth_consumer_key', consumer_key), ('oauth_nonce', nonce), ('oauth_signature_method', 'HMAC-SHA1'), ('oauth_timestamp', timestamp), ('oauth_version', '1.0')] if method == 'GET': cred_params.append(('query', query)) cred_params = urllib_urlencode(cred_params) """ivar: credential parameters required in the payload.""" query_str = cred_params sig_url = method.upper() + "&" + urllib_quote( url, "") + "&" + urllib_quote(query_str, "") sig = get_signature(consumer_secret, sig_url) """ivar: signature based on consumer secret to validate request.""" oauth_url = url + "?" + query_str + "&oauth_signature=" + urllib_quote( sig.decode(), "") return oauth_url
def urlencode(cls, value): if type(value) not in (str, bytes): value = str(value) out = "" for char in value: if type(char) is int: char = bytearray([char]) quoted = urllib_quote(char, safe='') out += quoted if quoted[0] != '%' else quoted.lower() return out \ .replace('-', '%2d') \ .replace('_', '%5f') \ .replace('~', '%7e')
def reportHostCrash(self, ret): try: if ret: try: exceptStack = self.workThread.getExceptStack() reporter = GetPluginDir('iptvdm/reporthostcrash.py') msg = urllib_quote('%s|%s|%s|%s' % ('HOST_CRASH', IPTVSubDownloaderWidget.IPTV_VERSION, self.hostName, self.getCategoryPath())) self.crashConsole = iptv_system('python "%s" "http://iptvplayer.vline.pl/reporthostcrash.php?msg=%s" "%s" 2&>1 > /dev/null' % (reporter, msg, exceptStack)) printDBG(msg) except Exception: printExc() self.workThread = None self.prevSelList = [] self.back_pressed() except Exception: printExc()
def searchLocation(self): if self.current.city.value != '': language = config.osd.language.value.replace('_', '-') if language == 'en-EN': language = 'en-US' elif language == 'no-NO': language = 'nn-NO' elif language == 'pl-PL': language = 'pl-PL' url = "http://weather.service.msn.com/find.aspx?src=outlook&outputview=search&weasearchstr=%s&culture=%s" % ( urllib_quote(self.current.city.value), language) getPage(url).addCallback(self.xmlCallback).addErrback(self.error) else: self.session.open( MessageBox, _('You need to enter a valid city name before you can search for the location code.' ), MessageBox.TYPE_ERROR)
def getWeatherData(self, degreetype, locationcode, city, callback, callbackShowIcon, callbackAllIconsDownloaded=None): self.initialize() self.city = city self.callback = callback self.callbackShowIcon = callbackShowIcon self.callbackAllIconsDownloaded = callbackAllIconsDownloaded u = "+and+u='c'" if degreetype == "F": u = "" url = "http://query.yahooapis.com/v1/public/yql?q=select+*+from+weather.forecast+where+woeid=%s%s&format=json" % ( urllib_quote(locationcode), u) getPage(url).addCallback(self.jsonCallback).addErrback(self.error)
def get(self, request, diffset_id=None, *args, **kwargs): """Returns the patched file. The file is returned as :mimetype:`text/plain` and is the result of applying the patch to the original file. """ try: attached_diffset = DiffSet.objects.filter(pk=diffset_id, history__isnull=True) if attached_diffset.exists(): filediff_resource = resources.filediff else: filediff_resource = resources.draft_filediff filediff = filediff_resource.get_object( request, diffset=diffset_id, *args, **kwargs) except ObjectDoesNotExist: return DOES_NOT_EXIST if filediff.deleted: return DOES_NOT_EXIST try: orig_file = get_original_file(filediff, request=request) except Exception as e: logging.error("Error retrieving original file: %s", e, exc_info=1, request=request) return FILE_RETRIEVAL_ERROR try: patched_file = get_patched_file(orig_file, filediff, request=request) except Exception as e: logging.error("Error retrieving patched file: %s", e, exc_info=1, request=request) return FILE_RETRIEVAL_ERROR resp = HttpResponse(patched_file, mimetype='text/plain') filename = urllib_quote(filediff.dest_file) resp['Content-Disposition'] = 'inline; filename=%s' % filename set_last_modified(resp, filediff.diffset.timestamp) return resp
def html_one_day(db, W): s = SQL_history(db) if type(W) in string_types: W = open(W, 'w').write W(_html_top) since = int(time.time()) - 24 * 3600 W('Deltas created from ' + time.ctime(since) + ' to ' + time.ctime() + '\n') F = list(SQL_history.fields) FE = SQL_history.fields_enum F[FE.architecture] = 'arch' del F[FE.forensic] F.insert(FE.delta_time, 'percent') del F[FE.distribution] W('<table class="one_day_work"><tr>') for j in F: W('<th>' + j.replace('_', ' ') + '</th>') W('</tr>\n') count = 0 for x in s.iterate_since(since): count += 1 x = list(x) if x[FE.delta]: x[FE.delta] = '<a href="/%s">delta</a>' % urllib_quote(x[FE.delta]) if x[FE.new_size] and x[FE.delta_size]: percent = ('%.1f%%' % (100. * x[FE.delta_size] / x[FE.new_size])) else: percent = '--' x[FE.ctime] = time.ctime(x[FE.ctime]) del x[FE.forensic] x.insert(FE.delta_time, percent) del x[FE.distribution] x = [('%.3f' % j) if isinstance(j, float) else j for j in x] x = ['' if (j == None) else j for j in x] W('<tr>') for j in x: W('<td>' + str(j) + '</td>') W('</tr>\n') if (count % 40) == 0: W('<tr>') for j in F: W('<th>' + j.replace('_', ' ') + '</th>') W('</tr>\n') W('</table></body></html>\n')
def search(self,callback,searchkey,_page=1,_per_page=30): """wct.search.webcams Search the webcams by the given query. Arguments devid (required) Your developer ID. If you do not have one, please signup for a developer ID. query (required) The query to search for. per_page (optional) Number of comments to return per page. If this argument is omitted, it defaults to 10. The maximum allowed value is 50. page (optional) The page of results to return. If this argument is omitted, it defaults to 1. """ cb = lambda raw: self.searchCB(raw,callback) self.get("wct.search.webcams",cb,None,query=urllib_quote(searchkey),page=_page,per_page=_per_page)
def getWeatherData(self, degreetype, locationcode, city, callback, callbackShowIcon, callbackAllIconsDownloaded=None): self.initialize() language = config.osd.language.value.replace('_', '-') if language == 'en-EN': language = 'en-US' elif language == 'no-NO': language = 'nn-NO' self.city = city self.callback = callback self.callbackShowIcon = callbackShowIcon self.callbackAllIconsDownloaded = callbackAllIconsDownloaded url = 'http://weather.service.msn.com/data.aspx?src=vista&weadegreetype=%s&culture=%s&wealocations=%s' % ( degreetype, language, urllib_quote(locationcode)) getPage(url).addCallback(self.xmlCallback).addErrback(self.error)
def getWeatherData(self, degreetype, locationcode, city, callback, callbackShowIcon, callbackAllIconsDownloaded=None): self.initialize() language = config.osd.language.value.replace("_", "-") if language == "en-EN": # hack language = "en-US" elif language == "no-NO": # hack language = "nn-NO" self.city = city self.callback = callback self.callbackShowIcon = callbackShowIcon self.callbackAllIconsDownloaded = callbackAllIconsDownloaded url = "http://weather.service.msn.com/data.aspx?src=windows&weadegreetype=%s&culture=%s&wealocations=%s" % ( degreetype, language, urllib_quote(locationcode)) getPage(url).addCallback(self.xmlCallback).addErrback(self.error)
def addServiceToBouquet(self, param): print "[WebComponents.BouquetEditor] addService with param = ", param sBouquetRef = param["sBouquetRef"] if sBouquetRef is None: return (False, "No bouquet given!") url = None if "Url" in param: if param['Url'] is not None: url = urllib_quote(param["Url"]) sRef = None if "sRef" in param: if param["sRef"] is not None: sRef = param["sRef"] if sRef is None: return (False, "No service given!") else: if url is not None: sRef += url sName = None if "Name" in param: if param["Name"] is not None: sName = param["Name"] sRefBefore = eServiceReference() if "sRefBefore" in param: if param["sRefBefore"] is not None: sRefBefore = eServiceReference(param["sRefBefore"]) bouquetRef = eServiceReference(sBouquetRef) mutableBouquetList = self.getMutableList(bouquetRef) if mutableBouquetList is not None: ref = eServiceReference(sRef) if sName: ref.setName(sName) if not mutableBouquetList.addService(ref, sRefBefore): mutableBouquetList.flushChanges() self.setRoot(sBouquetRef) return (True, "Service %s added." % self.getName(ref)) else: bouquetName = self.getName(bouquetRef) return (False, "Service %s already exists in bouquet %s." % (self.getName(ref), bouquetName)) return (False, "This service can not be added.")
def link_file(self, file, params, file_url=None): """ Parameters ---------- file : dict Of form {'file' : Buffered Reader for file} The buffered reader was made by opening the pdf using open(). params : dict Includes the following: 'title' = paper title 'id' = ID of the document to which the file will be attached (optional) '_return_type': return type of API.make_post_request (json, object, raw, or response) Returns ------- Object specified by params['_return_type']. Generally models.LinkedFile object """ base_url = 'https://api.mendeley.com' url = base_url + '/files' # Extract info from params title = params['title'] doc_id = params['id'] object_fh = models.File # Get rid of spaces in filename filename = urllib_quote(title) + '.pdf' filename = filename.replace('/', '%2F') headers = dict() headers['Content-Type'] = 'application/pdf' headers['Content-Disposition'] = 'attachment; filename=%s' % filename headers['Link'] = '<' + base_url + '/documents/' + doc_id + '>; rel="document"' API.make_post_request(API(), url, object_fh, params, headers=headers, files=file)
def citation_to_doi(citation): """ Uses a search to CrossRef.org to retrive paper DOI. Parameters ---------- citation : str Full journal article citation. Example: Senís, Elena, et al. "CRISPR/Cas9‐mediated genome engineering: An adeno‐associated viral (AAV) vector toolbox. Biotechnology journal 9.11 (2014): 1402-1412. Returns ------- _CitationDOISearchResponse Usage Notes ----------- This is relatively slow. I'm not sure how much of the slowness is due to parsing into parts versus using those parts to find a DOI (of course those could be done together). """ citation = urllib_quote(citation) # Search for citation on CrossRef.org to try to get a DOI link #TODO: Where is this documented???? #Examples: https://search.crossref.org/help/search # #Inserting /dois as an endpoint converts results from html to JSON api_search_url = 'http://search.crossref.org/dois?q=' + citation json_data = requests.get(api_search_url).json() #Multiple responses are possible. Note we might not have anything: best_match_data = json_data[0] return _CitationDOISearchResponse(best_match_data)
class OriginalFileResource(WebAPIResource): """Provides the unpatched file corresponding to a file diff.""" name = 'original_file' singleton = True allowed_item_mimetypes = ['text/plain'] @webapi_check_login_required def get(self, request, *args, **kwargs): """Returns the original unpatched file. The file is returned as :mimetype:`text/plain` and is the original file before applying a patch. """ try: filediff = resources.filediff.get_object(request, *args, **kwargs) except ObjectDoesNotExist: return DOES_NOT_EXIST if filediff.is_new: return DOES_NOT_EXIST try: orig_file = get_original_file(filediff, request=request) except Exception, e: logging.error("Error retrieving original file: %s", e, exc_info=1, request=request) return FILE_RETRIEVAL_ERROR resp = HttpResponse(orig_file, mimetype='text/plain') filename = urllib_quote(filediff.source_file) resp['Content-Disposition'] = 'inline; filename=%s' % filename set_last_modified(resp, filediff.diffset.timestamp) return resp
def __init__(self, ref_tags, ref_id): """ Parameters: ----------- ref_tags: bs4.element.Tag Html tags as soup of the reference. Information provided is that needed in order to form a citation for the given reference. ref_id: int The id of the reference as ordered in the citing entry. A value of 1 indicates that this object is the first reference in the bibliography. """ # Reference Bibliography Section: #-------------------------------- self.ref_id = ref_id + 1 # Input is 0 indexed self.title = findValue(ref_tags, 'span', 'articleTitle', 'class') authorlist = ref_tags.find_all('span', 'author', 'class') self.authors = [x.text for x in authorlist] # Note: we can also get individual authors if we would like. # # On Wiley, each reference author is given a separate <span> tag with the class 'author' # so individual authors can be extracted # self.publication = findValue(ref_tags, 'span', 'journalTitle', 'class') self.volume = findValue(ref_tags, 'span', 'vol', 'class') self.date = findValue(ref_tags, 'span', 'pubYear', 'class') firstp = findValue(ref_tags, 'span', 'pageFirst', 'class') lastp = findValue(ref_tags, 'span', 'pageLast', 'class') if (firstp is not None) and (lastp is not None): self.pages = firstp + '-' + lastp else: self.pages = None # Reference Meta Section: #------------------------------ self.crossref = None self.pubmed = None self.pubmed_id = None self.doi = None self.citetimes = None self.cas = None self.abstract = None self.pdf_link = None self.ref_references = None # External links (i.e. PubMed, CrossRef, CAS) are kept in a ul tag # Internal links (i.e. direct to abstract, references, etc.) are in a div # Need to check for both links = ref_tags.find('ul', 'externalReferences', 'class') if links is None: links = ref_tags.find('div', 'internalReferences', 'class') # Only proceed if either internal or external references were found if links is not None: links = links.find_all('li') # Check against all possible link options and save links. # href links are appended onto base URL ('http://onlinelibrary.wiley.com') # for link in links: label = link.text.lower() href = link.find('a', href=True)['href'] href = urllib_quote(href) if 'crossref' in label: self.doi = href[href.find('10.'):] # Grab everything starting with '10.' in link if self.doi == -1: self.doi = None self.doi = urllib_unquote(self.doi) # CrossRef link is in the form of _WY_URL/resolve/reference/XREF?id=10.####### self.crossref = _WY_URL + urllib_unquote(href) elif 'pubmed' in label: self.pubmed_id = re.search('[^id=]+$',href).group(0)[1:] # the [1:] is to get rid of leading '=' self.pubmed_id = urllib_unquote(self.pubmed_id) self.pubmed = _WY_URL + urllib_unquote(href) elif 'web ' in label: self.citetimes = re.search('[^: ]+$',label).group(0) elif label in ('cas', 'cas,'): self.cas = _WY_URL + urllib_unquote(href) elif 'abstract' in label: self.abstract = _WY_URL + urllib_unquote(href) elif 'pdf' in label: self.pdf_link = _WY_URL + urllib_unquote(href) elif 'references' in label: self.ref_references = _WY_URL + urllib_unquote(href)
except ObjectDoesNotExist: return DOES_NOT_EXIST if filediff.deleted: return DOES_NOT_EXIST try: orig_file = get_original_file(filediff, request=request) except Exception, e: logging.error("Error retrieving original file: %s", e, exc_info=1, request=request) return FILE_RETRIEVAL_ERROR try: patched_file = get_patched_file(orig_file, filediff, request=request) except Exception, e: logging.error("Error retrieving patched file: %s", e, exc_info=1, request=request) return FILE_RETRIEVAL_ERROR resp = HttpResponse(patched_file, mimetype='text/plain') filename = urllib_quote(filediff.dest_file) resp['Content-Disposition'] = 'inline; filename=%s' % filename set_last_modified(resp, filediff.diffset.timestamp) return resp patched_file_resource = PatchedFileResource()
def quote(value): return urllib_quote(value.encode('utf-8'))
def url(self): """Get the URL of an object.""" object_name = urllib_quote(self.name) return '%s/%s' % (self.container.url, object_name)
def getWeatherData(self, degreetype, locationcode, city, callback, callbackShowIcon, callbackAllIconsDownloaded = None ): self.initialize() self.city = city self.callback = callback self.callbackShowIcon = callbackShowIcon self.callbackAllIconsDownloaded = callbackAllIconsDownloaded u = "+and+u='c'" if degreetype == "F": u = "" url = "http://query.yahooapis.com/v1/public/yql?q=select+*+from+weather.forecast+where+woeid=%s%s&format=json" % (urllib_quote(locationcode), u) getPage(url).addCallback(self.jsonCallback).addErrback(self.error)
def getWeatherData(self, degreetype, locationcode, city, callback, callbackShowIcon, callbackAllIconsDownloaded = None): self.initialize() language = config.osd.language.value.replace('_', '-') if language == 'en-EN': language = 'en-US' elif language == 'no-NO': language = 'nn-NO' self.city = city self.callback = callback self.callbackShowIcon = callbackShowIcon self.callbackAllIconsDownloaded = callbackAllIconsDownloaded url = 'http://weather.service.msn.com/data.aspx?src=vista&weadegreetype=%s&culture=%s&wealocations=%s' % (degreetype, language, urllib_quote(locationcode)) getPage(url).addCallback(self.xmlCallback).addErrback(self.error)
def getWeatherData(self, degreetype, locationcode, city, callback, callbackShowIcon, callbackAllIconsDownloaded = None ): self.initialize() language = config.osd.language.value.replace("_","-") if language == "en-EN": # hack language = "en-US" elif language == "no-NO": # hack language = "nn-NO" self.city = city self.callback = callback self.callbackShowIcon = callbackShowIcon self.callbackAllIconsDownloaded = callbackAllIconsDownloaded url = "http://weather.service.msn.com/data.aspx?src=windows&weadegreetype=%s&culture=%s&wealocations=%s" % (degreetype, language, urllib_quote(locationcode)) getPage(url).addCallback(self.xmlCallback).addErrback(self.error)
def test_coinevo_uri(self): print('Testing coinevo: URI') wallet = Wallet() utf8string = [u'えんしゅう', u'あまやかす'] quoted_utf8string = [ urllib_quote(x.encode('utf8')) for x in utf8string ] ok = False try: res = wallet.make_uri() except: ok = True assert ok ok = False try: res = wallet.make_uri(address='') except: ok = True assert ok ok = False try: res = wallet.make_uri(address='kjshdkj') except: ok = True assert ok for address in [ '42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm', '4BxSHvcgTwu25WooY4BVmgdcKwZu5EksVZSZkDd6ooxSVVqQ4ubxXkhLF6hEqtw96i9cf3cVfLw8UWe95bdDKfRQeYtPwLm1Jiw7AKt2LY', '8AsN91rznfkBGTY8psSNkJBg9SZgxxGGRUhGwRptBhgr5XSQ1XzmA9m8QAnoxydecSh5aLJXdrgXwTDMMZ1AuXsN1EX5Mtm' ]: res = wallet.make_uri(address=address) assert res.uri == 'coinevo:' + address res = wallet.parse_uri(res.uri) assert res.uri.address == address assert res.uri.payment_id == '' assert res.uri.amount == 0 assert res.uri.tx_description == '' assert res.uri.recipient_name == '' assert not 'unknown_parameters' in res or len( res.unknown_parameters) == 0 res = wallet.make_uri(address=address, amount=11000000000) assert res.uri == 'coinevo:' + address + '?tx_amount=0.011' or res.uri == 'coinevo:' + address + '?tx_amount=0.011000000000' res = wallet.parse_uri(res.uri) assert res.uri.address == address assert res.uri.payment_id == '' assert res.uri.amount == 11000000000 assert res.uri.tx_description == '' assert res.uri.recipient_name == '' assert not 'unknown_parameters' in res or len( res.unknown_parameters) == 0 address = '42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm' res = wallet.make_uri(address=address, tx_description=utf8string[0]) assert res.uri == 'coinevo:' + address + '?tx_description=' + quoted_utf8string[ 0] res = wallet.parse_uri(res.uri) assert res.uri.address == address assert res.uri.payment_id == '' assert res.uri.amount == 0 assert res.uri.tx_description == utf8string[0] assert res.uri.recipient_name == '' assert not 'unknown_parameters' in res or len( res.unknown_parameters) == 0 res = wallet.make_uri(address=address, recipient_name=utf8string[0]) assert res.uri == 'coinevo:' + address + '?recipient_name=' + quoted_utf8string[ 0] res = wallet.parse_uri(res.uri) assert res.uri.address == address assert res.uri.payment_id == '' assert res.uri.amount == 0 assert res.uri.tx_description == '' assert res.uri.recipient_name == utf8string[0] assert not 'unknown_parameters' in res or len( res.unknown_parameters) == 0 res = wallet.make_uri(address=address, recipient_name=utf8string[0], tx_description=utf8string[1]) assert res.uri == 'coinevo:' + address + '?recipient_name=' + quoted_utf8string[ 0] + '&tx_description=' + quoted_utf8string[1] res = wallet.parse_uri(res.uri) assert res.uri.address == address assert res.uri.payment_id == '' assert res.uri.amount == 0 assert res.uri.tx_description == utf8string[1] assert res.uri.recipient_name == utf8string[0] assert not 'unknown_parameters' in res or len( res.unknown_parameters) == 0 res = wallet.make_uri(address=address, recipient_name=utf8string[0], tx_description=utf8string[1], amount=1000000000000) assert res.uri == 'coinevo:' + address + '?tx_amount=1.000000000000&recipient_name=' + quoted_utf8string[ 0] + '&tx_description=' + quoted_utf8string[1] res = wallet.parse_uri(res.uri) assert res.uri.address == address assert res.uri.payment_id == '' assert res.uri.amount == 1000000000000 assert res.uri.tx_description == utf8string[1] assert res.uri.recipient_name == utf8string[0] assert not 'unknown_parameters' in res or len( res.unknown_parameters) == 0 res = wallet.make_uri(address=address, recipient_name=utf8string[0], tx_description=utf8string[1], amount=1000000000000, payment_id='1' * 64) assert res.uri == 'coinevo:' + address + '?tx_payment_id=' + '1' * 64 + '&tx_amount=1.000000000000&recipient_name=' + quoted_utf8string[ 0] + '&tx_description=' + quoted_utf8string[1] res = wallet.parse_uri(res.uri) assert res.uri.address == address assert res.uri.payment_id == '1' * 64 assert res.uri.amount == 1000000000000 assert res.uri.tx_description == utf8string[1] assert res.uri.recipient_name == utf8string[0] assert not 'unknown_parameters' in res or len( res.unknown_parameters) == 0 # spaces must be encoded as %20 res = wallet.make_uri(address=address, tx_description=' ' + utf8string[1] + ' ' + utf8string[0] + ' ', amount=1000000000000) assert res.uri == 'coinevo:' + address + '?tx_amount=1.000000000000&tx_description=%20' + quoted_utf8string[ 1] + '%20' + quoted_utf8string[0] + '%20' res = wallet.parse_uri(res.uri) assert res.uri.address == address assert res.uri.payment_id == '' assert res.uri.amount == 1000000000000 assert res.uri.tx_description == ' ' + utf8string[ 1] + ' ' + utf8string[0] + ' ' assert res.uri.recipient_name == '' assert not 'unknown_parameters' in res or len( res.unknown_parameters) == 0 # the example from the docs res = wallet.parse_uri( 'coinevo:46BeWrHpwXmHDpDEUmZBWZfoQpdc6HaERCNmx1pEYL2rAcuwufPN9rXHHtyUA4QVy66qeFQkn6sfK8aHYjA3jk3o1Bv16em?tx_amount=239.39014&tx_description=donation' ) assert res.uri.address == '46BeWrHpwXmHDpDEUmZBWZfoQpdc6HaERCNmx1pEYL2rAcuwufPN9rXHHtyUA4QVy66qeFQkn6sfK8aHYjA3jk3o1Bv16em' assert res.uri.amount == 239390140000000 assert res.uri.tx_description == 'donation' assert res.uri.recipient_name == '' assert res.uri.payment_id == '' assert not 'unknown_parameters' in res or len( res.unknown_parameters) == 0 # malformed/invalid for uri in [ '', ':', 'coinevo', 'notcoinevo:42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm', 'COINEVO:42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm', 'COINEVO::42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm', 'coinevo:', 'coinevo:badaddress', 'coinevo:tx_amount=10', 'coinevo:?tx_amount=10', 'coinevo:42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm?tx_amount=-1', 'coinevo:42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm?tx_amount=1e12', 'coinevo:42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm?tx_amount=+12', 'coinevo:42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm?tx_amount=1+2', 'coinevo:42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm?tx_amount=A', 'coinevo:42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm?tx_amount=0x2', 'coinevo:42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm?tx_amount=222222222222222222222', 'coinevo:42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDn?tx_amount=10', 'coinevo:42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm&', 'coinevo:42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm&tx_amount', 'coinevo:42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm&tx_amount=', 'coinevo:42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm&tx_amount=10=', 'coinevo:42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm&tx_amount=10=&', 'coinevo:42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm&tx_amount=10=&foo=bar', 'coinevo:42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm?tx_amount=10&tx_amount=20', 'coinevo:42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm?tx_payment_id=1111111111111111', 'coinevo:4BxSHvcgTwu25WooY4BVmgdcKwZu5EksVZSZkDd6ooxSVVqQ4ubxXkhLF6hEqtw96i9cf3cVfLw8UWe95bdDKfRQeYtPwLm1Jiw7AKt2LY?tx_payment_id=' + '1' * 64, 'coinevo:9ujeXrjzf7bfeK3KZdCqnYaMwZVFuXemPU8Ubw335rj2FN1CdMiWNyFV3ksEfMFvRp9L9qum5UxkP5rN9aLcPxbH1au4WAB', 'coinevo:5K8mwfjumVseCcQEjNbf59Um6R9NfVUNkHTLhhPCmNvgDLVS88YW5tScnm83rw9mfgYtchtDDTW5jEfMhygi27j1QYphX38hg6m4VMtN29', 'coinevo:7A1Hr63MfgUa8pkWxueD5xBqhQczkusYiCMYMnJGcGmuQxa7aDBxN1G7iCuLCNB3VPeb2TW7U9FdxB27xKkWKfJ8VhUZthF', ]: ok = False try: res = wallet.parse_uri(uri) except: ok = True assert ok, res # unknown parameters but otherwise valid res = wallet.parse_uri('coinevo:' + address + '?tx_amount=239.39014&foo=bar') assert res.uri.address == address assert res.uri.amount == 239390140000000 assert res.unknown_parameters == ['foo=bar'], res res = wallet.parse_uri('coinevo:' + address + '?tx_amount=239.39014&foo=bar&baz=quux') assert res.uri.address == address assert res.uri.amount == 239390140000000 assert res.unknown_parameters == ['foo=bar', 'baz=quux'], res res = wallet.parse_uri('coinevo:' + address + '?tx_amount=239.39014&%20=%20') assert res.uri.address == address assert res.uri.amount == 239390140000000 assert res.unknown_parameters == ['%20=%20'], res res = wallet.parse_uri('coinevo:' + address + '?tx_amount=239.39014&unknown=' + quoted_utf8string[0]) assert res.uri.address == address assert res.uri.amount == 239390140000000 assert res.unknown_parameters == [u'unknown=' + quoted_utf8string[0] ], res
def searchLocation(self): if self.current.city.value != "": url = "http://query.yahooapis.com/v1/public/yql?q=select+*+from+geo.places+where+text='%s'&format=json" % (urllib_quote(self.current.city.value)) getPage(url).addCallback(self.jsonCallback).addErrback(self.error) else: self.session.open(MessageBox, _("You need to enter a valid city name before you can search for the location code."), MessageBox.TYPE_ERROR)
def quote(value, safe='/'): """ Patched version of urllib.quote that encodes utf-8 strings before quoting """ return urllib_quote(ensure_utf8_bytes(value), safe)
def encode_request_string(station): station_url = station.encode("latin-1") station_url = urllib_quote(station_url) return station_url.lower()
def quote(text): # quote url path segments, but leave + and @ intact return urllib_quote(text, '/+@')
def searchLocation(self): if self.current.city.value != "": language = config.osd.language.value.replace("_","-") if language == "en-EN": # hack language = "en-US" elif language == "no-NO": # hack language = "nn-NO" url = "http://weather.service.msn.com/find.aspx?src=vista&outputview=search&weasearchstr=%s&culture=%s" % (urllib_quote(self.current.city.value), language) getPage(url).addCallback(self.xmlCallback).addErrback(self.error) else: self.session.open(MessageBox, _("You need to enter a valid city name before you can search for the location code."), MessageBox.TYPE_ERROR)
def url(self): """Get the URL to access a container.""" region_endpoint = self._manager.get_region_url(self.region.name) container_name = urllib_quote(self.name).replace('/', '%2f') return '%s/%s' % (region_endpoint, container_name)
def searchLocation(self): if self.current.city.value != '': language = config.osd.language.value.replace('_', '-') if language == 'en-EN': language = 'en-US' elif language == 'no-NO': language = 'nn-NO' elif language == 'pl-PL': language = 'pl-PL' url = "http://weather.service.msn.com/find.aspx?src=outlook&outputview=search&weasearchstr=%s&culture=%s" % (urllib_quote(self.current.city.value), language) getPage(url).addCallback(self.xmlCallback).addErrback(self.error) else: self.session.open(MessageBox, _('You need to enter a valid city name before you can search for the location code.'), MessageBox.TYPE_ERROR)