def activate(self, *args, **kwargs): """ Transition: inactive -> active Set github webhook and activate all Presentations. """ try: provider = self.user.social_auth.get(provider="github") except UserSocialAuth.DoesNotExists: logger.error("No social auth provider for Github found on user") return github = Github(provider.access_token) try: repo = github.get_user().get_repo(self.name) except GithubException: logger.error("Could not find repository") return url = URL( scheme="https", host=self.site.domain, path=reverse("qraz:webhook", kwargs={"username": self.user.username, "repository": self.name}), ) try: hook = repo.create_hook( "web", {"url": url.as_string(), "content_type": "json", "secret": self.secret}, events=["push"], active=True, ) except GithubException as excp: logger.error("Could not create webhook: %s", excp) return self.hook = hook.id
def select(self, url): print "SELECTED!!" url = URL(url) field_name = '%s_exact' % self.facet_name url = url.append_query_param('selected_facets', '%s:%s' % (field_name, self.key)) self.selected = True return url
class Segments(object): def __init__(self, base, path, segments, defaults): self.base = PURL(base, path=path) self.segments = OrderedDict(zip(segments, defaults)) def build(self): segments = self.base.path_segments() + tuple(self.segments.values()) url = self.base.path_segments(segments) return url def __str__(self): return self.build().as_string() def _get_segment(self, segment): return self.segments[segment] def _set_segment(self, segment, value): self.segments[segment] = value @classmethod def _segment(cls, segment): return property( fget=lambda x: cls._get_segment(x, segment), fset=lambda x, v: cls._set_segment(x, segment, v), )
def set_param(request=None, url=None, **kwargs): if not request and not url: return '/' url = URL(path=request.path, query=request.META['QUERY_STRING']) if request else URL(url) for k, v in kwargs.items(): url = url.query_param(k, v) return url.as_string()
def __call__(self, request, extra_classes=None): """ Renders the element. """ a = builder.A(request.translate(self.text)) if self.request_method == 'GET': a.attrib['href'] = self.url if self.request_method == 'DELETE': url = URL(self.url).query_param( 'csrf-token', request.new_csrf_token()) a.attrib['ic-delete-from'] = url.as_string() if self.classes or extra_classes: classes = self.classes + (extra_classes or tuple()) a.attrib['class'] = ' '.join(classes) # add the hidden from public hint if needed if self.is_hidden_from_public: # This snippet is duplicated in the hidden-from-public-hint macro! hint = builder.I() hint.attrib['class'] = 'hidden-from-public-hint' hint.attrib['title'] = request.translate( _("This site is hidden from the general public") ) a.append(builder.I(' ')) a.append(hint) for key, value in self.attributes.items(): a.attrib[key] = request.translate(value) return tostring(a)
def get_bib(args): uploaded = load(args.data_file('repos', 'cdstar.json')) fname_to_cdstar = {} for type_ in ['texts', 'docs', 'data']: for hash_, paths in load(args.data_file('repos', type_ + '.json')).items(): if hash_ in uploaded: for path in paths: fname_to_cdstar[path.split('/')[-1]] = uploaded[hash_] for hash_, paths in load(args.data_file('repos', 'edmond.json')).items(): if hash_ in uploaded: for path in paths: fname_to_cdstar[path.split('/')[-1]] = uploaded[hash_] db = Database.from_file(args.data_file('repos', 'Dogon.bib'), lowercase=True) for rec in db: doc = Document(rec) newurls = [] for url in rec.get('url', '').split(';'): if not url.strip(): continue if url.endswith('sequence=1'): newurls.append(url) continue url = URL(url.strip()) if url.host() in ['dogonlanguages.org', 'github.com', '']: fname = url.path().split('/')[-1] doc.files.append((fname, fname_to_cdstar[fname])) else: newurls.append(url.as_string()) doc.rec['url'] = '; '.join(newurls) yield doc
def get_links(self, request): edit_link = URL(request.link(self.submission)) edit_link = edit_link.query_param("edit", "") edit_link = edit_link.query_param("return-to", request.url) return [Link(text=_("Edit submission"), url=edit_link.as_string(), classes=("edit-link",))]
def __init__(self, url, save_dir='tmp'): """ @url: full url of a site @save_dir: dir to save site """ # log self.logger = logger('file', 'sitelog.log', save_dir) self.logger.info('-' * 20) self.logger.info('start') self.logger.info('start func: __init__') self.logger.info('url: %s' % url) save_time = datetime.strftime(datetime.now(), '%Y%m%d%H%M') self.save_time = save_time self.save_dir = os.path.abspath(os.path.join(save_dir, save_time)) # create dir if not exist if not os.path.isdir(self.save_dir): os.makedirs(self.save_dir) self.url = url u = URL(url) # get host like: http://m.sohu.xom self.host = u.scheme() + '://' + u.host() print '%s: saving %s' % (save_time, self.url) self.logger.info('end func: __init__')
def get(self, url, default=NO_DEFAULT, headers=None): """Retrieve a Response object for a given URL. """ headers = headers or {} url = URL(url) row = self.db.execute( select([ responses.c.created, responses.c.host, responses.c.request_url, responses.c.accept, responses.c.url, responses.c.headers, responses.c.content]) .where(and_( responses.c.request_url == url.as_string(), responses.c.accept == headers.get('Accept', ''))) ).fetchone() if not row: log.info('cache miss %s' % url) row = self.add(url, headers) if row is None: if default is NO_DEFAULT: raise KeyError(url) log.info('invalid url %s' % url) return default else: log.info('cache hit %s' % url) return Response(*row)
def cc_link(req, license_url, button="regular"): if license_url == "http://en.wikipedia.org/wiki/Public_domain": license_url = "http://creativecommons.org/publicdomain/zero/1.0/" license_url = URL(license_url) if license_url.host() != "creativecommons.org": return comps = license_url.path().split("/") if len(comps) < 3: return # pragma: no cover known = { "zero": "Public Domain", "by": "Creative Commons Attribution License", "by-nc": "Creative Commons Attribution-NonCommercial License", "by-nc-nd": "Creative Commons Attribution-NonCommercial-NoDerivatives License", "by-nc-sa": "Creative Commons Attribution-NonCommercial-ShareAlike License", "by-nd": "Creative Commons Attribution-NoDerivatives License", "by-sa": "Creative Commons Attribution-ShareAlike License", } if comps[2] not in known: return icon = "cc-" + comps[2] + ("-small" if button == "small" else "") + ".png" img_attrs = dict(alt=known[comps[2]], src=req.static_url("clld:web/static/images/" + icon)) height, width = (15, 80) if button == "small" else (30, 86) img_attrs.update(height=height, width=width) return HTML.a(HTML.img(**img_attrs), href=license_url, rel="license")
def editbar_links(self): if self.request.is_logged_in: edit_url = URL(self.request.link(self.model.event, 'bearbeiten')) edit_url = edit_url.query_param( 'return-to', self.request.link(self.model.event) ) edit_link = Link( text=_("Edit"), url=edit_url.as_string(), classes=('edit-link', ) ) if self.event_deletable(self.model.event): delete_link = DeleteLink( text=_("Delete"), url=self.request.link(self.model.event), confirm=_("Do you really want to delete this event?"), yes_button_text=_("Delete event"), redirect_after=self.events_url ) else: delete_link = DeleteLink( text=_("Delete"), url=self.request.link(self.model.event), confirm=_("This event can't be deleted."), extra_information=_( "To remove this event, go to the ticket and reject it." ) ) return [edit_link, delete_link]
def get_wikipedia(args, name): url = URL( "http://en.wikipedia.org/w/api.php?format=json&action=query&prop=info&inprop=url") def get_result(res): """ { "query": { "pages": { "45724": { "contentmodel": "wikitext", "counter": "", "editurl": "http://en.wikipedia.org/w/index.php?title=Panthera_leo&action=edit", "fullurl": "http://en.wikipedia.org/wiki/Panthera_leo", "lastrevid": 535861485, "length": 71, "ns": 0, "pageid": 45724, "pagelanguage": "en", "redirect": "", "title": "Panthera leo", "touched": "2014-02-27T02:04:39Z" } } } } """ res = res.json().get('query', {}).get('pages', {}) return res.values()[0] if res else {} return get_data(args, name, 'wikipedia', url.query_param('titles', name), get_result)
def cc_link(req, license_url, button='regular'): if license_url == 'https://en.wikipedia.org/wiki/Public_domain': license_url = 'https://creativecommons.org/publicdomain/zero/1.0/' license_url = URL(license_url) if license_url.host() != 'creativecommons.org': return comps = license_url.path().split('/') if len(comps) < 3: return # pragma: no cover known = { 'zero': 'Public Domain', 'by': 'Creative Commons Attribution License', 'by-nc': 'Creative Commons Attribution-NonCommercial License', 'by-nc-nd': 'Creative Commons Attribution-NonCommercial-NoDerivatives License', 'by-nc-sa': 'Creative Commons Attribution-NonCommercial-ShareAlike License', 'by-nd': 'Creative Commons Attribution-NoDerivatives License', 'by-sa': 'Creative Commons Attribution-ShareAlike License'} if comps[2] not in known: return icon = 'cc-' + comps[2] + ('-small' if button == 'small' else '') + '.png' img_attrs = dict( alt=known[comps[2]], src=req.static_url('clld:web/static/images/' + icon)) height, width = (15, 80) if button == 'small' else (30, 86) img_attrs.update(height=height, width=width) return HTML.a(HTML.img(**img_attrs), href=license_url, rel='license')
def maybe_license_link(req, license, **kw): cc_link_ = cc_link(req, license, button=kw.pop('button', 'regular')) if cc_link_: return cc_link_ license_url = URL(license) if license_url.host(): return external_link(license_url, **kw) return license
def url(self): kwargs = { "username": self.repository.user.username, "repository": self.repository.name, "presentation": self.name, } url = URL(scheme="http", host=self.repository.site.domain, path=reverse("qraz:download", kwargs=kwargs)) return url.as_string()
def setUp(self): self.unicode_param = 'значение' # Python 2.6 requires bytes for quote self.urlencoded_param = quote(self.unicode_param.encode('utf8')) url = 'http://www.google.com/blog/article/1?q=' + self.urlencoded_param self.ascii_url = URL.from_string(url.encode('ascii')) # django request.get_full_path() returns url as unicode self.unicode_url = URL.from_string(url)
def info(self, url): """Interface method to be called when processing new images. This method ties together the DataProvider workflow. """ url = URL(url) return self.postprocess( self.info_for_id(self.id_from_url(url, url.host(), url.path_segments())))
def cached_path(args, url, rel): if url.startswith('/'): return url url_ = URL(url) cached = args.data_file('edmond', url_.path_segments()[-1]) if not cached.exists(): fname, headers = urlretrieve(url, '%s' % cached) return str(cached)
def append_path(url, url_path): target = URL(url_path) if url_path.startswith('/'): url = url.path(target.path()) else: url = url.add_path_segment(target.path()) if target.query(): url = url.query(target.query()) return url.as_string()
def get_video_id_from_url(self, video_url): video_url = URL(video_url) if 'youtube' not in video_url.host(): raise DownytError( 'Provided URL is not from YouTube: {}'.format(video_url) ) return video_url.query_param('v')
def handle_pending_submission(self, request): """ Renders a pending submission, takes it's input and allows the user to turn the submission into a complete submission, once all data is valid. Takes the following query parameters for customization:: * ``edit`` no validation is done on the first load if present * ``return-to`` the view redirects to this url once complete if present * ``title`` a custom title (required if external submission) * ``quiet`` no success messages are rendered if present """ collection = FormCollection(request.app.session()) form = request.get_form(self.form_class, data=self.data) form.action = request.link(self) if 'edit' not in request.GET: form.validate() if not request.POST: form.ignore_csrf_error() else: collection.submissions.update(self, form) # these parameters keep between form requests (the rest throw away) for param in {'return-to', 'title', 'quiet'}: if param in request.GET: action = URL(form.action).query_param(param, request.GET[param]) form.action = action.as_string() completable = not form.errors and 'edit' not in request.GET if completable and 'return-to' in request.GET: if 'quiet' not in request.GET: request.success(_("Your changes were saved")) return morepath.redirect(request.GET['return-to']) if 'title' in request.GET: title = request.GET['title'] else: title = self.form.title return { 'layout': FormSubmissionLayout(self, request, title), 'title': title, 'form': form, 'completable': completable, 'edit_link': request.link(self) + '?edit', 'complete_link': request.link(self, 'complete'), 'is_pending': self.state == 'pending', 'readonly': 'readonly' in request.GET, }
def test_facebook_login_url(): facebook_client = FacebookClient(local_host='localhost') facebook_login_url = URL(facebook_client.get_login_uri()) query = facebook_login_url.query_params() callback_url = URL(query['redirect_uri'][0]) func, _args, kwargs = resolve(callback_url.path()) assert func is oauth_callback assert kwargs['service'] == FACEBOOK assert query['scope'][0] == FacebookClient.scope assert query['client_id'][0] == str(FacebookClient.client_id)
def test_facebook_login_url(facebook_client, settings): settings.FACEBOOK_APP_ID = '112233' facebook_login_url = URL(facebook_client.get_login_uri()) query = facebook_login_url.query_params() callback_url = URL(query['redirect_uri'][0]) func, _args, kwargs = resolve(callback_url.path()) assert func is oauth_callback assert kwargs['service'] == FACEBOOK assert query['scope'][0] == facebook_client.scope assert query['client_id'][0] == str(facebook_client.client_id)
def test_google_login_url(): google_client = GoogleClient(local_host='local_host') google_login_url = URL(google_client.get_login_uri()) params = google_login_url.query_params() callback_url = URL(params['redirect_uri'][0]) func, _args, kwargs = resolve(callback_url.path()) assert func is oauth_callback assert kwargs['service'] == GOOGLE assert params['scope'][0] == GoogleClient.scope assert params['client_id'][0] == str(GoogleClient.client_id)
def test_google_login_url(google_client, settings): settings.GOOGLE_CLIENT_ID = '112233' google_login_url = URL(google_client.get_login_uri()) params = google_login_url.query_params() callback_url = URL(params['redirect_uri'][0]) func, _args, kwargs = resolve(callback_url.path()) assert func is oauth_callback assert kwargs['service'] == GOOGLE assert params['scope'][0] == google_client.scope assert params['client_id'][0] == str(google_client.client_id)
def append_path(url, path): target = URL(path) if target.path(): url = add_multi_path_segments(url, target.path()) if target.query(): url = url.query(target.query()) #print("Target: ", target.path) #raise NotImplementedError("target2: %r %r %r" % (target, target.query, url)) return url.as_string()
def test_facebook_login_url(self): """Facebook login url is properly generated""" facebook_client = FacebookClient(local_host='localhost') facebook_login_url = URL(facebook_client.get_login_uri()) query = facebook_login_url.query_params() callback_url = URL(query['redirect_uri'][0]) func, _args, kwargs = resolve(callback_url.path()) self.assertEquals(func, oauth_callback) self.assertEquals(kwargs['service'], FACEBOOK) self.assertEqual(query['scope'][0], FacebookClient.scope) self.assertEqual(query['client_id'][0], FacebookClient.client_id)
def test_google_login_url(self): """Google login url is properly generated""" google_client = GoogleClient(local_host='local_host') google_login_url = URL(google_client.get_login_uri()) params = google_login_url.query_params() callback_url = URL(params['redirect_uri'][0]) func, _args, kwargs = resolve(callback_url.path()) self.assertEquals(func, oauth_callback) self.assertEquals(kwargs['service'], GOOGLE) self.assertIn(params['scope'][0], GoogleClient.scope) self.assertEqual(params['client_id'][0], GoogleClient.client_id)
def get_eol(args, name): """ http://eol.org/api/search/1.0.json?q=Panthera+leo&page=1&exact=true&\ filter_by_taxon_concept_id=&filter_by_hierarchy_entry_id=&filter_by_string= """ url = URL('http://eol.org/api/search/1.0.json?page=1&exact=true') def get_result(res): res = res.json().get('results') return res[0] if res else {} return get_data(args, name, 'eol', url.query_param('q', name), get_result)
def __init__(self, facet, key, doc_count, request_url, selected, formatter=None): self.facet = facet self.key = key self.doc_count = doc_count self.request_url = URL(request_url) self.selected = selected self.show_count = True self.formatter = formatter
def parseJson(self, response): resjson = demjson.decode(demjson.decode(response.text)) for res in resjson: if res['knowledgeType'] == 'VideoKnowledge': item = IlexueItem() siteURL = URL(res['knowledgeUrl']).scheme('http').domain( self.host) item['siteURL'] = siteURL.as_string() item['onclick'] = '' item['pageURL'] = response.request.url item['title'] = res['title'] item['fileName'] = self.base + item['title'] yield Request(url=item['siteURL'], meta={'item1': item}, callback=self.parseVideoAndDocument)
def _isUserAuthenticatedImpl(self, cardId, timeout): userAuthUri = URL(self.apiEndPoint.value).add_path_segment( 'ispersonvalid').add_path_segment(str(cardId)) userReq = requests.get(userAuthUri.as_string(), auth=HTTPBasicAuth(self._apiUser.value, self._apiKey.value), timeout=timeout) userReq.raise_for_status() json = userReq.json() if isinstance(json, list): json = json[0] validUser = json.get('Valid', False) personnelId = int(json.get('PersonnelNumber', 0)) fullName = json.get('FullName', '') return (validUser, personnelId, fullName)
def format_external_link_in_label(url, label=None): label = label or URL(url).domain() return HTML.span(HTML.a(HTML.i('', class_="icon-share icon-white"), label, href=url, style="color: white"), class_="label label-info")
def group_action(booking, action): assert action in ('join', 'leave') if attendees_count == 1 and action == 'leave': traits = ( Intercooler( request_method='POST', redirect_after=request.class_link(BookingCollection) ), ) else: traits = ( Intercooler( request_method='POST', redirect_after=request.link(self) ), ) url = URL(request.link(self, action))\ .query_param('booking_id', booking.id)\ .as_string() return Link( text=(action == 'join' and _("Join Group") or _("Leave Group")), url=layout.csrf_protected_url(url), traits=traits )
def format_links(req, lang): def link(href, label, img, alt=None): return HTML.li( HTML.a( HTML.img(src=req.static_url('glottolog3:static/' + img), height="20", width="20", alt=alt or label), ' ', label, href=href, target="_blank", title=label, )) links = [] if lang.iso_code: for isosite in ISOSite.__subclasses__(): isosite = isosite() links.append(link(*isosite.href_label_img_alt(lang.iso_code))) pss = [ps() for ps in PartnerSite.__subclasses__()] for domain, _links in groupby(lang.jsondata['links'], lambda l: URL(l['url']).domain()): for ps in pss: if ps.match(domain): links.extend([link(*ps.href_label_img_alt(l)) for l in _links]) return HTML.ul(*links, **{'class': "nav nav-tabs nav-stacked"})
def license_name(license_url): if license_url == "http://commons.wikimedia.org/wiki/GNU_Free_Documentation_License": return 'GNU Free Documentation License' if license_url == 'http://en.wikipedia.org/wiki/Public_domain': license_url = 'http://creativecommons.org/publicdomain/zero/1.0/' license_url_ = URL(license_url) if license_url_.host() != 'creativecommons.org': return license_url comps = license_url_.path().split('/') if len(comps) < 3: return license_url return { 'zero': 'Public Domain', }.get(comps[2], '(CC) %s' % comps[2].upper())
def __call__(self, index, row): if index == 0: self.cols = {col: i for i, col in enumerate(row)} return row url = URL(row[self.cols['src']]) try: for filename in url.path_segments(): if filename in self.md: if self.md[filename].get('source_url'): row[self.cols['source']] = self.md[filename]['source_url'] self.count += 1 break except IndexError: pass return row
def assertIsRedirect(self, response, expected_url=None): self.assertTrue( response.status_code in (http_client.FOUND, http_client.MOVED_PERMANENTLY)) if expected_url: location = URL.from_string(response['Location']) self.assertEqual(expected_url, location.path())
def purl(self): """Access the current request's URL. For more convenient URL manipulations, we provide the current request's URL as `purl.URL <http://purl.readthedocs.org/en/latest/#purl.URL>`_ instance. """ return URL(self.url)
def test_url_can_be_created_with_username_only(self): u = URL(scheme='postgres', username='******', host='127.0.0.1', port='5432', path='/db_name') self.assertEqual('postgres://[email protected]:5432/db_name', str(u))
def _getValidUsersByCardIdImpl(self, timeout): validUsersUri = URL( self.apiEndPoint.value).add_path_segment('validpeople') validUsersReq = requests.get(validUsersUri.as_string(), auth=HTTPBasicAuth( self._apiUser.value, self._apiKey.value), timeout=timeout) validUsersReq.raise_for_status() json = validUsersReq.json() return { str(person['CardId']): User(person['PersonnelNumber'], person['CardId'], person['FullName'], person['Valid'], 3600 * 365) for person in json }
def process_onclick(self, item, request, result): # try: # lock.acquire(timeout=60) self.browser.switch_to.window(self.browser.window_handles[0]) handle = self.browser.current_window_handle handles = self.browser.window_handles for newhandle in handles: if newhandle != handle: self.browser.close() # 获取当前窗口句柄(窗口A) self.browser.header_overrides = DEFAULT_REQUEST_HEADERS # self.browser.add_cookie(cookies) # while len(self.browser.window_handles) > 1: if request.url != self.browser.current_url: self.browser.get(request.url) req = self.browser.wait_for_request(request.url, timeout=30) handle = self.browser.current_window_handle # 打开一个新的窗口 self.didOnlicks.append(URL(request.url).path() + '_' + item['onclick']) with open('didOnlicks.data', 'wb') as f: # f.write( pickle.dumps(list) ) pickle.dump(self.didOnlicks, f) if item['id'] != '': self.browser.find_element_by_id(item['id']).click() elif len(item['onclick']) > 0: self.browser.execute_script(item['onclick']) self.browser.wait_for_request(self.browser.last_request.path, timeout=10) for i in range(self.timeout): if len(self.browser.window_handles) > 1 or ( len(self.browser.window_handles) == 1 and len(item['onclick']) <= 0): break time.sleep(1) # self.wait.until((len(self.browser.window_handles) > 1)) # self.browser.find_element_by_id('xx').click() # 获取当前所有窗口句柄(窗口A、B) handles = self.browser.window_handles # 对窗口进行遍历 if len(handles) == 1: result = HtmlResponse(url=request.url, body=self.browser.page_source, request=request, encoding='utf-8', status=200) for newhandle in handles: # 筛选新打开的窗口B if newhandle != handle: # 切换到新打开的窗口B self.browser.switch_to.window(newhandle) self.browser.wait_for_request(self.browser.last_request.path, timeout=10) result = HtmlResponse(url=request.url, body=self.browser.page_source, request=request, encoding='utf-8', status=200) self.browser.close() self.browser.switch_to.window(handle) return result
def source_link(source): label = source host = URL(source).host() if host == 'commons.wikimedia.org': label = 'wikimedia' elif host == 'en.wikipedia.org': label = 'wikipedia' return maybe_external_link(source, label=label)
def by_id(self, id): url = URL(f'odata/Baustellen({int(id)})')\ .query_param('addGisLink', 'True') work = tuple( Roadwork(r) for r in self.client.get(url.as_string()).get('value', ())) if work: return work[0] # secondary lookup is against the subsections.. this probably calls # for an index eventually for r in self.roadwork: for section in r.sections: if section.id == id: return section
def get_camera(self): if self.config.URL: url = URL(self.config.URL) auth = url.username(), url.password() if url.username is None and url.password is None: auth = None url = URL(url.as_string(), username='', password='') return FoscamHTTPCamera(url.as_string(), auth) else: return USBCamera(self.config.DEVICE)
def perform_action(self): self.add_product_to_basket(price=D('6.99')) basket = Basket.objects.all()[0] basket.freeze() url = reverse('paypal-success-response', kwargs={'basket_id': basket.id}) url = URL().path(url)\ .query_param('PayerID', '12345')\ .query_param('token', 'EC-8P797793UC466090M') self.response = self.client.get(str(url), follow=True)
def extract_url(self, response): if len(response.all_url) > 0: get_domain_list = [] get_url_list = [] for url in response.all_url: if not url: continue end_fix = url[-4:len(url)] if '.jpg.png.gif.rar.zip.doc.pdf.css'.find(end_fix) != -1: continue opt = URL(url) url_domain = opt.domain() if not url_domain: url = response.page_prefix + '/' + url url_domain = response.page_domain elif not opt.scheme(): url = 'http://' + url if url_domain.find('eastmoney') == -1: continue response.pipe.get(response.spider_name + 'been_url:' + url) get_domain_list.append(url_domain) get_url_list.append(url) for url_domain in get_domain_list: response.pipe.get(response.spider_name + 'ban_host:' + url_domain) get_urlex_dmexp_list = response.pipe.execute() adv_len = len(get_url_list) if len(get_urlex_dmexp_list) == 0 or len( get_urlex_dmexp_list) != adv_len + len(get_domain_list): return for index in range(len(get_url_list)): url = get_url_list[index] exist_flag = get_urlex_dmexp_list[index] if exist_flag: continue is_ban_host = get_urlex_dmexp_list[index + adv_len] if is_ban_host: continue response.pipe.lpush(self.redis_key, url) response.pipe.execute() return True
async def forward_request(token, request): """ Takes the given token and forwards the request to the associated redirect (must exist in database). Returns a web response with the same status code as the returned by the redirect. If the response was successful, the token is deleted immediately. """ token_bound = request.app.db[token] url = token_bound['url'] payload = copy(dict(request.query)) payload.update(await request.json(loads=quiet_json)) payload['oauth_redirect_secret'] = token_bound['secret'] async with ClientSession() as session: if token_bound['method'] == 'GET': url = URL(url) for key, value in payload.items(): url = url.query_param(key, value) def send_request(): return session.get(url.as_string()) elif token_bound['method'] == 'POST': def send_request(): return session.post(url, data=json.dumps(payload)) elif token_bound['method'] == 'PUT': def send_request(): return session.put(url, data=json.dumps(payload)) async with send_request() as response: if 200 <= response.status < 300: del request.app.db[token] return web.HTTPFound(token_bound['success_url']) else: return web.HTTPFound(token_bound['error_url'])
def test_missing_shipping_address(self): from paypal.express.views import RedirectView with patch.object(RedirectView, 'as_payment_method') as as_payment_method: as_payment_method.return_value = True url = reverse('paypal-redirect') self.add_product_to_basket() response = self.client.get(url) self.assertEqual(reverse('checkout:shipping-address'), URL.from_string(response['Location']).path())
def do_fixup_recursive(self, obj): retval = False if 'href' in obj: _href = URL(obj['href']).scheme(self._base_uri.scheme()).host(self._base_uri.host()).port(self._base_uri.port() if self._base_uri.port() else (443 if self._base_uri.scheme() == 'https' else 80)) obj['href'] = str(_href) retval = True for key in obj: if isinstance(obj[key], list): for item in obj[key]: retval |= self.do_fixup_recursive(item) return retval
def perform_action(self): basket = Mock() basket.id = 1 basket.total_incl_tax = D('200') basket.all_lines = Mock(return_value=[]) basket.offer_discounts = [] basket.voucher_discounts = [] basket.shipping_discounts = [] methods = [Free()] url_str = get_paypal_url(basket, methods) self.url = URL.from_string(url_str)
def latest_posts(self): """Получаем последные статьи """ items = [] response = self.session.get('https://vnru.ru/news.html') if response.status_code != 200: return items self._collect_external_links(response) links_to_download = [] for link in response.html.absolute_links: url = URL(link) if url.path_segment(0) in ['news', 'korotkoj-strokoj' ] and not link.endswith('#comments'): links_to_download.append(link) for link in links_to_download: response = self.session.get(link) if response.status_code == 200: self._collect_external_links(response) if response.html.find('.article', first=True): date = response.html.find('.article__date', first=True).text date = self._format_date(date) article = response.html.find('.article', first=True) statistics = article.find('.article-head', first=True).find('div.icons', first=True) items.append({ 'url': link, 'title': response.html.find('h1', first=True).text, 'text': article.find('.article-text', first=True).text, 'date': date, 'views': statistics.find('div.icon__value')[0].text, 'likes': article.find('.article-share__like', first=True).text, }) return items
def perform_action(self): self.add_product_to_basket(price=D('23.99')) basket = Basket.objects.first() basket.freeze() self.url = reverse('paypal-handle-order', kwargs={'basket_id': basket.id}) url = URL().path(self.url) \ .query_param('PayerID', 'Y8K3PSJYN24D4') \ .query_param('token', 'EC-7F151994RW7618524') self.response = self.client.get(str(url), follow=True) self.order = Order.objects.first()
def download_template(source_uri_base, source_file, dest_dir, dest_filename, is_template, *args, **kwargs): dest_filename = os.path.join(dest_dir, dest_filename if dest_filename else source_file) with open(dest_filename, 'w') as fp: response = requests.get(URL(source_uri_base).add_path_segment(source_file).as_string()) response.raise_for_status() if is_template: content = rm.InlineTemplate(response.text, **kwargs).get_content() else: content = response.text fp.write(content) log.debug('Writing to %s. Content: %s', dest_filename, content) return dest_filename
def Just(name, search_text, url): keyword.add_keyword(name) find = keyword.extract_keywords(search_text) if find == []: print(name + "No FOUND") save(name, "No FOUND") else: Urls = keyword.extract_keywords(url) if Urls == []: u = URL(url) Get_icp(name, u) keyword.remove_keyword(name)
def search_fair_trade_federation(self): org_info = CERT_ORGS["fair trade federation"] url = URL(scheme=WEB_SCHEME, host=org_info["url"], path=org_info["search_path"], query=f"members_search={self.company_name}") source = requests.get(url) page = BeautifulSoup(source.content, 'html.parser') search_results = page.find_all(class_='members-box') for sr in search_results: return True return False
def _sendSessionDetails(self, user, sessionTime, tapsCounters, timeout): sessionUri = URL(self.apiEndPoint.value).add_path_segment('activity') payload = { 'sessionTime': sessionTime.isoformat(), 'personnelNumber': user.personnelId, 'Taps': { tapId: { 'amount': tapsCounters[tapId].volume } for tapId in tapsCounters } } userReq = requests.post(sessionUri.as_string(), auth=HTTPBasicAuth(self._apiUser.value, self._apiKey.value), json=payload, timeout=timeout) userReq.raise_for_status() log.info('Session info for user: %d:%s added with activities: %s', user.personnelId, user.fullName, str([activity['ActivityId'] for activity in userReq.json()])) return True
def get_data(q_link): url = URL(q_link) if url.domain() not in ['quora.com', 'www.quora.com']: return 'error, not quora' url = URL( scheme='https', host='www.quora.com', path=url.path(), query='share=1').as_string() soup = BeautifulSoup(requests.get(url).text) question = {} question['url'] = url question['title'] = soup.find("div", {"class": "question_text_edit"}).text question['topics'] = [topic.text for topic in soup.find_all("div", {"class": "topic_list_item"})] question['details'] = soup.find("div", {"class": "question_details_text"}).text answers = [] divs = soup.find_all("div", {"class": "pagedlist_item"}) try: ans_count = soup.find("div", {"class": "answer_header_text"}).text.strip() count = int(re.match(r'(\d+) Answers', ans_count).groups()[0]) except: return jsonify(question=question, answers=answers) question['answer_count'] = count count = len(divs) - 1 if count < 6 else 6 for i in range(count): one_answer = { 'votes': '-1', 'rank': 0, 'answer': '' } try: author = {} author['name'] = divs[i].find("div", {"class": "answer_user"}).find("span", {"class": "answer_user_wrapper"}).find("a", {"class": "user"}).string author['bio'] = divs[i].find("div", {"class": "answer_user"}).find("span", {"class": "answer_user_wrapper"}).find_all("span", {"class": "rep"})[1].find("span", {"class": "hidden"}).text except: author['name'] = 'Anonymous' author['bio'] = '' one_answer['author'] = author one_answer['votes'] = divs[i].find("span", {"class":"numbers"}).text html_block = divs[i].find("div", {"id": re.compile("(.*)_container")}).contents answer_html = '' for p in range(len(html_block) - 1): answer_html += str(html_block[p]) one_answer['answer_html'] = answer_html one_answer['answer'] = divs[i].find("div", {"class": "answer_content"}).text one_answer['rank'] = i + 1 answers.append(one_answer) return jsonify(question=question, answers=answers)
def wait(self, url): """ 如果最近下载过该网站则延迟下载 """ domain = URL(url).host() last_accessed = self.domains.get(domain) if self.delay > 0 and last_accessed is not None: sleep_secs = self.delay - (datetime.now() - last_accessed).seconds if sleep_secs > 0: time.sleep(sleep_secs) self.domains[domain] = datetime.now() pass