def activate(self, *args, **kwargs):
"""
Transition: inactive -> active
Set github webhook and activate all Presentations.
"""
try:
provider = self.user.social_auth.get(provider="github")
except UserSocialAuth.DoesNotExists:
logger.error("No social auth provider for Github found on user")
return
github = Github(provider.access_token)
try:
repo = github.get_user().get_repo(self.name)
except GithubException:
logger.error("Could not find repository")
return
url = URL(
scheme="https",
host=self.site.domain,
path=reverse("qraz:webhook", kwargs={"username": self.user.username, "repository": self.name}),
)
try:
hook = repo.create_hook(
"web",
{"url": url.as_string(), "content_type": "json", "secret": self.secret},
events=["push"],
active=True,
)
except GithubException as excp:
logger.error("Could not create webhook: %s", excp)
return
self.hook = hook.id
def select(self, url):
print "SELECTED!!"
url = URL(url)
field_name = '%s_exact' % self.facet_name
url = url.append_query_param('selected_facets', '%s:%s' % (field_name, self.key))
self.selected = True
return url
class Segments(object):
def __init__(self, base, path, segments, defaults):
self.base = PURL(base, path=path)
self.segments = OrderedDict(zip(segments, defaults))
def build(self):
segments = self.base.path_segments() + tuple(self.segments.values())
url = self.base.path_segments(segments)
return url
def __str__(self):
return self.build().as_string()
def _get_segment(self, segment):
return self.segments[segment]
def _set_segment(self, segment, value):
self.segments[segment] = value
@classmethod
def _segment(cls, segment):
return property(
fget=lambda x: cls._get_segment(x, segment),
fset=lambda x, v: cls._set_segment(x, segment, v),
)
def set_param(request=None, url=None, **kwargs):
if not request and not url:
return '/'
url = URL(path=request.path, query=request.META['QUERY_STRING']) if request else URL(url)
for k, v in kwargs.items():
url = url.query_param(k, v)
return url.as_string()
def __call__(self, request, extra_classes=None):
""" Renders the element. """
a = builder.A(request.translate(self.text))
if self.request_method == 'GET':
a.attrib['href'] = self.url
if self.request_method == 'DELETE':
url = URL(self.url).query_param(
'csrf-token', request.new_csrf_token())
a.attrib['ic-delete-from'] = url.as_string()
if self.classes or extra_classes:
classes = self.classes + (extra_classes or tuple())
a.attrib['class'] = ' '.join(classes)
# add the hidden from public hint if needed
if self.is_hidden_from_public:
# This snippet is duplicated in the hidden-from-public-hint macro!
hint = builder.I()
hint.attrib['class'] = 'hidden-from-public-hint'
hint.attrib['title'] = request.translate(
_("This site is hidden from the general public")
)
a.append(builder.I(' '))
a.append(hint)
for key, value in self.attributes.items():
a.attrib[key] = request.translate(value)
return tostring(a)
def get_bib(args):
uploaded = load(args.data_file('repos', 'cdstar.json'))
fname_to_cdstar = {}
for type_ in ['texts', 'docs', 'data']:
for hash_, paths in load(args.data_file('repos', type_ + '.json')).items():
if hash_ in uploaded:
for path in paths:
fname_to_cdstar[path.split('/')[-1]] = uploaded[hash_]
for hash_, paths in load(args.data_file('repos', 'edmond.json')).items():
if hash_ in uploaded:
for path in paths:
fname_to_cdstar[path.split('/')[-1]] = uploaded[hash_]
db = Database.from_file(args.data_file('repos', 'Dogon.bib'), lowercase=True)
for rec in db:
doc = Document(rec)
newurls = []
for url in rec.get('url', '').split(';'):
if not url.strip():
continue
if url.endswith('sequence=1'):
newurls.append(url)
continue
url = URL(url.strip())
if url.host() in ['dogonlanguages.org', 'github.com', '']:
fname = url.path().split('/')[-1]
doc.files.append((fname, fname_to_cdstar[fname]))
else:
newurls.append(url.as_string())
doc.rec['url'] = '; '.join(newurls)
yield doc
def get_links(self, request):
edit_link = URL(request.link(self.submission))
edit_link = edit_link.query_param("edit", "")
edit_link = edit_link.query_param("return-to", request.url)
return [Link(text=_("Edit submission"), url=edit_link.as_string(), classes=("edit-link",))]
def __init__(self, url, save_dir='tmp'):
"""
@url: full url of a site
@save_dir: dir to save site
"""
# log
self.logger = logger('file', 'sitelog.log', save_dir)
self.logger.info('-' * 20)
self.logger.info('start')
self.logger.info('start func: __init__')
self.logger.info('url: %s' % url)
save_time = datetime.strftime(datetime.now(), '%Y%m%d%H%M')
self.save_time = save_time
self.save_dir = os.path.abspath(os.path.join(save_dir, save_time))
# create dir if not exist
if not os.path.isdir(self.save_dir):
os.makedirs(self.save_dir)
self.url = url
u = URL(url)
# get host like: http://m.sohu.xom
self.host = u.scheme() + '://' + u.host()
print '%s: saving %s' % (save_time, self.url)
self.logger.info('end func: __init__')
def get(self, url, default=NO_DEFAULT, headers=None):
"""Retrieve a Response object for a given URL.
"""
headers = headers or {}
url = URL(url)
row = self.db.execute(
select([
responses.c.created,
responses.c.host,
responses.c.request_url,
responses.c.accept,
responses.c.url,
responses.c.headers,
responses.c.content])
.where(and_(
responses.c.request_url == url.as_string(),
responses.c.accept == headers.get('Accept', '')))
).fetchone()
if not row:
log.info('cache miss %s' % url)
row = self.add(url, headers)
if row is None:
if default is NO_DEFAULT:
raise KeyError(url)
log.info('invalid url %s' % url)
return default
else:
log.info('cache hit %s' % url)
return Response(*row)
def cc_link(req, license_url, button="regular"):
if license_url == "http://en.wikipedia.org/wiki/Public_domain":
license_url = "http://creativecommons.org/publicdomain/zero/1.0/"
license_url = URL(license_url)
if license_url.host() != "creativecommons.org":
return
comps = license_url.path().split("/")
if len(comps) < 3:
return # pragma: no cover
known = {
"zero": "Public Domain",
"by": "Creative Commons Attribution License",
"by-nc": "Creative Commons Attribution-NonCommercial License",
"by-nc-nd": "Creative Commons Attribution-NonCommercial-NoDerivatives License",
"by-nc-sa": "Creative Commons Attribution-NonCommercial-ShareAlike License",
"by-nd": "Creative Commons Attribution-NoDerivatives License",
"by-sa": "Creative Commons Attribution-ShareAlike License",
}
if comps[2] not in known:
return
icon = "cc-" + comps[2] + ("-small" if button == "small" else "") + ".png"
img_attrs = dict(alt=known[comps[2]], src=req.static_url("clld:web/static/images/" + icon))
height, width = (15, 80) if button == "small" else (30, 86)
img_attrs.update(height=height, width=width)
return HTML.a(HTML.img(**img_attrs), href=license_url, rel="license")
def editbar_links(self):
if self.request.is_logged_in:
edit_url = URL(self.request.link(self.model.event, 'bearbeiten'))
edit_url = edit_url.query_param(
'return-to', self.request.link(self.model.event)
)
edit_link = Link(
text=_("Edit"),
url=edit_url.as_string(),
classes=('edit-link', )
)
if self.event_deletable(self.model.event):
delete_link = DeleteLink(
text=_("Delete"),
url=self.request.link(self.model.event),
confirm=_("Do you really want to delete this event?"),
yes_button_text=_("Delete event"),
redirect_after=self.events_url
)
else:
delete_link = DeleteLink(
text=_("Delete"),
url=self.request.link(self.model.event),
confirm=_("This event can't be deleted."),
extra_information=_(
"To remove this event, go to the ticket and reject it."
)
)
return [edit_link, delete_link]
def get_wikipedia(args, name):
url = URL(
"http://en.wikipedia.org/w/api.php?format=json&action=query&prop=info&inprop=url")
def get_result(res):
"""
{
"query": {
"pages": {
"45724": {
"contentmodel": "wikitext",
"counter": "",
"editurl":
"http://en.wikipedia.org/w/index.php?title=Panthera_leo&action=edit",
"fullurl": "http://en.wikipedia.org/wiki/Panthera_leo",
"lastrevid": 535861485,
"length": 71,
"ns": 0,
"pageid": 45724,
"pagelanguage": "en",
"redirect": "",
"title": "Panthera leo",
"touched": "2014-02-27T02:04:39Z"
}
}
}
}
"""
res = res.json().get('query', {}).get('pages', {})
return res.values()[0] if res else {}
return get_data(args, name, 'wikipedia', url.query_param('titles', name), get_result)
def cc_link(req, license_url, button='regular'):
if license_url == 'https://en.wikipedia.org/wiki/Public_domain':
license_url = 'https://creativecommons.org/publicdomain/zero/1.0/'
license_url = URL(license_url)
if license_url.host() != 'creativecommons.org':
return
comps = license_url.path().split('/')
if len(comps) < 3:
return # pragma: no cover
known = {
'zero': 'Public Domain',
'by': 'Creative Commons Attribution License',
'by-nc': 'Creative Commons Attribution-NonCommercial License',
'by-nc-nd': 'Creative Commons Attribution-NonCommercial-NoDerivatives License',
'by-nc-sa': 'Creative Commons Attribution-NonCommercial-ShareAlike License',
'by-nd': 'Creative Commons Attribution-NoDerivatives License',
'by-sa': 'Creative Commons Attribution-ShareAlike License'}
if comps[2] not in known:
return
icon = 'cc-' + comps[2] + ('-small' if button == 'small' else '') + '.png'
img_attrs = dict(
alt=known[comps[2]],
src=req.static_url('clld:web/static/images/' + icon))
height, width = (15, 80) if button == 'small' else (30, 86)
img_attrs.update(height=height, width=width)
return HTML.a(HTML.img(**img_attrs), href=license_url, rel='license')
def maybe_license_link(req, license, **kw):
cc_link_ = cc_link(req, license, button=kw.pop('button', 'regular'))
if cc_link_:
return cc_link_
license_url = URL(license)
if license_url.host():
return external_link(license_url, **kw)
return license
def url(self):
kwargs = {
"username": self.repository.user.username,
"repository": self.repository.name,
"presentation": self.name,
}
url = URL(scheme="http", host=self.repository.site.domain, path=reverse("qraz:download", kwargs=kwargs))
return url.as_string()
def setUp(self):
self.unicode_param = 'значение'
# Python 2.6 requires bytes for quote
self.urlencoded_param = quote(self.unicode_param.encode('utf8'))
url = 'http://www.google.com/blog/article/1?q=' + self.urlencoded_param
self.ascii_url = URL.from_string(url.encode('ascii'))
# django request.get_full_path() returns url as unicode
self.unicode_url = URL.from_string(url)
def info(self, url):
"""Interface method to be called when processing new images.
This method ties together the DataProvider workflow.
"""
url = URL(url)
return self.postprocess(
self.info_for_id(self.id_from_url(url, url.host(), url.path_segments())))
def cached_path(args, url, rel):
if url.startswith('/'):
return url
url_ = URL(url)
cached = args.data_file('edmond', url_.path_segments()[-1])
if not cached.exists():
fname, headers = urlretrieve(url, '%s' % cached)
return str(cached)
def append_path(url, url_path):
target = URL(url_path)
if url_path.startswith('/'):
url = url.path(target.path())
else:
url = url.add_path_segment(target.path())
if target.query():
url = url.query(target.query())
return url.as_string()
def get_video_id_from_url(self, video_url):
video_url = URL(video_url)
if 'youtube' not in video_url.host():
raise DownytError(
'Provided URL is not from YouTube: {}'.format(video_url)
)
return video_url.query_param('v')
def handle_pending_submission(self, request):
""" Renders a pending submission, takes it's input and allows the
user to turn the submission into a complete submission, once all data
is valid.
Takes the following query parameters for customization::
* ``edit`` no validation is done on the first load if present
* ``return-to`` the view redirects to this url once complete if present
* ``title`` a custom title (required if external submission)
* ``quiet`` no success messages are rendered if present
"""
collection = FormCollection(request.app.session())
form = request.get_form(self.form_class, data=self.data)
form.action = request.link(self)
if 'edit' not in request.GET:
form.validate()
if not request.POST:
form.ignore_csrf_error()
else:
collection.submissions.update(self, form)
# these parameters keep between form requests (the rest throw away)
for param in {'return-to', 'title', 'quiet'}:
if param in request.GET:
action = URL(form.action).query_param(param, request.GET[param])
form.action = action.as_string()
completable = not form.errors and 'edit' not in request.GET
if completable and 'return-to' in request.GET:
if 'quiet' not in request.GET:
request.success(_("Your changes were saved"))
return morepath.redirect(request.GET['return-to'])
if 'title' in request.GET:
title = request.GET['title']
else:
title = self.form.title
return {
'layout': FormSubmissionLayout(self, request, title),
'title': title,
'form': form,
'completable': completable,
'edit_link': request.link(self) + '?edit',
'complete_link': request.link(self, 'complete'),
'is_pending': self.state == 'pending',
'readonly': 'readonly' in request.GET,
}
def test_facebook_login_url():
facebook_client = FacebookClient(local_host='localhost')
facebook_login_url = URL(facebook_client.get_login_uri())
query = facebook_login_url.query_params()
callback_url = URL(query['redirect_uri'][0])
func, _args, kwargs = resolve(callback_url.path())
assert func is oauth_callback
assert kwargs['service'] == FACEBOOK
assert query['scope'][0] == FacebookClient.scope
assert query['client_id'][0] == str(FacebookClient.client_id)
def test_facebook_login_url(facebook_client, settings):
settings.FACEBOOK_APP_ID = '112233'
facebook_login_url = URL(facebook_client.get_login_uri())
query = facebook_login_url.query_params()
callback_url = URL(query['redirect_uri'][0])
func, _args, kwargs = resolve(callback_url.path())
assert func is oauth_callback
assert kwargs['service'] == FACEBOOK
assert query['scope'][0] == facebook_client.scope
assert query['client_id'][0] == str(facebook_client.client_id)
def test_google_login_url():
google_client = GoogleClient(local_host='local_host')
google_login_url = URL(google_client.get_login_uri())
params = google_login_url.query_params()
callback_url = URL(params['redirect_uri'][0])
func, _args, kwargs = resolve(callback_url.path())
assert func is oauth_callback
assert kwargs['service'] == GOOGLE
assert params['scope'][0] == GoogleClient.scope
assert params['client_id'][0] == str(GoogleClient.client_id)
def test_google_login_url(google_client, settings):
settings.GOOGLE_CLIENT_ID = '112233'
google_login_url = URL(google_client.get_login_uri())
params = google_login_url.query_params()
callback_url = URL(params['redirect_uri'][0])
func, _args, kwargs = resolve(callback_url.path())
assert func is oauth_callback
assert kwargs['service'] == GOOGLE
assert params['scope'][0] == google_client.scope
assert params['client_id'][0] == str(google_client.client_id)
def append_path(url, path):
target = URL(path)
if target.path():
url = add_multi_path_segments(url, target.path())
if target.query():
url = url.query(target.query())
#print("Target: ", target.path)
#raise NotImplementedError("target2: %r %r %r" % (target, target.query, url))
return url.as_string()
def test_facebook_login_url(self):
"""Facebook login url is properly generated"""
facebook_client = FacebookClient(local_host='localhost')
facebook_login_url = URL(facebook_client.get_login_uri())
query = facebook_login_url.query_params()
callback_url = URL(query['redirect_uri'][0])
func, _args, kwargs = resolve(callback_url.path())
self.assertEquals(func, oauth_callback)
self.assertEquals(kwargs['service'], FACEBOOK)
self.assertEqual(query['scope'][0], FacebookClient.scope)
self.assertEqual(query['client_id'][0], FacebookClient.client_id)
def test_google_login_url(self):
"""Google login url is properly generated"""
google_client = GoogleClient(local_host='local_host')
google_login_url = URL(google_client.get_login_uri())
params = google_login_url.query_params()
callback_url = URL(params['redirect_uri'][0])
func, _args, kwargs = resolve(callback_url.path())
self.assertEquals(func, oauth_callback)
self.assertEquals(kwargs['service'], GOOGLE)
self.assertIn(params['scope'][0], GoogleClient.scope)
self.assertEqual(params['client_id'][0], GoogleClient.client_id)
def get_eol(args, name):
"""
http://eol.org/api/search/1.0.json?q=Panthera+leo&page=1&exact=true&\
filter_by_taxon_concept_id=&filter_by_hierarchy_entry_id=&filter_by_string=
"""
url = URL('http://eol.org/api/search/1.0.json?page=1&exact=true')
def get_result(res):
res = res.json().get('results')
return res[0] if res else {}
return get_data(args, name, 'eol', url.query_param('q', name), get_result)
def __init__(self,
facet,
key,
doc_count,
request_url,
selected,
formatter=None):
self.facet = facet
self.key = key
self.doc_count = doc_count
self.request_url = URL(request_url)
self.selected = selected
self.show_count = True
self.formatter = formatter
def parseJson(self, response):
resjson = demjson.decode(demjson.decode(response.text))
for res in resjson:
if res['knowledgeType'] == 'VideoKnowledge':
item = IlexueItem()
siteURL = URL(res['knowledgeUrl']).scheme('http').domain(
self.host)
item['siteURL'] = siteURL.as_string()
item['onclick'] = ''
item['pageURL'] = response.request.url
item['title'] = res['title']
item['fileName'] = self.base + item['title']
yield Request(url=item['siteURL'],
meta={'item1': item},
callback=self.parseVideoAndDocument)
def _isUserAuthenticatedImpl(self, cardId, timeout):
userAuthUri = URL(self.apiEndPoint.value).add_path_segment(
'ispersonvalid').add_path_segment(str(cardId))
userReq = requests.get(userAuthUri.as_string(),
auth=HTTPBasicAuth(self._apiUser.value,
self._apiKey.value),
timeout=timeout)
userReq.raise_for_status()
json = userReq.json()
if isinstance(json, list):
json = json[0]
validUser = json.get('Valid', False)
personnelId = int(json.get('PersonnelNumber', 0))
fullName = json.get('FullName', '')
return (validUser, personnelId, fullName)
def format_external_link_in_label(url, label=None):
label = label or URL(url).domain()
return HTML.span(HTML.a(HTML.i('', class_="icon-share icon-white"),
label,
href=url,
style="color: white"),
class_="label label-info")
def group_action(booking, action):
assert action in ('join', 'leave')
if attendees_count == 1 and action == 'leave':
traits = (
Intercooler(
request_method='POST',
redirect_after=request.class_link(BookingCollection)
),
)
else:
traits = (
Intercooler(
request_method='POST',
redirect_after=request.link(self)
),
)
url = URL(request.link(self, action))\
.query_param('booking_id', booking.id)\
.as_string()
return Link(
text=(action == 'join' and _("Join Group") or _("Leave Group")),
url=layout.csrf_protected_url(url),
traits=traits
)
def format_links(req, lang):
def link(href, label, img, alt=None):
return HTML.li(
HTML.a(
HTML.img(src=req.static_url('glottolog3:static/' + img),
height="20",
width="20",
alt=alt or label),
' ',
label,
href=href,
target="_blank",
title=label,
))
links = []
if lang.iso_code:
for isosite in ISOSite.__subclasses__():
isosite = isosite()
links.append(link(*isosite.href_label_img_alt(lang.iso_code)))
pss = [ps() for ps in PartnerSite.__subclasses__()]
for domain, _links in groupby(lang.jsondata['links'],
lambda l: URL(l['url']).domain()):
for ps in pss:
if ps.match(domain):
links.extend([link(*ps.href_label_img_alt(l)) for l in _links])
return HTML.ul(*links, **{'class': "nav nav-tabs nav-stacked"})
def license_name(license_url):
if license_url == "http://commons.wikimedia.org/wiki/GNU_Free_Documentation_License":
return 'GNU Free Documentation License'
if license_url == 'http://en.wikipedia.org/wiki/Public_domain':
license_url = 'http://creativecommons.org/publicdomain/zero/1.0/'
license_url_ = URL(license_url)
if license_url_.host() != 'creativecommons.org':
return license_url
comps = license_url_.path().split('/')
if len(comps) < 3:
return license_url
return {
'zero': 'Public Domain',
}.get(comps[2], '(CC) %s' % comps[2].upper())
def __call__(self, index, row):
if index == 0:
self.cols = {col: i for i, col in enumerate(row)}
return row
url = URL(row[self.cols['src']])
try:
for filename in url.path_segments():
if filename in self.md:
if self.md[filename].get('source_url'):
row[self.cols['source']] = self.md[filename]['source_url']
self.count += 1
break
except IndexError:
pass
return row
def assertIsRedirect(self, response, expected_url=None):
self.assertTrue(
response.status_code in (http_client.FOUND,
http_client.MOVED_PERMANENTLY))
if expected_url:
location = URL.from_string(response['Location'])
self.assertEqual(expected_url, location.path())
def purl(self):
"""Access the current request's URL.
For more convenient URL manipulations, we provide the current request's URL
as `purl.URL <http://purl.readthedocs.org/en/latest/#purl.URL>`_ instance.
"""
return URL(self.url)
def test_url_can_be_created_with_username_only(self):
u = URL(scheme='postgres',
username='******',
host='127.0.0.1',
port='5432',
path='/db_name')
self.assertEqual('postgres://[email protected]:5432/db_name', str(u))
def _getValidUsersByCardIdImpl(self, timeout):
validUsersUri = URL(
self.apiEndPoint.value).add_path_segment('validpeople')
validUsersReq = requests.get(validUsersUri.as_string(),
auth=HTTPBasicAuth(
self._apiUser.value,
self._apiKey.value),
timeout=timeout)
validUsersReq.raise_for_status()
json = validUsersReq.json()
return {
str(person['CardId']):
User(person['PersonnelNumber'], person['CardId'],
person['FullName'], person['Valid'], 3600 * 365)
for person in json
}
def process_onclick(self, item, request, result):
# try:
# lock.acquire(timeout=60)
self.browser.switch_to.window(self.browser.window_handles[0])
handle = self.browser.current_window_handle
handles = self.browser.window_handles
for newhandle in handles:
if newhandle != handle:
self.browser.close()
# 获取当前窗口句柄（窗口A）
self.browser.header_overrides = DEFAULT_REQUEST_HEADERS
# self.browser.add_cookie(cookies)
# while len(self.browser.window_handles) > 1:
if request.url != self.browser.current_url:
self.browser.get(request.url)
req = self.browser.wait_for_request(request.url, timeout=30)
handle = self.browser.current_window_handle
# 打开一个新的窗口
self.didOnlicks.append(URL(request.url).path() + '_' + item['onclick'])
with open('didOnlicks.data', 'wb') as f:
# f.write( pickle.dumps(list) )
pickle.dump(self.didOnlicks, f)
if item['id'] != '':
self.browser.find_element_by_id(item['id']).click()
elif len(item['onclick']) > 0:
self.browser.execute_script(item['onclick'])
self.browser.wait_for_request(self.browser.last_request.path, timeout=10)
for i in range(self.timeout):
if len(self.browser.window_handles) > 1 or (
len(self.browser.window_handles) == 1
and len(item['onclick']) <= 0):
break
time.sleep(1)
# self.wait.until((len(self.browser.window_handles) > 1))
# self.browser.find_element_by_id('xx').click()
# 获取当前所有窗口句柄（窗口A、B）
handles = self.browser.window_handles
# 对窗口进行遍历
if len(handles) == 1:
result = HtmlResponse(url=request.url,
body=self.browser.page_source,
request=request,
encoding='utf-8',
status=200)
for newhandle in handles:
# 筛选新打开的窗口B
if newhandle != handle:
# 切换到新打开的窗口B
self.browser.switch_to.window(newhandle)
self.browser.wait_for_request(self.browser.last_request.path,
timeout=10)
result = HtmlResponse(url=request.url,
body=self.browser.page_source,
request=request,
encoding='utf-8',
status=200)
self.browser.close()
self.browser.switch_to.window(handle)
return result
def source_link(source):
label = source
host = URL(source).host()
if host == 'commons.wikimedia.org':
label = 'wikimedia'
elif host == 'en.wikipedia.org':
label = 'wikipedia'
return maybe_external_link(source, label=label)
def by_id(self, id):
url = URL(f'odata/Baustellen({int(id)})')\
.query_param('addGisLink', 'True')
work = tuple(
Roadwork(r)
for r in self.client.get(url.as_string()).get('value', ()))
if work:
return work[0]
# secondary lookup is against the subsections.. this probably calls
# for an index eventually
for r in self.roadwork:
for section in r.sections:
if section.id == id:
return section
def get_camera(self):
if self.config.URL:
url = URL(self.config.URL)
auth = url.username(), url.password()
if url.username is None and url.password is None:
auth = None
url = URL(url.as_string(), username='', password='')
return FoscamHTTPCamera(url.as_string(), auth)
else:
return USBCamera(self.config.DEVICE)
def perform_action(self):
self.add_product_to_basket(price=D('6.99'))
basket = Basket.objects.all()[0]
basket.freeze()
url = reverse('paypal-success-response', kwargs={'basket_id': basket.id})
url = URL().path(url)\
.query_param('PayerID', '12345')\
.query_param('token', 'EC-8P797793UC466090M')
self.response = self.client.get(str(url), follow=True)
def extract_url(self, response):
if len(response.all_url) > 0:
get_domain_list = []
get_url_list = []
for url in response.all_url:
if not url:
continue
end_fix = url[-4:len(url)]
if '.jpg.png.gif.rar.zip.doc.pdf.css'.find(end_fix) != -1:
continue
opt = URL(url)
url_domain = opt.domain()
if not url_domain:
url = response.page_prefix + '/' + url
url_domain = response.page_domain
elif not opt.scheme():
url = 'http://' + url
if url_domain.find('eastmoney') == -1:
continue
response.pipe.get(response.spider_name + 'been_url:' + url)
get_domain_list.append(url_domain)
get_url_list.append(url)
for url_domain in get_domain_list:
response.pipe.get(response.spider_name + 'ban_host:' +
url_domain)
get_urlex_dmexp_list = response.pipe.execute()
adv_len = len(get_url_list)
if len(get_urlex_dmexp_list) == 0 or len(
get_urlex_dmexp_list) != adv_len + len(get_domain_list):
return
for index in range(len(get_url_list)):
url = get_url_list[index]
exist_flag = get_urlex_dmexp_list[index]
if exist_flag:
continue
is_ban_host = get_urlex_dmexp_list[index + adv_len]
if is_ban_host:
continue
response.pipe.lpush(self.redis_key, url)
response.pipe.execute()
return True
async def forward_request(token, request):
""" Takes the given token and forwards the request to the associated
redirect (must exist in database).
Returns a web response with the same status code as the returned by
the redirect.
If the response was successful, the token is deleted immediately.
"""
token_bound = request.app.db[token]
url = token_bound['url']
payload = copy(dict(request.query))
payload.update(await request.json(loads=quiet_json))
payload['oauth_redirect_secret'] = token_bound['secret']
async with ClientSession() as session:
if token_bound['method'] == 'GET':
url = URL(url)
for key, value in payload.items():
url = url.query_param(key, value)
def send_request():
return session.get(url.as_string())
elif token_bound['method'] == 'POST':
def send_request():
return session.post(url, data=json.dumps(payload))
elif token_bound['method'] == 'PUT':
def send_request():
return session.put(url, data=json.dumps(payload))
async with send_request() as response:
if 200 <= response.status < 300:
del request.app.db[token]
return web.HTTPFound(token_bound['success_url'])
else:
return web.HTTPFound(token_bound['error_url'])
def test_missing_shipping_address(self):
from paypal.express.views import RedirectView
with patch.object(RedirectView,
'as_payment_method') as as_payment_method:
as_payment_method.return_value = True
url = reverse('paypal-redirect')
self.add_product_to_basket()
response = self.client.get(url)
self.assertEqual(reverse('checkout:shipping-address'),
URL.from_string(response['Location']).path())
def do_fixup_recursive(self, obj):
retval = False
if 'href' in obj:
_href = URL(obj['href']).scheme(self._base_uri.scheme()).host(self._base_uri.host()).port(self._base_uri.port() if self._base_uri.port() else (443 if self._base_uri.scheme() == 'https' else 80))
obj['href'] = str(_href)
retval = True
for key in obj:
if isinstance(obj[key], list):
for item in obj[key]:
retval |= self.do_fixup_recursive(item)
return retval
def perform_action(self):
basket = Mock()
basket.id = 1
basket.total_incl_tax = D('200')
basket.all_lines = Mock(return_value=[])
basket.offer_discounts = []
basket.voucher_discounts = []
basket.shipping_discounts = []
methods = [Free()]
url_str = get_paypal_url(basket, methods)
self.url = URL.from_string(url_str)
def latest_posts(self):
"""Получаем последные статьи
"""
items = []
response = self.session.get('https://vnru.ru/news.html')
if response.status_code != 200:
return items
self._collect_external_links(response)
links_to_download = []
for link in response.html.absolute_links:
url = URL(link)
if url.path_segment(0) in ['news', 'korotkoj-strokoj'
] and not link.endswith('#comments'):
links_to_download.append(link)
for link in links_to_download:
response = self.session.get(link)
if response.status_code == 200:
self._collect_external_links(response)
if response.html.find('.article', first=True):
date = response.html.find('.article__date',
first=True).text
date = self._format_date(date)
article = response.html.find('.article', first=True)
statistics = article.find('.article-head',
first=True).find('div.icons',
first=True)
items.append({
'url':
link,
'title':
response.html.find('h1', first=True).text,
'text':
article.find('.article-text', first=True).text,
'date':
date,
'views':
statistics.find('div.icon__value')[0].text,
'likes':
article.find('.article-share__like', first=True).text,
})
return items
def perform_action(self):
self.add_product_to_basket(price=D('23.99'))
basket = Basket.objects.first()
basket.freeze()
self.url = reverse('paypal-handle-order',
kwargs={'basket_id': basket.id})
url = URL().path(self.url) \
.query_param('PayerID', 'Y8K3PSJYN24D4') \
.query_param('token', 'EC-7F151994RW7618524')
self.response = self.client.get(str(url), follow=True)
self.order = Order.objects.first()
def download_template(source_uri_base, source_file, dest_dir, dest_filename, is_template, *args, **kwargs):
dest_filename = os.path.join(dest_dir, dest_filename if dest_filename else source_file)
with open(dest_filename, 'w') as fp:
response = requests.get(URL(source_uri_base).add_path_segment(source_file).as_string())
response.raise_for_status()
if is_template:
content = rm.InlineTemplate(response.text, **kwargs).get_content()
else:
content = response.text
fp.write(content)
log.debug('Writing to %s. Content: %s', dest_filename, content)
return dest_filename
def Just(name, search_text, url):
keyword.add_keyword(name)
find = keyword.extract_keywords(search_text)
if find == []:
print(name + "No FOUND")
save(name, "No FOUND")
else:
Urls = keyword.extract_keywords(url)
if Urls == []:
u = URL(url)
Get_icp(name, u)
keyword.remove_keyword(name)
def search_fair_trade_federation(self):
org_info = CERT_ORGS["fair trade federation"]
url = URL(scheme=WEB_SCHEME,
host=org_info["url"],
path=org_info["search_path"],
query=f"members_search={self.company_name}")
source = requests.get(url)
page = BeautifulSoup(source.content, 'html.parser')
search_results = page.find_all(class_='members-box')
for sr in search_results:
return True
return False
def _sendSessionDetails(self, user, sessionTime, tapsCounters, timeout):
sessionUri = URL(self.apiEndPoint.value).add_path_segment('activity')
payload = {
'sessionTime': sessionTime.isoformat(),
'personnelNumber': user.personnelId,
'Taps': {
tapId: {
'amount': tapsCounters[tapId].volume
}
for tapId in tapsCounters
}
}
userReq = requests.post(sessionUri.as_string(),
auth=HTTPBasicAuth(self._apiUser.value,
self._apiKey.value),
json=payload,
timeout=timeout)
userReq.raise_for_status()
log.info('Session info for user: %d:%s added with activities: %s',
user.personnelId, user.fullName,
str([activity['ActivityId'] for activity in userReq.json()]))
return True
def get_data(q_link):
url = URL(q_link)
if url.domain() not in ['quora.com', 'www.quora.com']:
return 'error, not quora'
url = URL(
scheme='https',
host='www.quora.com',
path=url.path(),
query='share=1').as_string()
soup = BeautifulSoup(requests.get(url).text)
question = {}
question['url'] = url
question['title'] = soup.find("div", {"class": "question_text_edit"}).text
question['topics'] = [topic.text for topic in soup.find_all("div", {"class": "topic_list_item"})]
question['details'] = soup.find("div", {"class": "question_details_text"}).text
answers = []
divs = soup.find_all("div", {"class": "pagedlist_item"})
try:
ans_count = soup.find("div", {"class": "answer_header_text"}).text.strip()
count = int(re.match(r'(\d+) Answers', ans_count).groups()[0])
except:
return jsonify(question=question, answers=answers)
question['answer_count'] = count
count = len(divs) - 1 if count < 6 else 6
for i in range(count):
one_answer = {
'votes': '-1',
'rank': 0,
'answer': ''
}
try:
author = {}
author['name'] = divs[i].find("div", {"class": "answer_user"}).find("span", {"class": "answer_user_wrapper"}).find("a", {"class": "user"}).string
author['bio'] = divs[i].find("div", {"class": "answer_user"}).find("span", {"class": "answer_user_wrapper"}).find_all("span", {"class": "rep"})[1].find("span", {"class": "hidden"}).text
except:
author['name'] = 'Anonymous'
author['bio'] = ''
one_answer['author'] = author
one_answer['votes'] = divs[i].find("span", {"class":"numbers"}).text
html_block = divs[i].find("div", {"id": re.compile("(.*)_container")}).contents
answer_html = ''
for p in range(len(html_block) - 1):
answer_html += str(html_block[p])
one_answer['answer_html'] = answer_html
one_answer['answer'] = divs[i].find("div", {"class": "answer_content"}).text
one_answer['rank'] = i + 1
answers.append(one_answer)
return jsonify(question=question, answers=answers)
def wait(self, url):
"""
如果最近下载过该网站则延迟下载
"""
domain = URL(url).host()
last_accessed = self.domains.get(domain)
if self.delay > 0 and last_accessed is not None:
sleep_secs = self.delay - (datetime.now() - last_accessed).seconds
if sleep_secs > 0:
time.sleep(sleep_secs)
self.domains[domain] = datetime.now()
pass
