def get_links(self, request):
if not self.event:
return []
links = []
if self.event.state == "submitted":
link = URL(request.link(self.event, "publish"))
link = link.query_param("return-to", request.link(self.ticket))
links.append(Link(text=_("Accept event"), url=link.as_string(), classes=("accept-link",)))
links.append(
DeleteLink(
text=_("Reject event"),
url=request.link(self.event),
confirm=_("Do you really want to reject this event?"),
extra_information=_("Rejecting this event can't be undone."),
yes_button_text=_("Reject event"),
redirect_after=request.link(self.ticket),
)
)
link = URL(request.link(self.event, "bearbeiten"))
link = link.query_param("return-to", request.url)
links.append(Link(text=_("Edit event"), url=link.as_string(), classes=("edit-link",)))
return links
def get_camera(self):
if self.config.URL:
url = URL(self.config.URL)
auth = url.username(), url.password()
if url.username is None and url.password is None:
auth = None
url = URL(url.as_string(), username='', password='')
return FoscamHTTPCamera(url.as_string(), auth)
else:
return USBCamera(self.config.DEVICE)
def get_camera(self):
if self.config.URL:
url = URL(self.config.URL)
auth = url.username(), url.password()
if url.username is None and url.password is None:
auth = None
url = URL(url.as_string(), username='', password='')
return FoscamHTTPCamera(url.as_string(), auth)
else:
return USBCamera(self.config.DEVICE)
def activate(self, *args, **kwargs):
"""
Transition: inactive -> active
Set github webhook and activate all Presentations.
"""
try:
provider = self.user.social_auth.get(provider="github")
except UserSocialAuth.DoesNotExists:
logger.error("No social auth provider for Github found on user")
return
github = Github(provider.access_token)
try:
repo = github.get_user().get_repo(self.name)
except GithubException:
logger.error("Could not find repository")
return
url = URL(
scheme="https",
host=self.site.domain,
path=reverse("qraz:webhook", kwargs={"username": self.user.username, "repository": self.name}),
)
try:
hook = repo.create_hook(
"web",
{"url": url.as_string(), "content_type": "json", "secret": self.secret},
events=["push"],
active=True,
)
except GithubException as excp:
logger.error("Could not create webhook: %s", excp)
return
self.hook = hook.id
def get_bib(args):
uploaded = load(args.data_file('repos', 'cdstar.json'))
fname_to_cdstar = {}
for type_ in ['texts', 'docs', 'data']:
for hash_, paths in load(args.data_file('repos', type_ + '.json')).items():
if hash_ in uploaded:
for path in paths:
fname_to_cdstar[path.split('/')[-1]] = uploaded[hash_]
for hash_, paths in load(args.data_file('repos', 'edmond.json')).items():
if hash_ in uploaded:
for path in paths:
fname_to_cdstar[path.split('/')[-1]] = uploaded[hash_]
db = Database.from_file(args.data_file('repos', 'Dogon.bib'), lowercase=True)
for rec in db:
doc = Document(rec)
newurls = []
for url in rec.get('url', '').split(';'):
if not url.strip():
continue
if url.endswith('sequence=1'):
newurls.append(url)
continue
url = URL(url.strip())
if url.host() in ['dogonlanguages.org', 'github.com', '']:
fname = url.path().split('/')[-1]
doc.files.append((fname, fname_to_cdstar[fname]))
else:
newurls.append(url.as_string())
doc.rec['url'] = '; '.join(newurls)
yield doc
def set_param(request=None, url=None, **kwargs):
if not request and not url:
return '/'
url = URL(path=request.path, query=request.META['QUERY_STRING']) if request else URL(url)
for k, v in kwargs.items():
url = url.query_param(k, v)
return url.as_string()
def editbar_links(self):
if self.request.is_logged_in:
edit_url = URL(self.request.link(self.model.event, 'bearbeiten'))
edit_url = edit_url.query_param(
'return-to', self.request.link(self.model.event)
)
edit_link = Link(
text=_("Edit"),
url=edit_url.as_string(),
classes=('edit-link', )
)
if self.event_deletable(self.model.event):
delete_link = DeleteLink(
text=_("Delete"),
url=self.request.link(self.model.event),
confirm=_("Do you really want to delete this event?"),
yes_button_text=_("Delete event"),
redirect_after=self.events_url
)
else:
delete_link = DeleteLink(
text=_("Delete"),
url=self.request.link(self.model.event),
confirm=_("This event can't be deleted."),
extra_information=_(
"To remove this event, go to the ticket and reject it."
)
)
return [edit_link, delete_link]
def __call__(self, request, extra_classes=None):
""" Renders the element. """
a = builder.A(request.translate(self.text))
if self.request_method == 'GET':
a.attrib['href'] = self.url
if self.request_method == 'DELETE':
url = URL(self.url).query_param(
'csrf-token', request.new_csrf_token())
a.attrib['ic-delete-from'] = url.as_string()
if self.classes or extra_classes:
classes = self.classes + (extra_classes or tuple())
a.attrib['class'] = ' '.join(classes)
# add the hidden from public hint if needed
if self.is_hidden_from_public:
# This snippet is duplicated in the hidden-from-public-hint macro!
hint = builder.I()
hint.attrib['class'] = 'hidden-from-public-hint'
hint.attrib['title'] = request.translate(
_("This site is hidden from the general public")
)
a.append(builder.I(' '))
a.append(hint)
for key, value in self.attributes.items():
a.attrib[key] = request.translate(value)
return tostring(a)
def _getValidUsersByCardIdImpl(self, timeout):
validUsersUri = URL(self.apiEndPoint.value).add_path_segment('validpeople')
validUsersReq = requests.get(validUsersUri.as_string(), auth=HTTPBasicAuth(self._apiUser.value, self._apiKey.value), timeout=timeout)
validUsersReq.raise_for_status()
json = validUsersReq.json()
return {str(person['CardId']): User(person['PersonnelNumber'], person['CardId'], person['FullName'], person['Valid'], 3600 * 365)
for person in json}
def get(self, url, default=NO_DEFAULT, headers=None):
"""Retrieve a Response object for a given URL.
"""
headers = headers or {}
url = URL(url)
row = self.db.execute(
select([
responses.c.created,
responses.c.host,
responses.c.request_url,
responses.c.accept,
responses.c.url,
responses.c.headers,
responses.c.content])
.where(and_(
responses.c.request_url == url.as_string(),
responses.c.accept == headers.get('Accept', '')))
).fetchone()
if not row:
log.info('cache miss %s' % url)
row = self.add(url, headers)
if row is None:
if default is NO_DEFAULT:
raise KeyError(url)
log.info('invalid url %s' % url)
return default
else:
log.info('cache hit %s' % url)
return Response(*row)
def get_links(self, request):
edit_link = URL(request.link(self.submission))
edit_link = edit_link.query_param("edit", "")
edit_link = edit_link.query_param("return-to", request.url)
return [Link(text=_("Edit submission"), url=edit_link.as_string(), classes=("edit-link",))]
def url(self):
kwargs = {
"username": self.repository.user.username,
"repository": self.repository.name,
"presentation": self.name,
}
url = URL(scheme="http", host=self.repository.site.domain, path=reverse("qraz:download", kwargs=kwargs))
return url.as_string()
def handle_pending_submission(self, request):
""" Renders a pending submission, takes it's input and allows the
user to turn the submission into a complete submission, once all data
is valid.
Takes the following query parameters for customization::
* ``edit`` no validation is done on the first load if present
* ``return-to`` the view redirects to this url once complete if present
* ``title`` a custom title (required if external submission)
* ``quiet`` no success messages are rendered if present
"""
collection = FormCollection(request.app.session())
form = request.get_form(self.form_class, data=self.data)
form.action = request.link(self)
if 'edit' not in request.GET:
form.validate()
if not request.POST:
form.ignore_csrf_error()
else:
collection.submissions.update(self, form)
# these parameters keep between form requests (the rest throw away)
for param in {'return-to', 'title', 'quiet'}:
if param in request.GET:
action = URL(form.action).query_param(param, request.GET[param])
form.action = action.as_string()
completable = not form.errors and 'edit' not in request.GET
if completable and 'return-to' in request.GET:
if 'quiet' not in request.GET:
request.success(_("Your changes were saved"))
return morepath.redirect(request.GET['return-to'])
if 'title' in request.GET:
title = request.GET['title']
else:
title = self.form.title
return {
'layout': FormSubmissionLayout(self, request, title),
'title': title,
'form': form,
'completable': completable,
'edit_link': request.link(self) + '?edit',
'complete_link': request.link(self, 'complete'),
'is_pending': self.state == 'pending',
'readonly': 'readonly' in request.GET,
}
def get_submission_link(request, submission, confirm_link):
url = URL(request.link(submission))
url = url.query_param('return-to', confirm_link)
url = url.query_param('title', request.translate(
_("Details about the reservation"))
)
url = url.query_param('edit', 1)
url = url.query_param('quiet', 1)
return url.as_string()
def _sendSessionDetails(self, user, sessionTime, tapsCounters, timeout):
sessionUri = URL(self.apiEndPoint.value).add_path_segment('activity')
payload = {
'sessionTime': sessionTime.isoformat(),
'personnelNumber':user.personnelId,
'Taps': {tapId:{'amount':tapsCounters[tapId].volume} for tapId in tapsCounters}
}
userReq = requests.post(sessionUri.as_string(), auth=HTTPBasicAuth(self._apiUser.value, self._apiKey.value), json=payload, timeout=timeout)
userReq.raise_for_status()
log.info('Session info for user: %d:%s added with activities: %s', user.personnelId, user.fullName, str([activity['ActivityId'] for activity in userReq.json()]))
return True
def _isUserAuthenticatedImpl(self, cardId, timeout):
userAuthUri = URL(self.apiEndPoint.value).add_path_segment('ispersonvalid').add_path_segment(str(cardId))
userReq = requests.get(userAuthUri.as_string(), auth=HTTPBasicAuth(self._apiUser.value, self._apiKey.value), timeout=timeout)
userReq.raise_for_status()
json = userReq.json()
if isinstance(json, list):
json = json[0]
validUser = json.get('Valid', False)
personnelId = int(json.get('PersonnelNumber', 0))
fullName = json.get('FullName', '')
return (validUser, personnelId, fullName)
def get_links(self, request):
edit_link = URL(request.link(self.submission))
edit_link = edit_link.query_param('edit', '')
edit_link = edit_link.query_param('return-to', request.url)
return [
Link(
text=_('Edit submission'),
url=edit_link.as_string(),
classes=('edit-link', )
)
]
def get_links(self, request):
if self.deleted:
return []
links = []
data = self.reservations[0].data or {}
if not data.get("accepted"):
link = URL(request.link(self.reservations[0], "annehmen"))
link = link.query_param("return-to", request.url)
links.append(Link(text=_("Accept reservation"), url=link.as_string(), classes=("accept-link",)))
link = URL(request.link(self.reservations[0], "absagen"))
link = link.query_param("return-to", request.url)
links.append(
DeleteLink(
text=_("Reject reservation"),
url=link.as_string(),
confirm=_("Do you really want to reject this reservation?"),
extra_information=_("Rejecting this reservation can't be undone."),
yes_button_text=_("Reject reservation"),
request_method="GET",
redirect_after=request.url,
)
)
if self.submission:
link = URL(request.link(self.submission))
link = link.query_param("edit", "")
link = link.query_param("return-to", request.url)
link = link.query_param("title", request.translate(_("Details about the reservation")))
links.append(Link(text=_("Edit details"), url=link.as_string(), classes=("edit-link",)))
return links
def blog_feed(request):
"""
Proxy feeds from the blog, so they can be accessed via XHR requests.
We also convert RSS to ATOM so that clld's javascript Feed component can read them.
"""
if not request.params.get('path'):
raise HTTPNotFound()
path = URL(request.params['path'])
assert not path.host()
try:
return atom_feed(request, request.blog.url(path.as_string()))
except ConnectionError: # pragma: no cover
raise HTTPNotFound()
def by_filter(self, filter):
# note: addGisLink doesn't work here
url = URL('odata/Baustellen')\
.query_param('addGisLink', 'False')\
.query_param('$filter', filter)
records = self.client.get(url.as_string()).get('value', ())
records = (r for r in records if r['Internet'])
work = [Roadwork(r) for r in records]
work.sort(key=lambda r: r.title)
return work
def blog_feed(request):
"""
Proxy feeds from the blog, so they can be accessed via XHR requests.
We also convert RSS to ATOM so that clld's javascript Feed component can read them.
"""
if not request.params.get('path'):
raise HTTPNotFound()
path = URL(request.params['path'])
assert not path.host()
try:
return atom_feed(request, request.blog.url(path.as_string()))
except ConnectionError:
raise HTTPNotFound()
def get(self, url, **kw):
url = URL(url)
content = get_resource(
url.path_segment(-1) if url.path_segments() else 'dataset', self.prefix)
if content:
return MagicMock(
mimetype='application/rdf+xml',
content=content,
links=[dict(
url='{0}.html'.format(url.as_string),
ext='.html',
rel='alternate',
type='text/html')],
canonical_url=url.as_string())
def get_resource_url(self, id=None, action=None):
"""
Get a base URL for the resource with the given id and/or action.
:return: A string URL with the full path to the resource
"""
base_url = URL(Configuration.BASE_API_URL).add_path_segment(self._resource_name)
if id:
base_url = base_url.add_path_segment(id)
if action:
base_url = base_url.add_path_segment(action)
return base_url.as_string()
def _getInstallationPackages(self, currentVersion, packageType,
checkUnpublished, timeout):
packagesUri = URL(self.apiEndPoint.value) \
.add_path_segment('updates') \
.add_path_segment(packageType) \
.append_query_param('min-version_gt', currentVersion) \
.append_query_param('include-unpublished', 'true' if checkUnpublished else 'false')
availablePackages = requests.get(packagesUri.as_string(),
auth=HTTPBasicAuth(
self._apiUser.value,
self._apiKey.value),
timeout=timeout)
availablePackages.raise_for_status()
return availablePackages.json()
def parseJson(self, response):
resjson = demjson.decode(demjson.decode(response.text))
for res in resjson:
if res['knowledgeType'] == 'VideoKnowledge':
item = IlexueItem()
siteURL = URL(res['knowledgeUrl']).scheme('http').domain(
self.host)
item['siteURL'] = siteURL.as_string()
item['onclick'] = ''
item['pageURL'] = response.request.url
item['title'] = res['title']
item['fileName'] = self.base + item['title']
yield Request(url=item['siteURL'],
meta={'item1': item},
callback=self.parseVideoAndDocument)
def by_id(self, id):
url = URL(f'odata/Baustellen({int(id)})')\
.query_param('addGisLink', 'True')
work = tuple(
Roadwork(r)
for r in self.client.get(url.as_string()).get('value', ()))
if work:
return work[0]
# secondary lookup is against the subsections.. this probably calls
# for an index eventually
for r in self.roadwork:
for section in r.sections:
if section.id == id:
return section
def get_links(instr):
"""
Given a link-value, generate individual links as dict.
"""
if instr:
for link in [h.strip() for h in link_splitter.findall(instr)]:
url, params = link.split(">", 1)
url = URL(url[1:])
fname = url.path_segments()[-1] if url.path_segments() else ''
info = {
'url': url.as_string(),
'ext': fname.split('.', 1)[1] if '.' in fname else None,
'rel': 'related',
'type': 'application/octet-stream'}
for param in _splitstring(params, PARAMETER, "\s*;\s*"):
try:
a, v = param.split("=", 1)
info[a.lower()] = _unquotestring(v)
except ValueError: # pragma: no cover
info[param.lower()] = None
yield info
def handle_pending_submission(self, request):
""" Renders a pending submission, takes it's input and allows the
user to turn the submission into a complete submission, once all data
is valid.
"""
collection = FormCollection(request.app.session())
form = request.get_form(self.form_class, data=self.data)
form.action = request.link(self)
form.validate()
if not request.POST:
form.ignore_csrf_error()
else:
collection.submissions.update(self, form)
if 'return-to' in request.GET:
action = URL(form.action)
action = action.query_param('return-to', request.GET['return-to'])
form.action = action.as_string()
completable = not form.errors and 'edit' not in request.GET
if completable and 'return-to' in request.GET:
request.success(_(u"Your changes were saved"))
return morepath.redirect(request.GET['return-to'])
return {
'layout': FormSubmissionLayout(self, request),
'title': self.form.title,
'form': form,
'completable': completable,
'edit_link': request.link(self) + '?edit',
'complete_link': request.link(self, 'complete'),
'is_pending': self.state == 'pending',
'readonly': 'readonly' in request.GET
}
def handle_edit_event(self, request, form):
""" Edit an event.
An anonymous user might edit an initiated event, a logged in user can also
edit all events.
"""
assert_anonymous_access_only_temporary(request, self)
if form.submitted(request):
form.update_model(self)
request.success(_("Your changes were saved"))
if 'return-to' in request.GET:
return morepath.redirect(request.GET['return-to'])
return morepath.redirect(request.link(self))
form.apply_model(self)
if 'return-to' in request.GET:
action = URL(form.action)
action = action.query_param('return-to', request.GET['return-to'])
form.action = action.as_string()
layout = EventLayout(self, request)
layout.breadcrumbs.append(Link(_("Edit"), '#'))
layout.editbar_links = []
return {
'layout': layout,
'title': self.title,
'form': form,
'form_width': 'large'
}
def test_kwargs_take_priority_when_used_with_full_url(self):
u = URL('https://github.com', scheme='http')
self.assertEqual('http://github.com', u.as_string())
def test_full_url_can_be_used_as_first_param(self):
u = URL('https://github.com')
self.assertEqual('https://github.com', u.as_string())
def test_build_with_path_segments(self):
u = URL().path_segments(['path', 'to', 'page'])
self.assertEqual('/path/to/page', u.as_string())
def test_build_with_path_segments(self):
u = URL().path_segments(["path", "to", "page"])
self.assertEqual("/path/to/page", u.as_string())
from purl import URL
str_url = URL('https://www.google.com/search?q=google')
print(str_url)
print(str_url.as_string())
argument_url = URL(scheme='https',
host='www.google.com',
path='/search',
query='q=google')
print(argument_url)
print(argument_url.as_string())
inline_url = URL().scheme('https').domain('www.google.com').path(
'search').query_param('q', 'google')
print(inline_url)
print(inline_url.as_string())
u = URL('postgres://*****:*****@localhost:1234/test?ssl=true')
print(u.scheme())
print(u.host())
print(u.domain())
print(u.username())
print(u.password())
print(u.netloc())
print(u.port())
print(u.path())
print(u.query())
print(u.path_segments())
print(u.query_param('ssl'))
print(u.query_param('ssl', as_list=True))
print(u.query_params())
print(u.has_query_param('ssl'))
def test_build_with_path_segments(self):
u = URL().path_segments(['path', 'to', 'page'])
self.assertEqual('/path/to/page', u.as_string())
def for_username(self, username):
url = URL(self.action)
url = url.query_param('username', username)
return url.as_string()
def test_slashes_in_path(self):
u = URL().add_path_segment('test/egg')
self.assertEqual(u.as_string(), '/test%2Fegg')
def test_slashes_in_path(self):
u = URL('/something').path_segment(0, 'test/egg')
self.assertEqual(u.as_string(), '/test%2Fegg')
def test_slashes_in_path(self):
u = URL().add_path_segment('test/egg')
self.assertEqual(u.as_string(), '/test%2Fegg')
def test_kwargs_take_priority_when_used_with_full_url(self):
u = URL('https://github.com', scheme='http')
self.assertEqual('http://github.com', u.as_string())
def test_full_url_can_be_used_as_first_param(self):
u = URL('https://github.com')
self.assertEqual('https://github.com', u.as_string())
def test_slashes_in_path(self):
u = URL('/something').path_segment(0, 'test/egg')
self.assertEqual(u.as_string(), '/test%2Fegg')
def parse(self, response):
# 创建一个大的list存储所有的item
items = []
print(response.request.headers)
# pattern_html = re.compile(r'<div class="title".*?<a.*?href="(.*?)">(.*?)</a></span></div>', re.S)
htmls = re.findall(
re.compile(
r'(\/[plan|kng]+/[\S]*?.htm[l]??[?]?[\S]*?)[\'"].*?title=\"(.*?)\"',
re.S), response.text)
for html in htmls:
# 创建实例,并转化为字典
item = IlexueItem()
siteURL = URL(html[0]).scheme('http').domain(self.host)
item['id'] = ''
item['siteURL'] = siteURL.as_string()
item['onclick'] = ''
item['pageURL'] = response.request.url
item['title'] = 'more' if html[1] == '' else html[1]
item['fileName'] = self.base + item['title']
items.append(item)
# pattern_onclick = re.compile(r'onclick="(.*?)".*?>(\w*[·\.\.\.]*)<.*?(span>\(([0-9]+)\))*', re.S)
# onclicks = re.findall(pattern_onclick, response.text)
onclicks = re.findall(
re.compile(
r'(id="([\w|-]*)"[^\<|^\>]*)?((onclick="(.*?)")|(href="javascript:void\((.*?)\))).*?>([\w|"]*[·\.\.\.]*)<.*?(span>\(([0-9]+)\))*',
re.S), response.text)
for onclick in onclicks:
if onclick[7] != '' and 'displayData' in onclick[4] and (
onclick[9] == '' or onclick[9] == '0'):
continue
if 'amp;amp;' in response.request.url:
response.request.url = response.request.url.replace('amp;', '')
continue
# 创建实例,并转化为字典
item = IlexueItem()
item['id'] = onclick[1]
item['siteURL'] = ''
item['onclick'] = onclick[4]
item['pageURL'] = response.request.url
item['title'] = 'more' if onclick[7] == '' else onclick[7]
item['fileName'] = self.base + item['title']
items.append(item)
for item in items:
# 创建文件夹
fileName = item['fileName']
if not os.path.exists(fileName.replace('\"', '')):
os.makedirs(fileName.replace('\"', ''))
# 用meta传入下一层
print(item)
if item["siteURL"].find('video') > 0 or item['siteURL'].find(
'document') > 0:
yield Request(url=item['siteURL'],
meta={'item1': item},
callback=self.parseVideoAndDocument)
elif len(item['onclick']) > 0:
yield Request(url=item['pageURL'],
meta={'item1': item},
callback=self.parse,
dont_filter=True)
else:
yield Request(url=item['siteURL'],
meta={'item1': item},
callback=self.parse)
