def test_suffix_normalize():
for suffix, expected in REL_URL_TEST_CASES:
url = URL(REL_URL_BASE)
url.normalize(suffix)
assert url.path == URL(expected).path
return
def test_parse_equals_in_qp_value():
u = URL('http://localhost/?=x=x=x')
assert u.q[''] == 'x=x=x'
assert u.to_text() == 'http://localhost/?=x%3Dx%3Dx'
u = URL('http://localhost/?foo=x=x=x&bar=y')
assert u.q['foo'] == 'x=x=x'
assert u.q['bar'] == 'y'
def compose_url(base_url, url):
base_url = URL(base_url)
url = URL(url)
if not url.scheme:
absolute_url = base_url.navigate(url.to_text())
else:
absolute_url = url
return absolute_url.to_text()
def test_chained_navigate(expected, base, paths):
"""Chained :meth:`navigate` calls produces correct results."""
url = URL(base)
for path in paths:
url = url.navigate(path)
assert expected == url.to_text()
def test_rel_navigate():
for suffix, expected in REL_URL_TEST_CASES:
url = URL(REL_URL_BASE)
new_url = url.navigate(suffix)
assert new_url.to_text() == expected
new_url = url.navigate(URL(suffix))
assert new_url.to_text() == expected
return
def test_create_tag(user: UserClient):
"""Creates a tag specifying a custom organization."""
org = Organization(name='bar', tax_id='bartax')
tag = Tag(id='bar-1', org=org, provider=URL('http://foo.bar'), owner_id=user.user['id'])
db.session.add(tag)
db.session.commit()
tag = Tag.query.one()
assert tag.id == 'bar-1'
assert tag.provider == URL('http://foo.bar')
res, _ = user.get(res=Tag, item=tag.code, status=422)
assert res['type'] == 'TagNotLinked'
def _test_bad_utf8(): # not part of the API
bad_bin_url = 'http://xn--9ca.com/%00%FF/%C3%A9'
url = URL(bad_bin_url)
expected = ('http://\N{LATIN SMALL LETTER E WITH ACUTE}.com/'
'%00%FF/'
'\N{LATIN SMALL LETTER E WITH ACUTE}')
actual = url.to_text()
assert expected == actual
def test_unicodey():
unicodey = (u'http://\N{LATIN SMALL LETTER E WITH ACUTE}.com/'
u'\N{LATIN SMALL LETTER E}\N{COMBINING ACUTE ACCENT}'
u'?\N{LATIN SMALL LETTER A}\N{COMBINING ACUTE ACCENT}='
u'\N{LATIN SMALL LETTER I}\N{COMBINING ACUTE ACCENT}'
u'#\N{LATIN SMALL LETTER U}\N{COMBINING ACUTE ACCENT}')
url = URL(unicodey)
assert url.host == u'é.com'
assert url.path_parts[1] == u'\N{LATIN SMALL LETTER E}\N{COMBINING ACUTE ACCENT}'
assert url.to_text(full_quote=False) == unicodey
fully_quoted = 'http://xn--9ca.com/%C3%A9?%C3%A1=%C3%AD#%C3%BA'
assert url.to_text(full_quote=True) == fully_quoted
def prepare_urls(self, msg_or_text, direct_urls=False):
if isinstance(msg_or_text, Message):
urls = []
url_entities = msg_or_text.parse_entities(
types=[MessageEntity.URL])
for entity in url_entities:
url_str = url_entities[entity]
logger.debug("Entity URL Parsed: %s", url_str)
if "://" not in url_str:
url_str = "http://{}".format(url_str)
urls.append(URL(url_str))
text_link_entities = msg_or_text.parse_entities(
types=[MessageEntity.TEXT_LINK])
for entity in text_link_entities:
url_str = entity.url
logger.debug("Entity Text Link Parsed: %s", url_str)
urls.append(URL(url_str))
else:
urls = find_all_links(msg_or_text, default_scheme="http")
urls_dict = {}
for url in urls:
url_text = url.to_text(True)
url_parts_num = len([part for part in url.path_parts if part])
try:
if (
# SoundCloud: tracks, sets and widget pages, no /you/ pages
(self.SITES["sc"] in url.host and
(2 <= url_parts_num <= 3
or self.SITES["scapi"] in url_text) and
(not "you" in url.path_parts)) or
# Bandcamp: tracks and albums
(self.SITES["bc"] in url.host and
(2 <= url_parts_num <= 2)) or
# YouTube: videos and playlists
(self.SITES["yt"] in url.host and
("youtu.be" in url.host or "watch" in url.path
or "playlist" in url.path))):
if direct_urls or self.SITES["yt"] in url.host:
urls_dict[url_text] = get_direct_urls(url_text)
else:
urls_dict[url_text] = "http"
elif not any(
(site in url.host for site in self.SITES.values())):
urls_dict[url_text] = get_direct_urls(url_text)
except ProcessExecutionError:
logger.debug("youtube-dl get url failed: %s", url_text)
except URLError as exc:
urls_dict[url_text] = exc.status
return urls_dict
def get_gh_project_info(url):
ret = {}
org, repo = URL(url).path_parts[1:]
gh_url = URL('https://api.github.com/repos')
gh_url.path_parts += (org, repo)
project_url = gh_url.to_text()
project_data = _get_gh_json(project_url)
ret['star_count'] = project_data['stargazers_count']
gh_url.path_parts += ('tags', )
tags_url = gh_url.to_text()
tags_data = _get_gh_json(tags_url)
vtags_data = [td for td in tags_data if match_vtag(td['name'], PREFIXES)]
ret['release_count'] = len(vtags_data)
first_release = vtags_data[-1]
first_release_data = _get_gh_rel_data(first_release, PREFIXES)
for k, v in first_release_data.items():
ret['first_release_%s' % k] = v
latest_release = vtags_data[0]
latest_release_data = _get_gh_rel_data(latest_release, PREFIXES)
for k, v in latest_release_data.items():
ret['latest_release_%s' % k] = v
zv_releases = [
rel for rel in vtags_data
if match_vtag(rel['name'], PREFIXES).group('major') == '0'
]
ret['release_count_zv'] = len(zv_releases)
print ' .. %s releases, %s 0ver' % (ret['release_count'],
ret['release_count_zv'])
is_zerover = zv_releases[0] == latest_release
ret['is_zerover'] = is_zerover
if is_zerover:
return ret
last_zv_release = zv_releases[0]
last_zv_release_data = _get_gh_rel_data(last_zv_release, PREFIXES)
for k, v in last_zv_release_data.items():
ret['last_zv_release_%s' % k] = v
return ret
def parse_uri(cls, uri):
"""Parses a BIP21 Payment URI into this class"""
parsed = URL(uri)
currency = parsed.scheme
address = parsed.path
amount = parsed.qp['amount']
return cls(currency, address, amount)
def open(self,
uri: str,
res: str = None,
status: Status = 200,
query: Query = tuple(),
accept=JSON,
content_type=JSON,
item=None,
headers: dict = None,
token: str = None,
**kw) -> Res:
headers = headers or {}
if res:
resource_url = self.application.resources[res].url_prefix + '/'
uri = URL(uri).navigate(resource_url).to_text()
if token:
headers['Authorization'] = 'Basic {}'.format(token)
res = super().open(uri, status, query, accept, content_type, item,
headers, **kw)
# ereuse-utils checks for status code
# here we check for specific type
# (when response: {'type': 'foobar', 'code': 422})
_status = getattr(status, 'code', status)
if not isinstance(status, int) and res[1].status_code == _status:
assert status.__name__ == res[0]['type'], \
'Expected exception {0} but it was {1}'.format(status.__name__, res[0]['type'])
return res
def parse_uri(cls, uri):
"""Parses a EIP681 Payment URI into this class"""
parsed = URL(uri)
currency = parsed.scheme
address = parsed.path
value = parsed.qp['value']
return cls(currency, address, value)
def test_get_tags_endpoint(user: UserClient, app: Devicehub,
requests_mock: requests_mock.mocker.Mocker):
"""Performs GET /tags after creating 3 tags, 2 printable and one
not. Only the printable ones are returned.
"""
# Prepare test
with app.app_context():
org = Organization(name='bar', tax_id='bartax')
tag = Tag(id='bar-1', org=org, provider=URL('http://foo.bar'), owner_id=user.user['id'])
db.session.add(tag)
db.session.commit()
assert not tag.printable
requests_mock.post('https://example.com/',
# request
request_headers={
'Authorization': 'Basic {}'.format(DevicehubClient.encode_token(
'52dacef0-6bcb-4919-bfed-f10d2c96ecee'))
},
# response
json=['tag1id', 'tag2id'],
status_code=201)
user.post({}, res=Tag, query=[('num', 2)])
# Test itself
data, _ = user.get(res=Tag)
assert len(data['items']) == 2, 'Only 2 tags are printable, thus retreived'
# Order is created descending
assert data['items'][0]['id'] == 'tag2id'
assert data['items'][0]['printable']
assert data['items'][1]['id'] == 'tag1id'
assert data['items'][1]['printable'], 'Tags made this way are printable'
def test_utf8_url():
url_bytes = (b'http://\xd9\x85\xd8\xab\xd8\xa7\xd9\x84'
b'.\xd8\xa2\xd8\xb2\xd9\x85\xd8\xa7'
b'\xdb\x8c\xd8\xb4\xdb\x8c')
url = URL(url_bytes)
assert url.scheme == 'http'
assert url.host == u'مثال.آزمایشی'
def test_quoted_userinfo():
url = URL('http://wikipedia.org')
url.username = u'user'
url.password = u'p@ss'
assert url.to_text(full_quote=True) == 'http://*****:*****@wikipedia.org'
url = URL(u'http://beyonc\xe9:b\[email protected]')
# assert url.to_text(full_quote=False) == u'http://beyoncé:b%C3%[email protected]'
assert url.to_text(full_quote=True) == u'http://beyonc%C3%A9:b%C3%[email protected]'
def _get_links(self, group, name):
link_list = list(self.get_config(group, name, []))
for link in link_list:
if link['href'] and URL(link['href']).host:
link['is_external'] = True
else:
link['is_external'] = False
return link_list
def test_idna():
u1 = URL(u'http://bücher.ch')
assert u1.host == u'bücher.ch'
assert u1.to_text(full_quote=True) == 'http://xn--bcher-kva.ch'
assert u1.to_text(full_quote=False) == u'http://bücher.ch'
u2 = URL('https://xn--bcher-kva.ch')
assert u2.host == u'bücher.ch'
assert u2.to_text(full_quote=True) == 'https://xn--bcher-kva.ch'
assert u2.to_text(full_quote=False) == u'https://bücher.ch'
def test_normalize_with_case():
# from RFC 3986 Section 6.2.2
url1 = URL('example://a/b/c/%7Bfoo%7D')
url2 = URL('eXAMPLE://a/./b/../b/%63/%7bfoo%7d')
assert url1 != url2
url1.normalize()
url2.normalize()
assert url1 == url2
def test_tag_create_etags_cli(app: Devicehub, user: UserClient):
"""Creates an eTag through the CLI."""
# todo what happens to organization?
owner_id = user.user['id']
runner = app.test_cli_runner()
args = ('tag', 'add', '-p', 'https://t.ereuse.org', '-s', 'foo', 'DT-BARBAR', '-u', owner_id)
runner.invoke(*args)
with app.app_context():
tag = Tag.query.one() # type: Tag
assert tag.id == 'dt-barbar'
assert tag.secondary == 'foo'
assert tag.provider == URL('https://t.ereuse.org')
def test_etag_secondary(client: Client, app: Teal):
"""Tests creating, linking and accessing an ETag through
its secondary (NFC) id."""
with app.app_context():
et = ETag(secondary='NFCID')
db.session.add(et)
db.session.commit()
client.get('/', item='NFCID', accept=ANY, status=400)
with app.app_context():
tag = ETag.query.filter_by(secondary='NFCID').one()
tag.devicehub = URL('https://dh.com')
db.session.commit()
_, r = client.get('/', item='NFCID', accept=ANY, status=302)
assert r.location == 'https://dh.com/tags/FO-3MP5M/device'
def get_link_text(urls):
link_text = ""
for i, url in enumerate(urls):
link_text += "[Source Link #{}]({}) | `{}`\n".format(str(i + 1), url, URL(url).host)
direct_urls = urls[url].splitlines()
for direct_url in direct_urls:
if "http" in direct_url:
content_type = ""
if "googlevideo" in direct_url:
if "audio" in direct_url:
content_type = "Audio"
else:
content_type = "Video"
# direct_url = shorten_url(direct_url)
link_text += "• {} [Direct Link]({})\n".format(content_type, direct_url)
return link_text
def test_create_tags_cli(runner: FlaskCliRunner, client: Client):
"""Tests creating a tag."""
with NamedTemporaryFile('r+') as f:
result = runner.invoke(args=('create-tags', '100', '--csv', f.name,
'--etag'),
catch_exceptions=False)
assert result.exit_code == 0
urls = tuple(csv.reader(f))
assert len(urls) == 100
for url, *_ in urls:
assert 'http://foo.bar/' in url
assert len(url) == 23
# Tag exists but it has not been linked
client.get(res=Tag.t,
item=URL(url).path_parts[1],
status=NoRemoteTag)
def get_link_text(urls):
link_text = ""
for i, url in enumerate(urls):
link_text += "[Source Link #{}]({}) | `{}`\n".format(str(i + 1), url, URL(url).host)
direct_urls = urls[url].splitlines()
for direct_url in direct_urls:
if "http" in direct_url:
content_type = ""
if "googlevideo" in direct_url:
if "audio" in direct_url:
content_type = "Audio"
else:
content_type = "Video"
# direct_url = shorten_url(direct_url)
link_text += "• {} [Direct Link]({})\n".format(content_type, direct_url)
link_text += "\n*Note:* Final download URLs are only guaranteed to work on the same machine/IP where extracted"
return link_text
def import_tags(self, csv: Path):
"""Imports the database from a CSV from ``export``.
This truncates only the Tag table.
"""
db.session.execute('TRUNCATE TABLE {} RESTART IDENTITY'.format(
Tag.__table__.name))
max_id = 0
"""The maximum ID of the tags. Sequence is reset to this + 1."""
with csv.open() as f:
for row in csvm.reader(f):
id, secondary, dh, type, updated, created = row
cls = Tag if type == 'Tag' else ETag
dh = URL(dh) if dh else None
t = cls(id=id, secondary=secondary)
db.session.add(t)
max_id = max(max_id, cls.decode(id))
db.session.execute(
'ALTER SEQUENCE tag_id RESTART WITH {}'.format(max_id + 1))
db.session.commit()
def get_link_text(self, urls):
link_text = ""
for i, url in enumerate(urls):
link_text += "[Source Link #{}]({}) | `{}`\n".format(str(i + 1), url, URL(url).host)
direct_urls = urls[url].splitlines()
for j, direct_url in enumerate(direct_urls):
if "http" in direct_url:
content_type = ""
if "googlevideo" in direct_url:
if "audio" in direct_url:
content_type = "Audio"
else:
content_type = "Video"
if self.shortener:
try:
direct_url = self.shortener.short(direct_url)
# botan.shorten_url(original_url, botan_token, uid)
except:
pass
link_text += "• {} [Direct Link]({})\n".format(content_type, direct_url)
return link_text
def prepare_urls(self, msg=None, text=None, get_direct_urls=False):
if text:
urls = find_all_links(text, default_scheme="http")
elif msg:
urls = []
for url_str in msg.parse_entities(types=["url"]).values():
if "://" not in url_str:
url_str = "http://" + url_str
urls.append(URL(url_str))
else:
logger.debug("Text or msg is required")
return
urls_dict = {}
for url in urls:
url_text = url.to_text(True)
url_parts_num = len([part for part in url.path_parts if part])
if (
# SoundCloud: tracks, sets and widget pages
(self.SITES["sc"] in url.host and
(2 <= url_parts_num <= 3 or self.SITES["scapi"] in url_text))
or
# Bandcamp: tracks and albums
(self.SITES["bc"] in url.host and (2 <= url_parts_num <= 2)) or
# YouTube: videos and playlists
(self.SITES["yt"] in url.host and
("youtu.be" in url.host or "watch" in url.path
or "playlist" in url.path))):
if get_direct_urls or self.SITES["yt"] in url.host:
direct_urls = self.youtube_dl_get_direct_urls(url_text)
if direct_urls:
urls_dict[url_text] = direct_urls
else:
urls_dict[url_text] = "http"
elif not any((site in url.host for site in self.SITES.values())):
direct_urls = self.youtube_dl_get_direct_urls(url_text)
if direct_urls:
urls_dict[url_text] = direct_urls
if not urls_dict:
logger.info("No supported URLs found")
return urls_dict
def test_delete_tags(user: UserClient, client: Client):
"""Delete a named tag."""
# Delete Tag Named
g.user = User.query.one()
pc = Desktop(serial_number='sn1', chassis=ComputerChassis.Tower, owner_id=user.user['id'])
db.session.add(pc)
db.session.commit()
tag = Tag(id='bar', owner_id=user.user['id'], device_id=pc.id)
db.session.add(tag)
db.session.commit()
tag = Tag.query.all()[-1]
assert tag.id == 'bar'
# Is not possible delete one tag linked to one device
res, _ = user.delete(res=Tag, item=tag.id, status=422)
msg = 'The tag bar is linked to device'
assert msg in res['message'][0]
tag.device_id = None
db.session.add(tag)
db.session.commit()
# Is not possible delete one tag from an anonymous user
client.delete(res=Tag, item=tag.id, status=401)
# Is possible delete one normal tag
user.delete(res=Tag, item=tag.id)
user.get(res=Tag, item=tag.id, status=404)
# Delete Tag UnNamed
org = Organization(name='bar', tax_id='bartax')
tag = Tag(id='bar-1', org=org, provider=URL('http://foo.bar'), owner_id=user.user['id'])
db.session.add(tag)
db.session.commit()
tag = Tag.query.all()[-1]
assert tag.id == 'bar-1'
res, _ = user.delete(res=Tag, item=tag.id, status=422)
msg = 'This tag {} is unnamed tag. It is imposible delete.'.format(tag.id)
assert msg in res['message']
tag = Tag.query.all()[-1]
assert tag.id == 'bar-1'
def set_tags(self, devicehub: str, starting_tag: int, ending_tag: int,
csv: Path):
"""
"Sends" the tags to the specific devicehub,
so they can only be linked in that devicehub.
Actual implementation does not send them but rather update the
internal database as it did send them. You will need to create
them manually in the destination devicehub.
This method in the future will probably actually (virtually)
send them.
"""
assert starting_tag < ending_tag
assert URL(devicehub) and devicehub[
-1] != '/', 'Provide a valid URL without leading slash'
tags = Tag.query.filter(between(Tag.id, starting_tag,
ending_tag)).all()
for tag in tags:
tag.devicehub = devicehub
db.session.commit()
self.tags_to_csv(csv, tags)
print('All tags set to {}'.format(devicehub))
def test_inventory_create_delete_user(cli, tdb1, tdb2):
"""Tests creating two inventories with users, one user has
access to the first inventory and the other to both. Finally, deletes
the first inventory, deleting only the first user too.
"""
# Create first DB
cli.inv('tdb1')
cli.invoke('inv', 'add', '-n', 'Test DB1', '-on', 'ACME DB1', '-oi',
'acme-id', '-tu', 'https://example.com', '-tt',
'3c66a6ad-22de-4db6-ac46-d8982522ec40', '--common')
# Create an user for first DB
cli.invoke('user', 'add', '*****@*****.**', '-a', 'Foo', '-c', 'ES', '-p',
'Such password')
with tdb1.app_context():
# There is a row for the inventory
inv = Inventory.query.one() # type: Inventory
assert inv.id == 'tdb1'
assert inv.name == 'Test DB1'
assert inv.tag_provider == URL('https://example.com')
assert inv.tag_token == UUID('3c66a6ad-22de-4db6-ac46-d8982522ec40')
assert db.has_schema('tdb1')
org = Organization.query.one() # type: Organization
# assert inv.org_id == org.id
assert org.name == 'ACME DB1'
assert org.tax_id == 'acme-id'
user = User.query.one() # type: User
assert user.email == '*****@*****.**'
cli.inv('tdb2')
# Create a second DB
# Note how we don't create common anymore
cli.invoke('inv', 'add', '-n', 'Test DB2', '-on', 'ACME DB2', '-oi',
'acme-id-2', '-tu', 'https://example.com', '-tt',
'fbad1c08-ffdc-4a61-be49-464962c186a8')
# Create an user for with access for both DB
cli.invoke('user', 'add', '*****@*****.**', '-a', 'Bar', '-p', 'Wow password')
with tdb2.app_context():
inventories = Inventory.query.all() # type: List[Inventory]
assert len(inventories) == 2
assert inventories[0].id == 'tdb1'
assert inventories[1].id == 'tdb2'
assert db.has_schema('tdb2')
org_db2 = Organization.query.one()
assert org_db2 != org
assert org_db2.name == 'ACME DB2'
users = User.query.all() # type: List[User]
assert users[0].email == '*****@*****.**'
assert users[1].email == '*****@*****.**'
# Delete tdb1
cli.inv('tdb1')
cli.invoke('inv', 'del', '--yes')
with tdb2.app_context():
# There is only tdb2 as inventory
inv = Inventory.query.one() # type: Inventory
assert inv.id == 'tdb2'
# User [email protected] is deleted because it only
# existed in tdb1, but not [email protected] which existed
# in another inventory too (tdb2)
user = User.query.one() # type: User
assert user.email == '*****@*****.**'
assert not db.has_schema('tdb1')
assert db.has_schema('tdb2')
def get_gh_project_info(info):
ret = {}
url = info.get('gh_url')
if url is None:
return ret
org, repo = URL(url.rstrip('/')).path_parts[1:]
gh_url = URL('https://api.github.com/repos')
gh_url.path_parts += (org, repo)
project_url = gh_url.to_text()
project_data = _get_gh_json(project_url)
ret['star_count'] = project_data['stargazers_count']
gh_url.path_parts += ('tags', )
tags_url = gh_url.to_text()
tags_data = _get_gh_json(tags_url)
vtags_data = [td for td in tags_data if match_vtag(td['name'], PREFIXES)]
ret['release_count'] = len(vtags_data)
latest_release = vtags_data[0]
latest_release_data = _get_gh_rel_data(latest_release, PREFIXES)
for k, v in latest_release_data.items():
ret['latest_release_%s' % k] = v
vtags_data.sort(key=lambda x: version_key(x['name'], PREFIXES),
reverse=True)
first_release_version = info.get('first_release_version')
if first_release_version is None:
first_release = [
v for v in vtags_data
if version_key(v['name']) < version_key(latest_release['name'])
][-1]
else:
first_release = [
v for v in vtags_data if v['name'] == first_release_version
][0]
first_release_data = _get_gh_rel_data(first_release, PREFIXES)
for k, v in first_release_data.items():
ret['first_release_%s' % k] = v
zv_releases = [
rel for rel in vtags_data
if match_vtag(rel['name'], PREFIXES).group('major') == '0'
]
ret['release_count_zv'] = len(zv_releases)
print ' .. %s releases, %s 0ver' % (ret['release_count'],
ret['release_count_zv'])
is_zerover = latest_release in zv_releases
ret['is_zerover'] = is_zerover
if is_zerover:
return ret
last_zv_release = zv_releases[0]
first_nonzv_release = vtags_data[vtags_data.index(last_zv_release) - 1]
first_nonzv_release_data = _get_gh_rel_data(first_nonzv_release, PREFIXES)
ret['last_zv_release_version'] = last_zv_release['name']
for k, v in first_nonzv_release_data.items():
ret['first_nonzv_release_%s' % k] = v
#import pdb;pdb.set_trace()
return ret
