def find_urls(self, text: typing.Optional[str]) -> typing.Iterable[str]:
if text is None:
return
detect_raw_links = self.config.detect_raw_links
valid_schemes = self.config.valid_link_protocols
ban_secrets = self.config.ignore_links_with_secrets
if detect_raw_links:
valid_schemes.add(None)
for url in self._url_extractor.find_urls(text):
link: uri.URI = uri.URI(url)
scheme_name = None if link.scheme is None else link.scheme.name
if scheme_name not in valid_schemes:
continue
if ban_secrets:
if link.user is not None \
or link.username is not None\
or link.password is not None:
continue
yield url
def test_generate_minimal(self):
img_id = uuid.UUID('03012ba3-086c-4604-bd6a-aa3e1a78f389')
file = uri.URI("test.png")
minimal_xml = xm.generate_xml(
ImageMetadata(img_id, file, None, None, None, None))
expected = '<image id="{i}" file="{f}" />'.format(i=img_id, f=file)
self.assertEqual(expected, minimal_xml)
def _prepare_request(self, url, method, accept) :
cur_uri = uri.URI(self.base_uri, path = url)
s = self._get_session()
response = s.request(method = method, url = cur_uri, stream = True,
headers={'Accept': accept})
self.__treat_response_error(response)
return response
def _prepare_post_raw_request(self, url, body, accept, contentType) :
cur_uri = uri.URI(self.base_uri, path = url)
headers = {'Accept': accept,
'Content-type' : contentType}
s = self._get_session()
response = s.post(url = cur_uri, data = body, headers=headers)
self.__treat_response_error(response)
return response
def test_without_optional_elements(self):
img_id = uuid.UUID('03012ba3-086c-4604-bd6a-aa3e1a78f389')
file = uri.URI("test.png")
self.assertEqual(
ImageMetadata(img_id, file, None, None, None, None),
xm.parse_xml(
xm.generate_xml(
ImageMetadata(img_id, file, None, None, None, None))))
def _prepare_post_file_request(self, url, file_path, accept, contentType) :
cur_uri = uri.URI(self.base_uri, path = url)
headers = {'Accept': accept,
'Content-type' : contentType}
s = self._get_session()
with open(file_path, 'rb') as f :
response = s.post(url = cur_uri, data = f, headers=headers)
self.__treat_response_error(response)
return response
def _download_file(self, url, dst_path = './') :
cur_uri = uri.URI(self.base_uri, path = url)
s = self._get_session()
with s.get(cur_uri, stream = True) as r:
d = r.headers['content-disposition']
filename = re.findall("filename=(.+);", d)[0]
filepath = os.path.join(dst_path, filename)
with open(filepath, 'wb') as f :
shutil.copyfileobj(r.raw, f)
return
def __init__(self, cert_file, cert_pass, port = NetworkPredefinedPorts.MainNet.value, address = 'localhost', verify_ca=True) :
if port is None :
port = NetworkPredefinedPorts.MainNet.value
self.verify_ca = verify_ca
self.base_uri = uri.URI(f'https://{address}:{port}/')
#self.__certificate = self.__get_cert_from_file(cert_file, cert_pass)
self.__certificate = PKCS12Certificate(cert_file, cert_pass)
self.network = RestNetwork(self)
self._session = None
self.__pem_file = None
def setUp(self):
"""Definine our test data with most of the common use cases"""
# Scheme, host, path
self.ftp_case = uri.URI.parse_uri('ftp://ftp.is.co.za/rfc/rfc1808.txt')
# Scheme, ipv6, path, query
self.ldap_case = uri.URI.parse_uri(
'ldap://[2001:db8::7]/c=GB?objectClass?one')
# Path, nothing else
self.news_case = uri.URI.parse_uri(
"news:comp.infosystems.www.servers.unix")
# Path with scheme
self.urn_case = uri.URI.parse_uri(
"urn:oasis:names:specification:docbook:dtd:xml:4.1.2")
# Path with scheme and port
self.telnet_case = uri.URI.parse_uri("telnet://192.0.2.16:80/")
# Scheme, authority, path, query, fragment
self.foo_case = uri.URI.parse_uri(
"foo://[email protected]:8042/over/there?name=ferret#nose")
self.sld_case = uri.URI.parse_uri(
"http://www.amazon.co.uk/search/product_page?id=52342")
# Scheme, host, path, query, fragment
self.gmail_case = uri.URI.parse_uri(
'https://www.google.com/search?q=setter+python&oq=setter+python&aqs=chrome..69i57j0l3.9438j0&sourceid=chrome&ie=UTF-8'
)
# Same as the above, but from direct initialization
self.gmail_case_from_init = uri.URI(
path='/search',
scheme='https',
authority='www.google.com',
query=
'q=setter+python&oq=setter+python&aqs=chrome..69i57j0l3.9438j0&sourceid=chrome&ie=UTF-8'
)
# Pure path with dot segments
self.dot_case = uri.URI(path='/a/b/c/./../../g')
# Another pure path with dot segments
self.dot_case2 = uri.URI(path='mid/content=5/../6')
# A more complete IPV6 sample, with a port
self.sample_ipv6_case = uri.URI.parse_uri(
'http://[FEDC:BA98:7654:3210:FEDC:BA98:7654:3210]:80/index.html')
# An example with a percent-encoded authority
self.percent_host = uri.URI(authority='herp%2fballoon', scheme='ftP')
def get_sources_list(source_stream):
from visplay.sources import source_constructors as sc
all_sources = []
sources_yaml = yaml.load(source_stream)
# Handle imports
if 'import' in sources_yaml and sources_yaml['import']:
for name in sources_yaml['import']:
source = sources_yaml['import'][name]
path = uri.URI(source)
new_source = sc[str(path.scheme)](name, path, is_import=True)
all_sources.append(new_source)
if 'add' in sources_yaml and sources_yaml['add']:
# calls the corresponding source constructor (LocalSource/HTTPSource)
# with the arguments provided, then appends to the list
for source in sources_yaml['add']:
path = uri.URI(source)
new_source = sc[str(path.scheme)](source, path)
all_sources.append(new_source)
return all_sources
def test_with_optional_elements(self):
img_id = uuid.UUID('03012ba3-086c-4604-bd6a-aa3e1a78f389')
file = uri.URI("test.png")
author = "John Doe"
universe = "Example"
characters = ["M", "Q"]
tags = ["a", "bee"]
self.assertEqual(
ImageMetadata(img_id, file, author, universe, characters, tags),
xm.parse_xml(
xm.generate_xml(
ImageMetadata(img_id, file, author, universe, characters,
tags))))
def _prepare_post_request(self, url, body, accept) :
cur_uri = uri.URI(self.base_uri, path = url)
if issubclass(type(body) ,BaseModel) :
json_data = body.json()
else :
json_data = BaseModel.to_json(body)
headers = {'Accept': accept,
'Content-type' : "application/json; charset=utf-8"}
s = self._get_session()
response = s.post(url = cur_uri, headers=headers, json = json_data)
self.__treat_response_error(response)
return response
def test_changedir(self):
"""
Test that we can change directories
"""
self.dot_case.chdir('.')
self.assertEqual(self.dot_case.__repr__(), '/a/b/c/./../../g/.')
self.dot_case.chdir('..')
self.assertEqual(self.dot_case.__repr__(), '/a/b/c/./../../g/./..')
self.dot_case = uri.URI(path='/a/b/c/./../../g')
self.dot_case.chdir('./')
self.assertEqual(self.dot_case.__repr__(), '/a/b/c/./../../g/./')
self.dot_case.chdir('/../')
self.assertEqual(self.dot_case.__repr__(), '/a/b/c/./../../g/./../')
def test_generate_single_element_and_subtree(self):
img_id = uuid.UUID('03012ba3-086c-4604-bd6a-aa3e1a78f389')
file = uri.URI("test.png")
universe = "unknown"
tags = ["a", "b"]
new_xml = xm.generate_xml(
ImageMetadata(img_id, file, None, universe, None, tags))
expected = ('<image id="{i}" file="{f}">' +
'<universe>{u}</universe>' +
'<tags><tag>{t1}</tag><tag>{t2}</tag></tags>' +
'</image>').format(i=img_id,
f=file,
u=universe,
t1=tags[0],
t2=tags[1])
self.assertEqual(expected, new_xml)
def resolve_link(templated_url, params):
template = URITemplate(templated_url)
variables = template.variables
# variables = template.parts.reduce((accumulator, part) => {
# if not r.is_empty(part.variables):
# return r.concat(part.variables.map(variable => variable.name), accumulator)
#
# return accumulator
# }, [])
url = template.expand(params)
query_object = uri.URI(url)
full_params = params
# full_params = r.merge(r.omit(variables, params), query_object)
# url_without_query_string = URI(url).query('').toString()
return {
'href': url,
'params': full_params
}
def __extractData(self, bloburi):
blobPath = str(uri.URI(bloburi).path).replace('/raw/', '')
fileio = self.__readFileFromDataLake(blobPath)
# taa5 format can be extracted following:
# skip row 0, make row 1 the header, skip rows 2 and 3 (metadata)
station_id_str = {self.config['postDataMap']['station_id']: str}
meta = pd.read_csv(io.BytesIO(fileio),
encoding="utf8",
nrows=1,
header=None)
towerId = meta[1][0].split("_")[1]
self.__stationId = self.__getStationId(towerId)
return pd.read_csv(io.BytesIO(fileio),
encoding="utf8",
dtype=station_id_str,
header=[1],
skiprows=[2, 3])
def test_parse_legacy(self):
xml_string_legacy = "<image id=\"03012ba3-086c-4604-bd6a-aa3e1a78f389\" filename=\"tt.png\"></image>"
metadata = ImageMetadata(
uuid.UUID('03012ba3-086c-4604-bd6a-aa3e1a78f389'),
uri.URI("tt.png"), None, None, None, None)
self.assertEqual(metadata, xm.parse_xml(xml_string_legacy))
