def reconstruct(self, simptcs, detector):
self.pfinput = PFInput(simptcs)
elements = self.pfinput.element_list()
elements = merge_clusters(elements, 'hcal_in')
elements = merge_clusters(elements, 'ecal_in')
self.links = Links(elements, distance)
self.pfreco = PFReconstructor(self.links, detector, self.logger)
def test_link_1(self):
elements = map(TestElement, range(10))
links = Links(elements, distance)
distances = links.values()
self.assertTrue(max(distances) == 2)
self.assertEqual(elements[0].linked, [elements[1], elements[2]])
self.assertEqual(links.info(elements[2], elements[4]), 2)
self.assertIsNone(links.info(elements[2], elements[5]), None)
def visited_domains():
try:
frm, to = get_visited_domains_params(request.args, ("from", "to"))
except Exception as e:
return make_json_response(error=e, code=HTTPStatus.BAD_REQUEST)
try:
data = redis_connector.get_by_range(frm, to)
except Exception as e:
return make_json_response(status=ERROR_INTERNAL, code=HTTPStatus.INTERNAL_SERVER_ERROR)
links = Links(data)
return make_json_response(key=DOMAINS_KEY, data=links.get_domains())
def get_all_asset_links(zettel_paths):
all_Links = Links()
# For each zettel.
for zettel_path in zettel_paths:
# Get the contents of the zettel.
with open(zettel_path, 'r', encoding='utf8') as file:
contents = file.read()
# Get the asset links in the contents and save them
# with all the other links in the Links object.
links = get_asset_links(contents, zettel_path)
all_Links.add(links)
return all_Links
def get_asset_links(contents, zettel_path):
assert os.path.isabs(zettel_path) or zettel_path == ''
links = Links()
for link_match in re.finditer(asset_link_pattern, contents):
link_dict = link_match.groupdict()
# URLs that end with '.html' or '.htm' could be in the list.
# Ignore them, but not locally saved files with those endings.
if link_dict['link'].endswith('.html') or link_dict['link'].endswith(
'.htm'):
if html_link_is_URL(link_dict['link']):
continue
links.append(link_dict['link'], link_dict['name'], zettel_path)
return links
def _enhance_element_tree(self, e):
for element in e.getiterator():
for child in list(element):
if len(element.findall(child.tag)) > 1:
setattr(element, child.tag, element.findall(child.tag))
elif len(list(child)) == 0:
setattr(element, child.tag, child.text)
else:
setattr(element, child.tag, element.find(child.tag))
l = []
for element in e.getiterator('link'):
d = {
'href': element.attrib.get('href'),
'rel': element.attrib.get('rel'),
'type': element.attrib.get('type') or 'application/xml'
}
l.append(d)
e.links = lambda: Links(l)
e.link = lambda x: e.links().get(x)
return e
def merge_clusters(elements, layer):
merged = []
elem_in_layer = []
elem_other = []
for elem in elements:
if elem.layer == layer:
elem_in_layer.append(elem)
else:
elem_other.append(elem)
links = Links(elem_in_layer, distance)
for group in links.groups.values():
if len(group) == 1:
merged.append(group[0])
continue
supercluster = None
for cluster in group:
if supercluster is None:
supercluster = copy.copy(cluster)
merged.append(supercluster)
continue
else:
supercluster += cluster
merged.extend(elem_other)
return merged
def download(self, keyword):
source = Links()
print('Collecting downloadable links of {}...'.format(keyword))
links = source.collect(keyword)
print(
'Downloading images of {} from collected links...'.format(keyword))
self.mkdir('{}/{}'.format(self.path, keyword))
n_links = len(links)
for index, link in enumerate(links):
try:
print(
'Downloading this image based on the keyword {} from {}: {}/{}'
.format(keyword, link, index + 1, n_links))
response = requests.get(link, stream=True)
ext = self.get_extension(link)
raw_path = '{}/{}/{}'.format(self.path, keyword,
str(index).zfill(4))
path = raw_path + '.' + ext
self.save(response, path)
del response
print("Validating image file")
ext2 = self.validate(path)
if ext2 is None:
print('Unreadable file - {}'.format(link))
os.remove(path)
else:
if ext != ext2:
path2 = raw_path + '.' + ext2
os.rename(path, path2)
print('Renaming extension {} -> {}'.format(ext, ext2))
except Exception as e:
print('Download failed.', e)
continue
def main(args):
global links
links = Links()
app.run(host='0.0.0.0', debug=args.debug, port=args.port)
def links(self):
r = self._link_header_to_array()
return Links(r)
def __init__(self, driver, timeout=10):
self.driver = driver
self.driver.implicitly_wait(timeout)
self.links = Links()
def setup(self):
self.links = Links()
self.main_page = MainPage(self.driver)
self.data = Data()
self.driver.get(self.links.landing)
def links(self):
"""
Returns the Links of the header
"""
r = self._link_header_to_array()
return Links(r)