def document_final_pass_before_toc(soup,
remove,
remove_selectors,
res=None,
location=None):
if res is None:
logger.warn('no res passed')
res = AugmentedResult()
if location is None:
location = LocationUnknown()
logger.info('reorganizing contents in <sections>')
with timeit('find body'):
body = soup.find('body')
if body is None:
msg = 'Cannot find <body>:\n%s' % indent(str(soup)[:1000], '|')
raise ValueError(msg)
with timeit('reorganize_contents'):
body2 = reorganize_contents(body)
process_assignment(body2, res, location)
body.replace_with(body2)
# Removing stuff
with timeit('remove stuff'):
do_remove_stuff(body2, remove_selectors, remove)
with timeit('move_things_around'):
move_things_around(soup=soup, res=res)
def fix_notes_assignees(soup, res):
id2element, duplicates = get_id2element(soup, 'id')
assert isinstance(res, AugmentedResult), type(res)
# logger.warn('here: %s' % len(res.notes))
for note in res.notes:
locations = note.locations
if len(locations) == 1:
location = list(locations.values())[0]
if isinstance(location, HTMLIDLocation):
ID = location.element_id
if ID in id2element:
element = id2element[ID]
has_assignees = any(
_.startswith('for:') for _ in note.tags)
if not has_assignees:
assignees = get_assignees_from_parents(element)
if assignees:
tags = list(note.tags)
for a in assignees:
tags.append('for:%s' % a)
note.tags = tuple(sorted(set(tags)))
else:
pass
# logger.warn('could not find assignees for %s' % ID)
else:
pass
logger.warn('could not find element %r' % ID)
def disk_event_dir_create_interpret(disk_rep, dirname, name):
d = get_dir(disk_rep, dirname)
if name in d:
msg = 'Cannot create directory "%s" that already exists' % name
if True:
logger.warn(msg) # FIXME
# else:
# raise InvalidDiskOperation(msg)
else:
d[name] = ProxyDirectory()
def __init__(self, tag, depth, name, _id, items):
self.tag = tag
self.name = name
self.depth = depth
self.id = _id
self.items = items
self.number = None
if ":" in self.id:
# Get "sub", "sec", "part", etc.
self.header_level = self.id.split(":")[0]
else:
self.header_level = 'unknown'
logger.warn(self.id)
def authenticate(self, login, password):
user_info = self.users[login].info
for p in user_info.authentication_ids:
if p.provider == 'password':
pwd = p.password
match = password == pwd
if not match:
msg = 'Password %s does not match with stored %s.' % (
password, pwd)
logger.warn(msg)
user_info.account_last_active = datetime.now()
return match
return False
def runTest(self):
# turn off access control for user list
MCDPResourceRoot.__acl__.append((Allow, Everyone, Privileges.ACCESS))
MCDPResourceRoot.__acl__.append(
(Allow, Everyone, Privileges.VIEW_USER_LIST))
MCDPResourceRoot.__acl__.append(
(Allow, Everyone, Privileges.VIEW_USER_PROFILE_PUBLIC))
if MCDPConstants.test_spider_exclude_images:
exclude = ['png', 'pdf', 'dot', 'svg', 'txt']
else:
exclude = []
ushelf = '/repos/bundled/shelves/%s' % another_name_for_unittests_shelf
bugs = [
ushelf + '/libraries/basic/models/sum2f_rcomp/views/solver',
ushelf +
'/libraries/pop/models/pop_example_3_7_newsyntax/views/ndp_repr/',
# this refers to a library that is not in this shelf
ushelf + '/libraries/making/models/test1/views/syntax/',
ushelf + '/libraries/documents/test_par.html',
ushelf + '/libraries/documents/test_subfigures.html',
]
for b in bugs:
self.testapp.get(b)
# this should not redirect
url = '/repos/bundled/shelves/%s/libraries/documents/align.html' % another_name_for_unittests_shelf
res = self.testapp.get(url)
if '302' in res.status:
msg = 'Document redirect: %s -> %s' % (url,
res.headers['location'])
msg += '\n' + indent(res.body, '> ')
raise Exception(msg)
# another test
_, res = self.get_maybe_follow('/tree/')
assert_not_contains(res.body, 'None')
# another test
_, res = self.get_maybe_follow('/repos/')
assert_not_contains(res.body, 'None')
_, res = self.get_maybe_follow('/')
assert_not_contains(res.body, 'function shelf')
def ignore(url, parsed): # @UnusedVariable
if url == 'http://localhost/authomatic':
return True
if 'confirm' in url:
return True
# > - http://localhost/confirm_creation_similar
# > - http://localhost/confirm_bind
# > - http://localhost/confirm_creation_create
# > - http://localhost/confirm_bind_bind
# > - http://localhost/confirm_creation
if ':' in parsed.path:
return True
if 'exit' in parsed.path: # skip actions
return True
if parsed.netloc and parsed.netloc != u'localhost':
return True
if 'solver' in parsed.path:
return True
for x in exclude:
if x in parsed.path: return True
return False
spider = Spider(self.get_maybe_follow, ignore=ignore)
spider.visit(ushelf + '/libraries/making/models/test1/views/syntax/')
spider.visit(ushelf + '/libraries/documents/test_subfigure.html')
spider.visit('/tree')
max_fails = 10
max_pages = 100
try:
spider.go(max_fails=max_fails, max_pages=max_pages)
except KeyboardInterrupt:
pass
spider.log_summary()
if spider.skipped:
for url in sorted(spider.skipped):
logger.warn('Skipped %s' % url)
if spider.failed or spider.not_found:
msg = ''
if spider.not_found:
msg += 'These URLs not found:'
for f, e in spider.not_found.items():
msg += '\n- %s' % f
if spider.failed:
msg += '\nErrors for these URLs:'
for f, e in spider.failed.items():
msg += '\n- %s' % f
msg += '\n referrers: \n' + "\n - ".join(
spider.referrers[f])
if False:
for f, e in spider.failed.items():
msg += '\n URL: ' + f
msg += '\n referrers: \n' + "\n - ".join(
spider.referrers[f])
body = e[e.index('<html'):]
s = project_html(body)
msg += '\n' + indent(s, ' > ')
# msg += '\n' + indent(str(e), ' > ')
# msg += '\n'.join('- %s' % _ for _ in sorted(spider.failed))
raise_desc(Exception, msg)