def get():
"""
Processes the user inputs and returns tests or lists to process
:return:
"""
args = arguments()
tests = []
resources = []
report_name = ''
if args.plan:
tests = test.get(plan=args.plan)
report_name = 'plan ' + args.plan
elif args.test:
tests = test.get(name=args.test)
report_name = 'test ' + args.test
elif args.tag:
tests = test.get(tag=args.tag)
report_name = 'dynamic'
elif args.list:
if args.list == 'plan':
resources = plan.get()
elif args.list == 'test':
resources = test.get()
elif args.list == 'tag':
resources = tag.get()
else:
tests = test.get()
report_name = 'all'
return [tests, report_name, resources, args.list]
def ishostAllowed(host):
if host.split('.')[-1].isdigit():
thread_logger.warn("Invalid host:".format(host), extra=req)
return False
#pdb.set_trace()
tags = tag_store.get(host)
if not tags:
thread_logger.warn("{0} isn't allowed: empty tags".format(host),
extra=req)
return False
for tag in tag_store.get(host):
if not rule.isTagAllowed(tag):
thread_logger.warn("{0}:{1} isn't allowed".format(host, tag),
extra=req)
return False
return True
def search(self, keyword, mode='all'):
"""search using given keyword and mode.
:param keyword: keyword to search.
:param mode: mode used to search.
:type keyword: str
:type mode: str
:return: list of found media/object.
:rtype: list
"""
self._check_search_input(keyword)
# anime search can received empty keyword but can't 1 or 2 characters.
# so use the min, max limit used by mode 'all'
if mode == 'anime':
return self.search_anime(keyword)
elif mode == 'manga':
return self.search_manga(keyword)
# the query have following format for each mode:
query_dict = {
'all': 'search/all?q={query}',
}
not_implemented_mode_dict = {
# already implemented
# 'anime': 'anime.php?q={query}',
# 'manga': 'manga.php?q={query}',
# not yet implemented.
'character': 'character.php?q={query}',
'people': 'people.php?q={query}',
'clubs': 'clubs.php?action=find&cn={query}',
'users': 'users.php?q={query}',
# no object/class created for this search.
'news': 'news/search?q={query}',
'featured': 'featured/search?q={query}',
'forum': 'forum/?action=search&u=&uloc=1&loc=-1&q={query}',
}
# check mode
if mode not in query_dict and mode not in not_implemented_mode_dict:
raise ValueError('Search mode is not available.')
elif mode in not_implemented_mode_dict:
raise NotImplementedError('"{}" category search is not yet implemented.'.format(mode))
url = 'https://myanimelist.net'
search_page_url = '/'.join([url, query_dict[mode]])
search_page_url = search_page_url.format(**{'query': keyword})
search_page = self.session.get(search_page_url).text
html_soup = utilities.get_clean_dom(search_page, fix_html=False)
result = []
categories = ['characters', 'anime', 'manga', 'people', 'clubs']
disallowed_url_part = [
'myanimelist.net/topanime.php',
'myanimelist.net/login',
'/login.php',
]
for catg in categories:
article = html_soup.select_one('#{}'.format(catg)).find_next('article')
if catg == 'clubs':
article_divs = [x for x in article.select('div') if x.select_one('a')]
a_tags = [
x.select_one('a') for x in article_divs if x.select_one('a').get('href')]
links = [x.get('href') for x in a_tags]
a_tags_result = list(map(self.load_from_url, links))
result.extend(a_tags_result)
else:
a_tags = article.select('.information a')
# find all link to correct object.
a_tags_result = []
for tag in a_tags:
link = tag.get('href')
# pass the login link
is_skipped_url = any(x in link for x in disallowed_url_part)
if is_skipped_url:
continue
a_tags_result.append(self.load_from_url(link))
# fix the bug on when parsing manga on search page.
# it is caused by unclosed a tag on 'article > div > div.information > div'
if catg == 'manga' and len(a_tags_result) == 1:
a_tags_hrefs = [x.get('href') for x in html_soup.select('a') if x.get('href')]
manga_link = [x for x in a_tags_hrefs if 'myanimelist.net/manga/' in x]
a_tags_result = list(map(self.load_from_url, manga_link))
result.extend(a_tags_result)
# parse club for all mode search
return list(set(result))
def trigger(self, event, **kwargs):
for tagName in self.tags:
t = tag.get(tagName)
if t and event in t.events:
t.events[event](self, kwargs)
