def search(self, query, page=None, detailed=False): """Sends a POST request and retrieves a list of applications matching the query term(s). :param query: search query term(s) to retrieve matching apps :param page: the page number to retrieve. Max is 12. :param detailed: if True, sends request per app for its full detail :return: a list of apps matching search terms """ page = 0 if page is None else int(page) if page > len(self._pagtok) - 1: raise ValueError('Parameter \'page\' ({page}) must be between 0 and 12.'.format( page=page)) pagtok = self._pagtok[page] data = generate_post_data(0, 0, pagtok) self.params.update({ 'q': quote_plus(query), 'c': 'apps', }) response = send_request('POST', self._search_url, data, self.params) soup = BeautifulSoup(response.content, 'lxml', from_encoding='utf8') if detailed: apps = self._parse_multiple_apps(response) else: apps = [parse_card_info(app) for app in soup.select('div[data-uitype=500]')] return apps
def developer(self, developer, results=None, page=None, detailed=False): if not isinstance(developer, basestring) or developer.isdigit(): raise ValueError( 'Parameter \'developer\' must be the developer name, not the developer id.' ) results = s.DEV_RESULTS if results is None else results page = 0 if page is None else page page_num = (results // 20) * page if not 0 <= page_num <= 12: raise ValueError( 'Page out of range. (results // 20) * page must be between 0 - 12' ) pagtok = self._pagtok[page_num] url = build_url('developer', developer) data = generate_post_data(results, 0, pagtok) response = send_request('POST', url, data, self.params) if detailed: apps = self._parse_multiple_apps(response) else: soup = BeautifulSoup(response.content, 'lxml', from_encoding='utf8') apps = [ parse_card_info(app) for app in soup.select('div[data-uitype="500"]') ] return apps
def search(self, query, page=None, detailed=False): """Sends a POST request and retrieves a list of applications matching the query term(s). :param query: search query term(s) to retrieve matching apps :param page: the page number to retrieve. Max is 12. :param detailed: if True, sends request per app for its full detail :return: a list of apps matching search terms """ page = 0 if page is None else int(page) if page > len(self._pagtok) - 1: raise ValueError( "Parameter 'page' ({page}) must be between 0 and 12.".format( page=page)) pagtok = self._pagtok[page] data = generate_post_data(0, 0, pagtok) self.params.update({"q": quote_plus(query), "c": "apps"}) response = send_request("POST", self._search_url, data, self.params) soup = BeautifulSoup(response.content, "lxml", from_encoding="utf8") if detailed: apps = self._parse_multiple_apps(response) else: apps = [ parse_cluster_card_info(app) for app in soup.select("div.Vpfmgd") ] return apps
def search(self, query, page=None, detailed=False): page = 0 if page is None else int(page) if page > len(self._pagtok) - 1: raise ValueError( 'Parameter \'page\' ({page}) must be between 0 and 12.'.format( page=page)) pagtok = self._pagtok[page] data = generate_post_data(0, 0, pagtok) self.params.update({ 'q': quote_plus(query), 'c': 'apps', }) response = send_request('POST', self._search_url, data, self.params) soup = BeautifulSoup(response.content, 'lxml', from_encoding='utf8') if detailed: apps = self._parse_multiple_apps(response) else: apps = [ parse_card_info(app) for app in soup.select('div[data-uitype="500"]') ] return apps
def search(self, query, page=None, detailed=False): """Sends a POST request and retrieves a list of applications matching the query term(s). :param query: search query term(s) to retrieve matching apps :param page: the page number to retrieve. Max is 12. :param detailed: if True, sends request per app for its full detail :return: a list of apps matching search terms """ page = 0 if page is None else int(page) if page > len(self._pagtok) - 1: raise ValueError('Parameter \'page\' ({page}) must be between 0 and 12.'.format( page=page)) pagtok = self._pagtok[page] data = generate_post_data(0, 0, pagtok) self.params.update({ 'q': quote_plus(query), 'c': 'apps', }) response = send_request('POST', self._search_url, data, self.params) soup = BeautifulSoup(response.content, 'lxml', from_encoding='utf8') if detailed: apps = self._parse_multiple_apps(response) else: apps = [parse_card_info(app) for app in soup.select('div[data-uitype="500"]')] return apps
def developer(self, developer, results=None, page=None, detailed=False): """Sends a POST request and retrieves a list of the developer's published applications on the Play Store. :param developer: developer name to retrieve apps from, e.g. 'Disney' :param results: the number of app results to retrieve :param page: the page number to retrieve :param detailed: if True, sends request per app for its full detail :return: a list of app dictionaries """ if not isinstance(developer, basestring) or developer.isdigit(): raise ValueError('Parameter \'developer\' must be the developer name, not the developer id.') results = s.DEV_RESULTS if results is None else results page = 0 if page is None else page page_num = (results // 20) * page if not 0 <= page_num <= 12: raise ValueError('Page out of range. (results // 20) * page must be between 0 - 12') pagtok = self._pagtok[page_num] url = build_url('developer', developer) data = generate_post_data(results, 0, pagtok) response = send_request('POST', url, data, self.params) if detailed: apps = self._parse_multiple_apps(response) else: soup = BeautifulSoup(response.content, 'lxml', from_encoding='utf8') apps = [parse_card_info(app) for app in soup.select('div[data-uitype="500"]')] return apps
def developer(self, developer, results=None, page=None, detailed=False): """Sends a POST request and retrieves a list of the developer's published applications on the Play Store. :param developer: developer name to retrieve apps from, e.g. 'Disney' :param results: the number of app results to retrieve :param page: the page number to retrieve :param detailed: if True, sends request per app for its full detail :return: a list of app dictionaries """ if not isinstance(developer, basestring) or developer.isdigit(): raise ValueError('Parameter \'developer\' must be the developer name, not the developer id.') results = s.DEV_RESULTS if results is None else results page = 0 if page is None else page page_num = (results // 20) * page if not 0 <= page_num <= 12: raise ValueError('Page out of range. (results // 20) * page must be between 0 - 12') pagtok = self._pagtok[page_num] url = build_url('developer', developer) data = generate_post_data(results, 0, pagtok) response = send_request('POST', url, data, self.params) if detailed: apps = self._parse_multiple_apps(response) else: soup = BeautifulSoup(response.content, 'lxml', from_encoding='utf8') apps = [parse_card_info(app) for app in soup.select('div[data-uitype=500]')] return apps
def collection( self, collection_id, category_id=None, results=None, page=None, age=None, detailed=False, ): """Sends a POST request and fetches a list of applications belonging to the collection and an optional category. :param collection_id: the collection id, e.g. 'NEW_FREE'. :param category_id: (optional) the category id, e.g. 'GAME_ACTION'. :param results: the number of apps to retrieve at a time. :param page: page number to retrieve; limitation: page * results <= 500. :param age: an age range to filter by (only for FAMILY categories) :param detailed: if True, sends request per app for its full detail :return: a list of app dictionaries """ if collection_id not in COLLECTIONS and not collection_id.startswith( "promotion"): raise ValueError("Invalid collection_id '{collection}'.".format( collection=collection_id)) collection_name = COLLECTIONS.get(collection_id) or collection_id category = "" if category_id is None else CATEGORIES.get(category_id) if category is None: raise ValueError("Invalid category_id '{category}'.".format( category=category_id)) results = s.NUM_RESULTS if results is None else results if results > 120: raise ValueError("Number of results cannot be more than 120.") page = 0 if page is None else page if page * results > 500: raise ValueError( "Start (page * results) cannot be greater than 500.") if category.startswith("FAMILY") and age is not None: self.params["age"] = AGE_RANGE[age] url = build_collection_url(category, collection_name) data = generate_post_data(results, page) response = send_request("POST", url, data, self.params) if detailed: apps = self._parse_multiple_apps(response) else: soup = BeautifulSoup(response.content, "lxml", from_encoding="utf8") apps = [ parse_card_info(app_card) for app_card in soup.select('div[data-uitype="500"]') ] return apps
def test_first_page_data(self): expected = { "ipf": 1, "xhr": 1, "start": self.page * self.results, "num": self.results, } self.assertEqual(generate_post_data(self.results, self.page), expected)
def test_first_page_data(self): expected = { 'ipf': 1, 'xhr': 1, 'start': self.page * self.results, 'num': self.results } self.assertEqual(generate_post_data(self.results, self.page), expected)
def test_page_token(self): expected = { "ipf": 1, "xhr": 1, "start": 0, "num": 0, "pagTok": self.pag_tok } self.assertEqual(generate_post_data(0, 0, self.pag_tok), expected)
def test_page_token(self): expected = { 'ipf': 1, 'xhr': 1, 'start': 0, 'num': 0, 'pagTok': self.pag_tok } self.assertEqual(generate_post_data(0, 0, self.pag_tok), expected)
async def send_request(self, method, url, data=None, params={}, allow_redirects=False): req_args = dict(method=method, url=url, params=params, data=utils.generate_post_data() if not data and method == 'POST' else data, allow_redirects=allow_redirects) async with self._session.request(**req_args) as response: response.raise_for_status() return await response.text()
def collection(self, collection_id, category_id=None, results=None, page=None, age=None, detailed=False): """Sends a POST request and fetches a list of applications belonging to the collection and an optional category. :param collection_id: the collection id, e.g. 'NEW_FREE'. :param category_id: (optional) the category id, e.g. 'GAME_ACTION'. :param results: the number of apps to retrieve at a time. :param page: page number to retrieve; limitation: page * results <= 500. :param age: an age range to filter by (only for FAMILY categories) :param detailed: if True, sends request per app for its full detail :return: a list of app dictionaries """ if (collection_id not in COLLECTIONS and not collection_id.startswith('promotion')): raise ValueError('Invalid collection_id \'{collection}\'.'.format( collection=collection_id)) collection_name = COLLECTIONS.get(collection_id) or collection_id category = '' if category_id is None else CATEGORIES.get(category_id) if category is None: raise ValueError('Invalid category_id \'{category}\'.'.format( category=category_id)) results = s.NUM_RESULTS if results is None else results if results > 120: raise ValueError('Number of results cannot be more than 120.') page = 0 if page is None else page if page * results > 500: raise ValueError('Start (page * results) cannot be greater than 500.') if category.startswith('FAMILY') and age is not None: self.params['age'] = AGE_RANGE[age] url = build_collection_url(category, collection_name) data = generate_post_data(results, page) response = send_request('POST', url, data, self.params) if detailed: apps = self._parse_multiple_apps(response) else: soup = BeautifulSoup(response.content, 'lxml', from_encoding='utf8') apps = [parse_card_info(app_card) for app_card in soup.select('div[data-uitype="500"]')] return apps
def collection(self, collection_id, category_id=None, results=None, page=None, age=None, detailed=False): if (collection_id not in COLLECTIONS and not collection_id.startswith('promotion')): raise ValueError('Invalid collection_id \'{collection}\'.'.format( collection=collection_id)) collection_name = COLLECTIONS.get(collection_id) or collection_id category = '' if category_id is None else CATEGORIES.get(category_id) if category is None: raise ValueError('Invalid category_id \'{category}\'.'.format( category=category_id)) results = s.NUM_RESULTS if results is None else results if results > 120: raise ValueError('Number of results cannot be more than 120.') page = 0 if page is None else page if page * results > 500: raise ValueError( 'Start (page * results) cannot be greater than 500.') if category.startswith('FAMILY') and age is not None: self.params['age'] = AGE_RANGE[age] url = build_collection_url(category, collection_name) data = generate_post_data(results, page) response = send_request('POST', url, data, self.params) if detailed: apps = self._parse_multiple_apps(response) else: soup = BeautifulSoup(response.content, 'lxml', from_encoding='utf8') apps = [ parse_card_info(app_card) for app_card in soup.select('div[data-uitype="500"]') ] return apps
async def search(self, token, results=None, page=0): if page > MAX_PAGE_SIZE_FOR_SEARCH: raise ValueError( 'Page value [{page}] must be between 0 and {page_limit}'. format(page=page, page_limit=MAX_PAGE_SIZE_FOR_SEARCH)) url = settings.SEARCH_URL params = dict(self._params, q=quote_plus(token), c='apps') data = utils.generate_post_data(0, 0, pagtok=settings.PAGE_TOKENS[page]) try: response = await self.send_request('POST', url, data, params=params) soup = BeautifulSoup(response, 'lxml') except ClientResponseError as e: raise ValueError('INVALID_TOKEN: {app}. {error}'.format( token=token, error=e)) apps = list( map(utils.parse_card_info, soup.select('div[data-uitype="500"]'))) return prune_data(apps)
async def collection(self, coln_id, catg_id=None, results=None, page=None): coln_name = coln_id if coln_id.startswith( 'promotion') else lists.COLLECTIONS.get(coln_id) if coln_name is None: raise ValueError( 'INVALID_COLLECTION_ID: {coln}'.format(coln=coln_id)) catg_name = '' if catg_id is None else lists.CATEGORIES.get(catg_id) if catg_name is None: raise ValueError( 'INVALID_CATEGORY_ID: {catg}'.format(catg=catg_id)) results = settings.NUM_RESULTS if results is None else results if results > 120: raise ValueError('Number of results cannot be more than 120.') page = 0 if page is None else page if page * results > 500: raise ValueError( 'Start (page * results) cannot be greater than 500.') url = utils.build_collection_url(catg_name, coln_name) data = utils.generate_post_data(results, page) try: response = await self.send_request('POST', url, data, params=self._params) soup = BeautifulSoup(response, 'lxml') except ClientResponseError as e: raise ValueError( 'INVALID_COLLECTION_OR_CATEGORY_ID: {coln}; {catg} {error}'. format(coln=coln_id, catg=catg_id, error=e)) apps = list( map(utils.parse_card_info, soup.select('div[data-uitype="500"]'))) # TODO: soup parsing failing for certain scenarios # $ref: Exception #1 @ observed_error.log return prune_data(apps)
def test_only_num_results(self): expected = {'ipf': 1, 'xhr': 1, 'num': self.results} self.assertEqual(generate_post_data(self.results), expected)
def test_default_post_data(self): expected = {'ipf': 1, 'xhr': 1} self.assertEqual(generate_post_data(), expected)
def test_default_post_data(self): expected = {"ipf": 1, "xhr": 1} self.assertEqual(generate_post_data(), expected)
def test_only_num_results(self): expected = {"ipf": 1, "xhr": 1, "num": self.results} self.assertEqual(generate_post_data(self.results), expected)