def search(self, query, page=None, detailed=False):
"""Sends a POST request and retrieves a list of applications matching
the query term(s).
:param query: search query term(s) to retrieve matching apps
:param page: the page number to retrieve. Max is 12.
:param detailed: if True, sends request per app for its full detail
:return: a list of apps matching search terms
"""
page = 0 if page is None else int(page)
if page > len(self._pagtok) - 1:
raise ValueError('Parameter \'page\' ({page}) must be between 0 and 12.'.format(
page=page))
pagtok = self._pagtok[page]
data = generate_post_data(0, 0, pagtok)
self.params.update({
'q': quote_plus(query),
'c': 'apps',
})
response = send_request('POST', self._search_url, data, self.params)
soup = BeautifulSoup(response.content, 'lxml', from_encoding='utf8')
if detailed:
apps = self._parse_multiple_apps(response)
else:
apps = [parse_card_info(app)
for app in soup.select('div[data-uitype=500]')]
return apps
def developer(self, developer, results=None, page=None, detailed=False):
if not isinstance(developer, basestring) or developer.isdigit():
raise ValueError(
'Parameter \'developer\' must be the developer name, not the developer id.'
)
results = s.DEV_RESULTS if results is None else results
page = 0 if page is None else page
page_num = (results // 20) * page
if not 0 <= page_num <= 12:
raise ValueError(
'Page out of range. (results // 20) * page must be between 0 - 12'
)
pagtok = self._pagtok[page_num]
url = build_url('developer', developer)
data = generate_post_data(results, 0, pagtok)
response = send_request('POST', url, data, self.params)
if detailed:
apps = self._parse_multiple_apps(response)
else:
soup = BeautifulSoup(response.content,
'lxml',
from_encoding='utf8')
apps = [
parse_card_info(app)
for app in soup.select('div[data-uitype="500"]')
]
return apps
def search(self, query, page=None, detailed=False):
"""Sends a POST request and retrieves a list of applications matching
the query term(s).
:param query: search query term(s) to retrieve matching apps
:param page: the page number to retrieve. Max is 12.
:param detailed: if True, sends request per app for its full detail
:return: a list of apps matching search terms
"""
page = 0 if page is None else int(page)
if page > len(self._pagtok) - 1:
raise ValueError(
"Parameter 'page' ({page}) must be between 0 and 12.".format(
page=page))
pagtok = self._pagtok[page]
data = generate_post_data(0, 0, pagtok)
self.params.update({"q": quote_plus(query), "c": "apps"})
response = send_request("POST", self._search_url, data, self.params)
soup = BeautifulSoup(response.content, "lxml", from_encoding="utf8")
if detailed:
apps = self._parse_multiple_apps(response)
else:
apps = [
parse_cluster_card_info(app)
for app in soup.select("div.Vpfmgd")
]
return apps
def search(self, query, page=None, detailed=False):
page = 0 if page is None else int(page)
if page > len(self._pagtok) - 1:
raise ValueError(
'Parameter \'page\' ({page}) must be between 0 and 12.'.format(
page=page))
pagtok = self._pagtok[page]
data = generate_post_data(0, 0, pagtok)
self.params.update({
'q': quote_plus(query),
'c': 'apps',
})
response = send_request('POST', self._search_url, data, self.params)
soup = BeautifulSoup(response.content, 'lxml', from_encoding='utf8')
if detailed:
apps = self._parse_multiple_apps(response)
else:
apps = [
parse_card_info(app)
for app in soup.select('div[data-uitype="500"]')
]
return apps
def search(self, query, page=None, detailed=False):
"""Sends a POST request and retrieves a list of applications matching
the query term(s).
:param query: search query term(s) to retrieve matching apps
:param page: the page number to retrieve. Max is 12.
:param detailed: if True, sends request per app for its full detail
:return: a list of apps matching search terms
"""
page = 0 if page is None else int(page)
if page > len(self._pagtok) - 1:
raise ValueError('Parameter \'page\' ({page}) must be between 0 and 12.'.format(
page=page))
pagtok = self._pagtok[page]
data = generate_post_data(0, 0, pagtok)
self.params.update({
'q': quote_plus(query),
'c': 'apps',
})
response = send_request('POST', self._search_url, data, self.params)
soup = BeautifulSoup(response.content, 'lxml', from_encoding='utf8')
if detailed:
apps = self._parse_multiple_apps(response)
else:
apps = [parse_card_info(app)
for app in soup.select('div[data-uitype="500"]')]
return apps
def developer(self, developer, results=None, page=None, detailed=False):
"""Sends a POST request and retrieves a list of the developer's
published applications on the Play Store.
:param developer: developer name to retrieve apps from, e.g. 'Disney'
:param results: the number of app results to retrieve
:param page: the page number to retrieve
:param detailed: if True, sends request per app for its full detail
:return: a list of app dictionaries
"""
if not isinstance(developer, basestring) or developer.isdigit():
raise ValueError('Parameter \'developer\' must be the developer name, not the developer id.')
results = s.DEV_RESULTS if results is None else results
page = 0 if page is None else page
page_num = (results // 20) * page
if not 0 <= page_num <= 12:
raise ValueError('Page out of range. (results // 20) * page must be between 0 - 12')
pagtok = self._pagtok[page_num]
url = build_url('developer', developer)
data = generate_post_data(results, 0, pagtok)
response = send_request('POST', url, data, self.params)
if detailed:
apps = self._parse_multiple_apps(response)
else:
soup = BeautifulSoup(response.content, 'lxml', from_encoding='utf8')
apps = [parse_card_info(app)
for app in soup.select('div[data-uitype="500"]')]
return apps
def developer(self, developer, results=None, page=None, detailed=False):
"""Sends a POST request and retrieves a list of the developer's
published applications on the Play Store.
:param developer: developer name to retrieve apps from, e.g. 'Disney'
:param results: the number of app results to retrieve
:param page: the page number to retrieve
:param detailed: if True, sends request per app for its full detail
:return: a list of app dictionaries
"""
if not isinstance(developer, basestring) or developer.isdigit():
raise ValueError('Parameter \'developer\' must be the developer name, not the developer id.')
results = s.DEV_RESULTS if results is None else results
page = 0 if page is None else page
page_num = (results // 20) * page
if not 0 <= page_num <= 12:
raise ValueError('Page out of range. (results // 20) * page must be between 0 - 12')
pagtok = self._pagtok[page_num]
url = build_url('developer', developer)
data = generate_post_data(results, 0, pagtok)
response = send_request('POST', url, data, self.params)
if detailed:
apps = self._parse_multiple_apps(response)
else:
soup = BeautifulSoup(response.content, 'lxml', from_encoding='utf8')
apps = [parse_card_info(app)
for app in soup.select('div[data-uitype=500]')]
return apps
def collection(
self,
collection_id,
category_id=None,
results=None,
page=None,
age=None,
detailed=False,
):
"""Sends a POST request and fetches a list of applications belonging to
the collection and an optional category.
:param collection_id: the collection id, e.g. 'NEW_FREE'.
:param category_id: (optional) the category id, e.g. 'GAME_ACTION'.
:param results: the number of apps to retrieve at a time.
:param page: page number to retrieve; limitation: page * results <= 500.
:param age: an age range to filter by (only for FAMILY categories)
:param detailed: if True, sends request per app for its full detail
:return: a list of app dictionaries
"""
if collection_id not in COLLECTIONS and not collection_id.startswith(
"promotion"):
raise ValueError("Invalid collection_id '{collection}'.".format(
collection=collection_id))
collection_name = COLLECTIONS.get(collection_id) or collection_id
category = "" if category_id is None else CATEGORIES.get(category_id)
if category is None:
raise ValueError("Invalid category_id '{category}'.".format(
category=category_id))
results = s.NUM_RESULTS if results is None else results
if results > 120:
raise ValueError("Number of results cannot be more than 120.")
page = 0 if page is None else page
if page * results > 500:
raise ValueError(
"Start (page * results) cannot be greater than 500.")
if category.startswith("FAMILY") and age is not None:
self.params["age"] = AGE_RANGE[age]
url = build_collection_url(category, collection_name)
data = generate_post_data(results, page)
response = send_request("POST", url, data, self.params)
if detailed:
apps = self._parse_multiple_apps(response)
else:
soup = BeautifulSoup(response.content,
"lxml",
from_encoding="utf8")
apps = [
parse_card_info(app_card)
for app_card in soup.select('div[data-uitype="500"]')
]
return apps
def test_first_page_data(self):
expected = {
"ipf": 1,
"xhr": 1,
"start": self.page * self.results,
"num": self.results,
}
self.assertEqual(generate_post_data(self.results, self.page), expected)
def test_first_page_data(self):
expected = {
'ipf': 1,
'xhr': 1,
'start': self.page * self.results,
'num': self.results
}
self.assertEqual(generate_post_data(self.results, self.page), expected)
def test_page_token(self):
expected = {
"ipf": 1,
"xhr": 1,
"start": 0,
"num": 0,
"pagTok": self.pag_tok
}
self.assertEqual(generate_post_data(0, 0, self.pag_tok), expected)
def test_page_token(self):
expected = {
'ipf': 1,
'xhr': 1,
'start': 0,
'num': 0,
'pagTok': self.pag_tok
}
self.assertEqual(generate_post_data(0, 0, self.pag_tok), expected)
async def send_request(self,
method,
url,
data=None,
params={},
allow_redirects=False):
req_args = dict(method=method,
url=url,
params=params,
data=utils.generate_post_data()
if not data and method == 'POST' else data,
allow_redirects=allow_redirects)
async with self._session.request(**req_args) as response:
response.raise_for_status()
return await response.text()
def collection(self, collection_id, category_id=None, results=None,
page=None, age=None, detailed=False):
"""Sends a POST request and fetches a list of applications belonging to
the collection and an optional category.
:param collection_id: the collection id, e.g. 'NEW_FREE'.
:param category_id: (optional) the category id, e.g. 'GAME_ACTION'.
:param results: the number of apps to retrieve at a time.
:param page: page number to retrieve; limitation: page * results <= 500.
:param age: an age range to filter by (only for FAMILY categories)
:param detailed: if True, sends request per app for its full detail
:return: a list of app dictionaries
"""
if (collection_id not in COLLECTIONS and
not collection_id.startswith('promotion')):
raise ValueError('Invalid collection_id \'{collection}\'.'.format(
collection=collection_id))
collection_name = COLLECTIONS.get(collection_id) or collection_id
category = '' if category_id is None else CATEGORIES.get(category_id)
if category is None:
raise ValueError('Invalid category_id \'{category}\'.'.format(
category=category_id))
results = s.NUM_RESULTS if results is None else results
if results > 120:
raise ValueError('Number of results cannot be more than 120.')
page = 0 if page is None else page
if page * results > 500:
raise ValueError('Start (page * results) cannot be greater than 500.')
if category.startswith('FAMILY') and age is not None:
self.params['age'] = AGE_RANGE[age]
url = build_collection_url(category, collection_name)
data = generate_post_data(results, page)
response = send_request('POST', url, data, self.params)
if detailed:
apps = self._parse_multiple_apps(response)
else:
soup = BeautifulSoup(response.content, 'lxml', from_encoding='utf8')
apps = [parse_card_info(app_card)
for app_card in soup.select('div[data-uitype="500"]')]
return apps
def collection(self,
collection_id,
category_id=None,
results=None,
page=None,
age=None,
detailed=False):
if (collection_id not in COLLECTIONS
and not collection_id.startswith('promotion')):
raise ValueError('Invalid collection_id \'{collection}\'.'.format(
collection=collection_id))
collection_name = COLLECTIONS.get(collection_id) or collection_id
category = '' if category_id is None else CATEGORIES.get(category_id)
if category is None:
raise ValueError('Invalid category_id \'{category}\'.'.format(
category=category_id))
results = s.NUM_RESULTS if results is None else results
if results > 120:
raise ValueError('Number of results cannot be more than 120.')
page = 0 if page is None else page
if page * results > 500:
raise ValueError(
'Start (page * results) cannot be greater than 500.')
if category.startswith('FAMILY') and age is not None:
self.params['age'] = AGE_RANGE[age]
url = build_collection_url(category, collection_name)
data = generate_post_data(results, page)
response = send_request('POST', url, data, self.params)
if detailed:
apps = self._parse_multiple_apps(response)
else:
soup = BeautifulSoup(response.content,
'lxml',
from_encoding='utf8')
apps = [
parse_card_info(app_card)
for app_card in soup.select('div[data-uitype="500"]')
]
return apps
async def search(self, token, results=None, page=0):
if page > MAX_PAGE_SIZE_FOR_SEARCH:
raise ValueError(
'Page value [{page}] must be between 0 and {page_limit}'.
format(page=page, page_limit=MAX_PAGE_SIZE_FOR_SEARCH))
url = settings.SEARCH_URL
params = dict(self._params, q=quote_plus(token), c='apps')
data = utils.generate_post_data(0,
0,
pagtok=settings.PAGE_TOKENS[page])
try:
response = await self.send_request('POST',
url,
data,
params=params)
soup = BeautifulSoup(response, 'lxml')
except ClientResponseError as e:
raise ValueError('INVALID_TOKEN: {app}. {error}'.format(
token=token, error=e))
apps = list(
map(utils.parse_card_info, soup.select('div[data-uitype="500"]')))
return prune_data(apps)
async def collection(self, coln_id, catg_id=None, results=None, page=None):
coln_name = coln_id if coln_id.startswith(
'promotion') else lists.COLLECTIONS.get(coln_id)
if coln_name is None:
raise ValueError(
'INVALID_COLLECTION_ID: {coln}'.format(coln=coln_id))
catg_name = '' if catg_id is None else lists.CATEGORIES.get(catg_id)
if catg_name is None:
raise ValueError(
'INVALID_CATEGORY_ID: {catg}'.format(catg=catg_id))
results = settings.NUM_RESULTS if results is None else results
if results > 120:
raise ValueError('Number of results cannot be more than 120.')
page = 0 if page is None else page
if page * results > 500:
raise ValueError(
'Start (page * results) cannot be greater than 500.')
url = utils.build_collection_url(catg_name, coln_name)
data = utils.generate_post_data(results, page)
try:
response = await self.send_request('POST',
url,
data,
params=self._params)
soup = BeautifulSoup(response, 'lxml')
except ClientResponseError as e:
raise ValueError(
'INVALID_COLLECTION_OR_CATEGORY_ID: {coln}; {catg} {error}'.
format(coln=coln_id, catg=catg_id, error=e))
apps = list(
map(utils.parse_card_info, soup.select('div[data-uitype="500"]')))
# TODO: soup parsing failing for certain scenarios
# $ref: Exception #1 @ observed_error.log
return prune_data(apps)
def test_only_num_results(self):
expected = {'ipf': 1, 'xhr': 1, 'num': self.results}
self.assertEqual(generate_post_data(self.results), expected)
def test_default_post_data(self):
expected = {'ipf': 1, 'xhr': 1}
self.assertEqual(generate_post_data(), expected)
def test_default_post_data(self):
expected = {"ipf": 1, "xhr": 1}
self.assertEqual(generate_post_data(), expected)
def test_only_num_results(self):
expected = {"ipf": 1, "xhr": 1, "num": self.results}
self.assertEqual(generate_post_data(self.results), expected)
