def discover_apps(self, subcategory, app_list=None):
"""Given a subcategory, discovers all apps contained therein.
Parameters
----------
subcategory : SubCategory
The SubCategory to search.
app_list : AppList
Only there to ensure PlayStore compatibility.
Returns
-------
AppList
An AppList object containing the discovered apps.
"""
LOGGER.info(f"Discovering apps for {subcategory.parent.name()} - {subcategory.name()}")
wrapper = Proto.ResponseWrapper()
if app_list is not None:
raise Maximum
payload = Proto.Payload()
list_response = Proto.ListResponse()
doc = list_response.doc.add()
child = doc.child.add()
response = get(f'{SERVER}{subcategory.data()}')
if response.status_code != 200:
raise RequestError(response.reason)
soup = BeautifulSoup(response.content.decode(), 'html.parser')
package_list = soup.find(id='package-list')
packages = package_list.find_all(name='a')
next_pages = package_list.find_all(class_='nav page')
for next_page in next_pages:
next_response = get(f'{SERVER}{next_page.find(name="a").attrs["href"]}')
if next_response.status_code != 200:
continue
next_package_list = BeautifulSoup(next_response.content.decode(), 'html.parser').find(id='package-list')
next_packages = next_package_list.find_all(name='a')
packages += next_packages
for package in packages:
try:
details = self._get_details(package.attrs['href'])
except (RequestError, ConnectionError) as e:
LOGGER.exception(e)
continue
except CategoryError:
LOGGER.debug(f"{package.attrs['href']} is a category, not a package.")
continue
sub_child = child.child.add()
sub_child.docid = details.docid
sub_child.title = details.title
sub_child.creator = details.creator
sub_child.descriptionShort = details.descriptionShort
sub_child.details.CopyFrom(details.details)
sub_child.descriptionHtml = details.descriptionHtml
payload.listResponse.CopyFrom(list_response)
wrapper.payload.CopyFrom(payload)
LOGGER.debug(wrapper)
return AppList(wrapper, subcategory, self)
def _get_details(self, path):
"""Given a partial path on the F-Droid.org server, queries the server for the application details.
Parameters
----------
path : str
A partial path on the F-Droid server. Can be automatically generated by discover_apps.
Returns
-------
Proto.DocV2
The protobuf representation of the package details.
"""
if "/categories/" in path:
raise CategoryError
try:
LOGGER.debug(f'{SERVER}{path}')
response = get(f'{SERVER}{path}')
LOGGER.debug(response.url)
except ConnectionError:
raise RequestError(f'\n\tUrl:\t{SERVER}{path}')
if response.status_code != 200:
raise RequestError(f'\n\tReason:\t{response.reason}\n\tCode:{response.status_code}'
f'\n\tUrl:\n{response.url}')
package = BeautifulSoup(response.content.decode(), 'html.parser').find(class_='package')
title = package.find(class_='package-name').text.strip()
creator = 'F-Droid'
description_short = package.find(class_='package-summary').text.strip()
url = response.url[:-1] if response.url.endswith('/') else response.url
docid = url.split('/')[-1]
package_version = package.find(class_='package-version', id='latest')
header = package_version.find(class_='package-version-header')
links = header.find_all(name='a')
version = links[0].attrs['name'].strip()
version_code = int(links[1].attrs['name'].strip())
html_description = package.find(class_='package-description').text.strip()
proto = Proto.DocV2()
proto.docid = docid
proto.title = title
proto.creator = creator
proto.descriptionShort = description_short
doc_details = Proto.DocumentDetails()
details = Proto.AppDetails()
details.versionCode = version_code
details.versionString = version
doc_details.appDetails.CopyFrom(details)
proto.details.CopyFrom(doc_details)
proto.descriptionHtml = html_description
return proto
def _home(self):
"""Queries the Google Play Store for its landing page.
Only used to determine the validity of a previously issued token.
"""
headers = self._base_headers()
parameters = {
'c': 3,
'nocache_isui': True
}
response = get('https://android.clients.google.com/fdfe/homeV2',
headers=headers,
params=parameters,
verify=True)
message = ResponseWrapper.FromString(response.content)
if message.commands.displayErrorMessage != "":
raise AuthenticationError(message.commands.displayErrorMessage)
def subcategories(self, category, free=True):
"""Given a category, returns a list of its subcategories.
Free is only there to ensure PlayStore compatibility.
Parameters
----------
category : Category
The category to query for subcategories.
In the case of F-Droid we only have one level of categories, so the sole parent "category" is F-Droid itself
free : boolean
Only exists to ensure PlayStore compatibility.
Returns
-------
list
A list of Objects.SubCategory.
"""
response = get(SERVER + '/en/packages')
if response.status_code != 200:
raise RequestError(response.reason)
return self._parse_categories(response, category)
def categories(self):
"""Retrieves all categories available in the PlayStore at the moment.
Returns
-------
list
A list of categories in protobuf format.
"""
self._login()
response = get(SERVER + 'fdfe/browse',
params={'c': 3},
headers=self._base_headers())
try:
proto_response = ResponseWrapper.FromString(response.content)
except DecodeError:
LOGGER.error(f'Categories query provided invalid data\n'
f'Without categories, we cannot proceed. Exiting now.')
LOGGER.error(response.content)
exit(1)
if proto_response.commands.displayErrorMessage:
raise RequestError(proto_response.commands.displayErrorMessage)
return category_list(proto_response)
def details(self, package):
"""Retrieves the details for an app given its package name.
Intended for manual use only.
Parameters
----------
package : str
A package name.
Returns
-------
DocV2
The details of the application in protobuf format.
"""
self._login()
response = get(SERVER + 'fdfe/details',
params={'doc': package},
headers=self._base_headers())
proto_response = ResponseWrapper.FromString(response.content)
if proto_response.commands.displayErrorMessage:
raise RequestError(proto_response.commands.displayErrorMessage)
return proto_response.payload.detailsResponse.docV2
def download(self, app):
"""Downloads an app as apk file and provides the path it can be found in.
Parameters
----------
app : App
The app to download.
Returns
-------
str
The path the application was saved to.
"""
LOGGER.debug(f'Downloading {app.package_name()}')
app.write_to_file()
response = get(f'{SERVER}/repo/{app.package_name()}_{app.version_code()}.apk')
if response.status_code != 200:
raise RequestError(f'\n\tReason:\t{response.reason}\n\tCode:{response.status_code}'
f'\n\tUrl:\n{response.url}')
with open(app.apk_file(), 'wb+') as apk_file:
apk_file.write(response.content)
LOGGER.info(f'Successfully downloaded {app.package_name()} to {app.apk_file()}')
return app.apk_file()
def discover_apps(self, subcategory, app_list=None):
"""Given a subcategory, discovers all apps contained therein.
Parameters
----------
subcategory : SubCategory
The SubCategory to search.
app_list : AppList
If an AppList is provided, it will be extended instead of creating a new one.
Returns
-------
AppList
An AppList object containing the discovered apps.
"""
# LOGGER.info(f"Discovering apps for {subcategory.parent.name()} - {subcategory.name()}")
self._login()
# LOGGER.info(f'at {SERVER}fdfe/{subcategory.data()}')
response = get(SERVER + f'fdfe/{subcategory.data()}',
headers=self._base_headers())
proto_response = ResponseWrapper.FromString(response.content)
# except DecodeError:
# LOGGER.error(f'Subcategory {subcategory.name()} of {subcategory.parent.name()} provided invalid data\n'
# f'\tCould not initialize AppList')
# return app_list if app_list else []
if proto_response.commands.displayErrorMessage:
raise RequestError(proto_response.commands.displayErrorMessage)
if app_list:
return app_list.update(proto_response)
else:
try:
return AppList(proto_response, subcategory, self)
except IndexError:
LOGGER.error(f'Subcategory {subcategory.name()} of {subcategory.parent.name()} provided invalid data\n'
f'\tCould not initialize AppList')
return []
def subcategories(self, category, free=True):
"""Given a category, retrieves its subcategories.
In Google Play, theses are grouped by Top Selling, Top Grossing, etc.
If free is set, only categories that may contain free Applications are returned.
Note that these may include e.g. Top Grossing due to In App purchases.
Parameters
----------
category : Category
The category to query for subcategories.
free : bool
Specifies whether or not only free applications should be returned.
Normally, this should be true, otherwise, the applications need to be purchased on the Google Play Account.
This functionality is not supported by this API.
Returns
-------
list
A list of subcategories.
"""
self._login()
response = get(SERVER + 'fdfe/browse',
params={'c': 3,
'cat': category.id()},
headers=self._base_headers())
try:
proto_response = ResponseWrapper.FromString(response.content)
except DecodeError:
LOGGER.error(f'Category{category.name()} provided invalid data\n'
f'\tCould not retrieve subcategories')
LOGGER.error(response.content)
return []
if proto_response.commands.displayErrorMessage:
raise RequestError(proto_response.commands.displayErrorMessage)
return list(filter(lambda x: 'paid' not in x.id() if free else lambda x2: True,
subcategory_list(proto_response, category)))
def download(self, app):
"""Downloads an application given by an object.
Note that the purchase is handled automatically, just like the download of additional files and split apks.
This method is intended for automatic access by a crawler that received the App object from this API.
Parameters
----------
app : App
The app to download.
Returns
-------
str
The path the app's apk file was downloaded to.
"""
"""
Given an app, purchases it and downloads all the corresponding files.
Purchasing is necessary for free apps to receive a download token
:returns: The file path to the .apk
"""
LOGGER.info(f'Downloading {app.package_name()}')
self._login()
app.write_to_file()
params = {
'ot': app.offer_type(),
'doc': app.package_name(),
'vc': str(app.version_code())
}
download_token = self._purchase_free(app, params)
params['dtok'] = download_token
response = get(SERVER + 'fdfe/delivery',
params=params,
headers=self._base_headers())
proto_response = ResponseWrapper.FromString(response.content)
error = proto_response.commands.displayErrorMessage
if error != '':
if 'busy' in error:
raise Wait('Server was busy')
else:
raise RequestError(error)
elif proto_response.payload.deliveryResponse.appDeliveryData.downloadUrl == '':
LOGGER.error(f'App {app.package_name()} was not purchased!')
return ''
cookie = proto_response.payload.deliveryResponse.appDeliveryData.downloadAuthCookie[0]
download_response = get(url=proto_response.payload.deliveryResponse.appDeliveryData.downloadUrl,
cookies={str(cookie.name): str(cookie.value)},
headers=self._base_headers())
with open(app.apk_file(), 'wb+') as apk_file:
apk_file.write(download_response.content)
LOGGER.debug(f'Successfully downloaded apk to {app.apk_file()}')
for apk_split in proto_response.payload.deliveryResponse.appDeliveryData.split:
split_response = get(url=apk_split.downloadUrl,
headers=self._base_headers())
with open(app.splits() + apk_split.name, 'wb+') as split_file:
split_file.write(split_response.content)
obb_type = {
0: 'main',
1: 'patch'
}
for obb in proto_response.payload.deliveryResponse.appDeliveryData.additionalFile:
obb_response = get(url=obb.downloadUrl,
headers=self._base_headers())
obb_file_name = f'{obb_type[obb.fileType]}.{obb.versionCode}.{app.package_name()}.obb'
with open(app.additional_files() + obb_file_name, 'wb+') as obb_file:
obb_file.write(obb_response.content)
return app.apk_file()