async def backoff_needed(
status: str,
response: aiohttp.ClientResponse,
) -> bool:
if status not in BACKOFF_STATUSES:
return False
# not all 403 errors are rate limit error
if status == '403':
msg = await response.json()
if not msg:
# undefined behavior, probably a server problem, better backoff
WARNING('wcpan.drive.google') << '403 with empty error message'
return True
domain = msg['error']['errors'][0]['domain']
if domain != 'usageLimits':
return False
# the request timedout
if status == '408':
# NOTE somehow this error shows html instead of json
WARNING('wcpan.drive.google') << '408 request timed out'
# No need to backoff because in this case the whole request cannot be
# resumed at all.
return False
return True
async def get(self):
db = self.request.app['db']
w = self.request.app['weather']
city_id = self.request.match_info.get('city_id')
city_id = int(city_id)
# first look up from cache, so we can save API quota
weather_data = db.get_weather_by_city_id(city_id)
if weather_data:
DEBUG('weather') << 'got from cache'
DEBUG('weather') << weather_data
return aw.json_response(weather_data)
# if the cache is invalid, fetch new one
weather_data = await w.get_weather_by_city_id(city_id)
if weather_data:
db.update_weather(city_id, weather_data)
DEBUG('weather') << 'got from openweather'
DEBUG('weather') << weather_data
return aw.json_response(weather_data)
# HACK quota exceed, return a random data
WARNING('weather') << 'quota exceed'
return aw.json_response({
'temp': 28,
'temp_min': 26,
'temp_max': 30,
'icon': 300,
})
async def _download_file(self, node, local_path, full_path):
drive = self._context.drive
# retry until succeed
while True:
try:
remote_path = await drive.get_path(node)
INFO('ddld') << 'downloading:' << remote_path
ok = await drive.download_file(node, local_path)
INFO('ddld') << 'checking:' << remote_path
local_hash = await self._md5sum(full_path)
INFO('ddld') << local_hash << remote_path
except wdg.DownloadError as e:
ERROR('ddld') << 'download failed:' << str(e)
except OSError as e:
if e.errno == 36:
WARNING('ddld') << 'download failed: file name too long'
return False
# fatal unknown error
raise
else:
remote_hash = node.md5
if local_hash != remote_hash:
INFO('ddld') << 'md5 mismatch:' << full_path
full_path.unlink()
else:
break
return True
async def _download(self, node, local_path, need_mtime):
if not node or not local_path:
return False
if node.trashed:
return False
try:
if await self._check_existence(node, local_path):
if not need_mtime:
full_path = local_path / node.name
ok = update_mtime(full_path, dt.datetime.now().timestamp())
return ok
return True
except OSError as e:
if e.errno == 36:
WARNING('ddld') << 'download failed: file name too long'
return False
# fatal unknown error
raise
DEBUG('ddld') << 'different'
required_space = await self._get_node_size(node)
if await self._need_recycle(required_space):
if need_mtime and self._is_too_old(node):
DEBUG('ddld') << 'too old'
self.abort()
return False
await self._reserve_space(node)
with self._reserve_pending_file(required_space):
rv = await self._download_glue(node, local_path, need_mtime)
return rv
async def _download_folder(self, node, full_path, need_mtime):
try:
full_path.mkdir(parents=True, exist_ok=True)
except OSError:
WARNING('ddld') << 'mkdir failed:' << full_path
return False
children = await self._context.drive.get_children(node)
for child in children:
ok = await self._download_glue(child, full_path, need_mtime)
if not ok:
return False
return True
async def _download_from_offset(self) -> 'aiohttp.StreamResponse':
try:
return await self._download(
file_id=self._node.id_,
range_=(self._offset, self._node.size),
acknowledge_abuse=False,
)
except DownloadAbusiveFileError:
# FIXME automatically accept abuse files for now
WARNING('wcpan.drive.google') << f'{self._node.id_} is an abusive file'
return await self._download(
file_id=self._node.id_,
range_=(self._offset, self._node.size),
acknowledge_abuse=True,
)
async def backoff_needed(response: Response) -> bool:
if response.status not in BACKOFF_STATUSES:
return False
# if it is not a rate limit error, it could be handled immediately
if response.status == '403':
msg = await response.json()
if not msg:
WARNING('wcpan.drive.google') << '403 with empty error message'
# probably server problem, backoff for safety
return True
domain = msg['error']['errors'][0]['domain']
if domain != 'usageLimits':
return False
INFO('wcpan.drive.google') << msg['error']['message']
return True
async def fetch(
self,
method: Text,
url: Text,
args: Dict[Text, Any] = None,
headers: Dict[Text, Text] = None,
body: ReadableContent = None,
raise_internal_error: bool = False,
) -> 'Response':
while True:
await self._maybe_backoff()
try:
rv = await self._do_request(method, url, args, headers, body,
raise_internal_error)
return rv
except NetworkError as e:
if e.status != '599':
raise
if e._response.raise_internal_error:
raise
WARNING('wcpan.drive.google') << str(e)