def findDedicateInPage(pageIndex, buff):
print("START: scan page {0}".format(pageIndex))
page = Page(URL, pageIndex, PAGE_SIZE)
for photo in page.fetchPhotos():
if Photo.fetchOneById(photo.id) is None:
buff.append(photo)
print("SCAN: {0} is qulified because cannot find record locally".format(photo.id))
else:
print("SKIP: {0} has a local record.".format(photo.id))
def getPaging(pageIndex, downloadPhotoFn, tracker):
pageObject = Page(URL, pageIndex, PAGE_SIZE)
photos = pageObject.fetchPhotos()
print("PREPARE: create download task for Page: {0}".format(pageIndex))
with concurrent.futures.ThreadPoolExecutor(max_workers=MAX_DOWNLOAD_WORKER) as downloadExecutor:
executor = downloadExecutor
#executor = None
for photo in photos:
if tracker is not None:
tracker.addTask(1)
if executor is None:
downloadPhotoFn(photo, tracker)
else:
executor.submit(downloadPhotoFn, photo, tracker)
