def findDedicateInPage(pageIndex, buff):
print("START: scan page {0}".format(pageIndex))
page = Page(URL, pageIndex, PAGE_SIZE)
for photo in page.fetchPhotos():
if Photo.fetchOneById(photo.id) is None:
buff.append(photo)
print("SCAN: {0} is qulified because cannot find record locally".format(photo.id))
else:
print("SKIP: {0} has a local record.".format(photo.id))
def downloadPhoto(photo, tracker):
url = photo.full
#url = photo.thumb
file_name = '{0}.jpg'.format(get_filename(url))
download_file_name = path.join(DOWNLOAD, file_name)
archive_file_name = path.join(ARCHIVE, file_name)
delete_file_name = path.join(DELETE, file_name)
if Photo.fetchOneById(photo.id) is not None:
print("SKIP: {0} has record".format(photo.id))
if path.exists(archive_file_name):
print("SKIP: downloading {0} becase exist of {1}, url {2}".format(file_name, archive_file_name, url))
os.rename(archive_file_name, delete_file_name)
if tracker is not None:
tracker.finishTask(1)
tracker.report()
return
if path.exists(archive_file_name):
print("SKIP: downloading {0} becase exist of {1}, url {2}".format(file_name, archive_file_name, url))
photo.insertOrUpdate()
os.rename(archive_file_name, delete_file_name)
if tracker is not None:
tracker.finishTask(1)
tracker.report()
return
if DRY_RUN == False:
r_start_time = int(time() * 1000)
r = requests.get(url = url)
open(download_file_name, 'wb').write(r.content)
r_end_time = int(time() * 1000)
r_using_time = r_end_time - r_start_time
print("FINISH: downloading {0}(use {1} ms)"
.format(file_name, r_using_time))
photo.insertOrUpdate()
else:
print("DRYRUN: downloading {0}".format(file_name))
if tracker is not None:
tracker.finishTask(1)
tracker.report()
