def findDedicateInPage(pageIndex, buff): print("START: scan page {0}".format(pageIndex)) page = Page(URL, pageIndex, PAGE_SIZE) for photo in page.fetchPhotos(): if Photo.fetchOneById(photo.id) is None: buff.append(photo) print("SCAN: {0} is qulified because cannot find record locally".format(photo.id)) else: print("SKIP: {0} has a local record.".format(photo.id))
def downloadPhoto(photo, tracker): url = photo.full #url = photo.thumb file_name = '{0}.jpg'.format(get_filename(url)) download_file_name = path.join(DOWNLOAD, file_name) archive_file_name = path.join(ARCHIVE, file_name) delete_file_name = path.join(DELETE, file_name) if Photo.fetchOneById(photo.id) is not None: print("SKIP: {0} has record".format(photo.id)) if path.exists(archive_file_name): print("SKIP: downloading {0} becase exist of {1}, url {2}".format(file_name, archive_file_name, url)) os.rename(archive_file_name, delete_file_name) if tracker is not None: tracker.finishTask(1) tracker.report() return if path.exists(archive_file_name): print("SKIP: downloading {0} becase exist of {1}, url {2}".format(file_name, archive_file_name, url)) photo.insertOrUpdate() os.rename(archive_file_name, delete_file_name) if tracker is not None: tracker.finishTask(1) tracker.report() return if DRY_RUN == False: r_start_time = int(time() * 1000) r = requests.get(url = url) open(download_file_name, 'wb').write(r.content) r_end_time = int(time() * 1000) r_using_time = r_end_time - r_start_time print("FINISH: downloading {0}(use {1} ms)" .format(file_name, r_using_time)) photo.insertOrUpdate() else: print("DRYRUN: downloading {0}".format(file_name)) if tracker is not None: tracker.finishTask(1) tracker.report()