def update_keywords(): sm_api = SmugmugAPI() def worker(): logger.info('[Worker started]') while True: item = q.get() try: sm_api.update_image_keywords(*item) finally: q.task_done() q = JoinableQueue(maxsize=100) for i in range(50): gevent.spawn(worker) photos = (Photo.select(Photo.local_path, Photo.ext_key).where( (Photo.status == 'uploaded'))) photos = list(photos) print("Total photos to update:", len(photos)) cnt = 0 for p in photos: cnt += 1 print(cnt) keywords = get_keywords(p.local_path) q.put((p.ext_key, keywords)) q.join()
def upload_photos_in_pending(with_failed=True): q_filter = ['pending'] if with_failed: q_filter.append('failed') photos = (Photo.select(Photo.local_path, Photo.ext_album_key).where( (Photo.status << q_filter))) photos = list(photos) def worker(): logger.info('[New worker started]') while True: item = q.get() try: upload_photo(item) finally: q.task_done() q = JoinableQueue(maxsize=10) for i in range(UPLOADING_WORKERS_COUNT): gevent.spawn(worker) for p in photos: q.put((p.local_path, p.ext_album_key)) q.join()
def show_stat_by_md5(self): md5_success_count = 0 md5_failed_count = 0 md5_not_found = 0 for photos_chunk in self.api.get_remote_images(): logger.info("[INFO] Total photos in API response: %d", len(photos_chunk)) for p in photos_chunk: p_md5 = p.get('ArchivedMD5') if not p_md5: md5_failed_count += 1 logger.info("[ERROR] ArchivedMD5 is NULL") continue p_db = Photo.select().where(Photo.local_md5 == p_md5) if not p_db: logger.info("[ERROR] MD5 %s not found in DB for file: %s", p_md5, p.get('FileName')) md5_not_found += 1 continue else: p_db = p_db[0] if p.get('FileName') == os.path.basename(p_db.local_path): p_db.ext_md5 = p_md5 p_db.save() md5_success_count += 1 else: logger.info("[ERROR] MD5 not equal. local: %s remote: %s", p_db.local_path, p.get('FileName')) md5_failed_count += 1 logger.info( "\n\nSuccess: %d\nFailed: %d\nNot found: %d\nTotal: %d", md5_success_count, md5_failed_count, md5_not_found, md5_failed_count + md5_not_found + md5_success_count)
def sync_files_with_db(files_tree): photos_to_upload = {root_path: {} for root_path in files_tree.keys()} total_new_photos = 0 photos = set() for root_path, folders in files_tree.items(): for folder, files_bundle in folders.items(): files = files_bundle['files'] logger.info('Album: %s', folder) logger.info("\tTotal photos: %d", len(files)) for paths_chunk in chunks(files, 300): _photos = (Photo.select(Photo.local_path).where( (Photo.local_path << paths_chunk))) photos.update(set(_photos)) db_photos = {_p.local_path for _p in photos} logger.info("\tPhotos exist in DB: %d", len(db_photos)) local_photos = set(files) new_photos = local_photos - db_photos if new_photos: photos_to_upload[root_path][folder] = { 'files': list(new_photos), 'album_uri': None, } total_new_photos += len(new_photos) else: logger.info( "All photos already exist in DB. Upload skipped\n\n") del files_tree if total_new_photos: logger.info("Total new photos to upload: %d", total_new_photos) return photos_to_upload
def clean_hidden_files_from_db(): photos_in_db = Photo.select(Photo.local_path) logger.info("Total photos in DB: %d", photos_in_db.count()) hidden_files_to_remove = [] for p in photos_in_db: file_name = os.path.basename(p.local_path) if file_name.startswith('.'): logger.info("Hidden file found: %s", file_name) hidden_files_to_remove.append(p.local_path) logger.info("Total hidden files count: %d", len(hidden_files_to_remove)) if hidden_files_to_remove: with db.atomic(): for files_chunk in chunks(hidden_files_to_remove, 300): _removed_cnt = Photo.delete().where( (Photo.local_path << files_chunk) ).execute() logger.info("Removed: %d", _removed_cnt)
def get_photos_without_md5(): photos = Photo.select().where(Photo.local_md5 == None) logger.info("Total photos without MD5: %d", len(photos)) return photos