def update_keywords():
sm_api = SmugmugAPI()
def worker():
logger.info('[Worker started]')
while True:
item = q.get()
try:
sm_api.update_image_keywords(*item)
finally:
q.task_done()
q = JoinableQueue(maxsize=100)
for i in range(50):
gevent.spawn(worker)
photos = (Photo.select(Photo.local_path, Photo.ext_key).where(
(Photo.status == 'uploaded')))
photos = list(photos)
print("Total photos to update:", len(photos))
cnt = 0
for p in photos:
cnt += 1
print(cnt)
keywords = get_keywords(p.local_path)
q.put((p.ext_key, keywords))
q.join()
def upload_photos_in_pending(with_failed=True):
q_filter = ['pending']
if with_failed:
q_filter.append('failed')
photos = (Photo.select(Photo.local_path, Photo.ext_album_key).where(
(Photo.status << q_filter)))
photos = list(photos)
def worker():
logger.info('[New worker started]')
while True:
item = q.get()
try:
upload_photo(item)
finally:
q.task_done()
q = JoinableQueue(maxsize=10)
for i in range(UPLOADING_WORKERS_COUNT):
gevent.spawn(worker)
for p in photos:
q.put((p.local_path, p.ext_album_key))
q.join()
def show_stat_by_md5(self):
md5_success_count = 0
md5_failed_count = 0
md5_not_found = 0
for photos_chunk in self.api.get_remote_images():
logger.info("[INFO] Total photos in API response: %d",
len(photos_chunk))
for p in photos_chunk:
p_md5 = p.get('ArchivedMD5')
if not p_md5:
md5_failed_count += 1
logger.info("[ERROR] ArchivedMD5 is NULL")
continue
p_db = Photo.select().where(Photo.local_md5 == p_md5)
if not p_db:
logger.info("[ERROR] MD5 %s not found in DB for file: %s",
p_md5, p.get('FileName'))
md5_not_found += 1
continue
else:
p_db = p_db[0]
if p.get('FileName') == os.path.basename(p_db.local_path):
p_db.ext_md5 = p_md5
p_db.save()
md5_success_count += 1
else:
logger.info("[ERROR] MD5 not equal. local: %s remote: %s",
p_db.local_path, p.get('FileName'))
md5_failed_count += 1
logger.info(
"\n\nSuccess: %d\nFailed: %d\nNot found: %d\nTotal: %d",
md5_success_count,
md5_failed_count,
md5_not_found,
md5_failed_count + md5_not_found + md5_success_count)
def sync_files_with_db(files_tree):
photos_to_upload = {root_path: {} for root_path in files_tree.keys()}
total_new_photos = 0
photos = set()
for root_path, folders in files_tree.items():
for folder, files_bundle in folders.items():
files = files_bundle['files']
logger.info('Album: %s', folder)
logger.info("\tTotal photos: %d", len(files))
for paths_chunk in chunks(files, 300):
_photos = (Photo.select(Photo.local_path).where(
(Photo.local_path << paths_chunk)))
photos.update(set(_photos))
db_photos = {_p.local_path for _p in photos}
logger.info("\tPhotos exist in DB: %d", len(db_photos))
local_photos = set(files)
new_photos = local_photos - db_photos
if new_photos:
photos_to_upload[root_path][folder] = {
'files': list(new_photos),
'album_uri': None,
}
total_new_photos += len(new_photos)
else:
logger.info(
"All photos already exist in DB. Upload skipped\n\n")
del files_tree
if total_new_photos:
logger.info("Total new photos to upload: %d", total_new_photos)
return photos_to_upload
def clean_hidden_files_from_db():
photos_in_db = Photo.select(Photo.local_path)
logger.info("Total photos in DB: %d", photos_in_db.count())
hidden_files_to_remove = []
for p in photos_in_db:
file_name = os.path.basename(p.local_path)
if file_name.startswith('.'):
logger.info("Hidden file found: %s", file_name)
hidden_files_to_remove.append(p.local_path)
logger.info("Total hidden files count: %d", len(hidden_files_to_remove))
if hidden_files_to_remove:
with db.atomic():
for files_chunk in chunks(hidden_files_to_remove, 300):
_removed_cnt = Photo.delete().where(
(Photo.local_path << files_chunk)
).execute()
logger.info("Removed: %d", _removed_cnt)
def get_photos_without_md5():
photos = Photo.select().where(Photo.local_md5 == None)
logger.info("Total photos without MD5: %d", len(photos))
return photos