コード例 #1
0
def update_keywords():
    sm_api = SmugmugAPI()

    def worker():
        logger.info('[Worker started]')
        while True:
            item = q.get()
            try:
                sm_api.update_image_keywords(*item)
            finally:
                q.task_done()

    q = JoinableQueue(maxsize=100)
    for i in range(50):
        gevent.spawn(worker)

    photos = (Photo.select(Photo.local_path, Photo.ext_key).where(
        (Photo.status == 'uploaded')))
    photos = list(photos)
    print("Total photos to update:", len(photos))
    cnt = 0
    for p in photos:
        cnt += 1
        print(cnt)
        keywords = get_keywords(p.local_path)
        q.put((p.ext_key, keywords))

    q.join()
コード例 #2
0
def upload_photos_in_pending(with_failed=True):
    q_filter = ['pending']
    if with_failed:
        q_filter.append('failed')

    photos = (Photo.select(Photo.local_path, Photo.ext_album_key).where(
        (Photo.status << q_filter)))
    photos = list(photos)

    def worker():
        logger.info('[New worker started]')
        while True:
            item = q.get()
            try:
                upload_photo(item)
            finally:
                q.task_done()

    q = JoinableQueue(maxsize=10)
    for i in range(UPLOADING_WORKERS_COUNT):
        gevent.spawn(worker)

    for p in photos:
        q.put((p.local_path, p.ext_album_key))

    q.join()
コード例 #3
0
    def show_stat_by_md5(self):
        md5_success_count = 0
        md5_failed_count = 0
        md5_not_found = 0

        for photos_chunk in self.api.get_remote_images():
            logger.info("[INFO] Total photos in API response: %d",
                        len(photos_chunk))

            for p in photos_chunk:
                p_md5 = p.get('ArchivedMD5')

                if not p_md5:
                    md5_failed_count += 1
                    logger.info("[ERROR] ArchivedMD5 is NULL")
                    continue

                p_db = Photo.select().where(Photo.local_md5 == p_md5)

                if not p_db:
                    logger.info("[ERROR] MD5 %s not found in DB for file: %s",
                                p_md5, p.get('FileName'))
                    md5_not_found += 1
                    continue
                else:
                    p_db = p_db[0]

                if p.get('FileName') == os.path.basename(p_db.local_path):
                    p_db.ext_md5 = p_md5
                    p_db.save()
                    md5_success_count += 1
                else:
                    logger.info("[ERROR] MD5 not equal. local: %s remote: %s",
                                p_db.local_path, p.get('FileName'))

                    md5_failed_count += 1

            logger.info(
                "\n\nSuccess: %d\nFailed: %d\nNot found: %d\nTotal: %d",
                md5_success_count,
                md5_failed_count,
                md5_not_found,
                md5_failed_count + md5_not_found + md5_success_count)
コード例 #4
0
def sync_files_with_db(files_tree):
    photos_to_upload = {root_path: {} for root_path in files_tree.keys()}
    total_new_photos = 0

    photos = set()
    for root_path, folders in files_tree.items():
        for folder, files_bundle in folders.items():
            files = files_bundle['files']

            logger.info('Album: %s', folder)
            logger.info("\tTotal photos: %d", len(files))

            for paths_chunk in chunks(files, 300):
                _photos = (Photo.select(Photo.local_path).where(
                    (Photo.local_path << paths_chunk)))

                photos.update(set(_photos))

            db_photos = {_p.local_path for _p in photos}

            logger.info("\tPhotos exist in DB: %d", len(db_photos))

            local_photos = set(files)
            new_photos = local_photos - db_photos

            if new_photos:
                photos_to_upload[root_path][folder] = {
                    'files': list(new_photos),
                    'album_uri': None,
                }
                total_new_photos += len(new_photos)
            else:
                logger.info(
                    "All photos already exist in DB. Upload skipped\n\n")

    del files_tree

    if total_new_photos:
        logger.info("Total new photos to upload: %d", total_new_photos)

    return photos_to_upload
コード例 #5
0
def clean_hidden_files_from_db():
    photos_in_db = Photo.select(Photo.local_path)

    logger.info("Total photos in DB: %d", photos_in_db.count())

    hidden_files_to_remove = []

    for p in photos_in_db:
        file_name = os.path.basename(p.local_path)
        if file_name.startswith('.'):
            logger.info("Hidden file found: %s", file_name)

            hidden_files_to_remove.append(p.local_path)

    logger.info("Total hidden files count: %d", len(hidden_files_to_remove))

    if hidden_files_to_remove:
        with db.atomic():
            for files_chunk in chunks(hidden_files_to_remove, 300):
                _removed_cnt = Photo.delete().where(
                    (Photo.local_path << files_chunk)
                ).execute()

                logger.info("Removed: %d", _removed_cnt)
コード例 #6
0
def get_photos_without_md5():
    photos = Photo.select().where(Photo.local_md5 == None)
    logger.info("Total photos without MD5: %d", len(photos))
    return photos