def sync_structure(self):
media_data = find_media_files()
self.media_data = sync_files_with_db(media_data)
if not self.media_data.get('albums'):
logger.info("No new photos to upload")
return
root_node_uri = self.api.get_root_node()
# Create new folders
for folder_path, files_bundle in self.media_data['albums'].items():
folders = folder_path.strip('/').split('/')
prev_node_uri = root_node_uri
for cnt, f in enumerate(folders, start=1):
if cnt == len(folders):
nt = 'Album'
else:
nt = 'Folder'
current_node_uri = self.api.get_or_create_node(
prev_node_uri, f, node_type=nt
)
if cnt == len(folders):
files_bundle['album_uri'] = current_node_uri
prev_node_uri = current_node_uri
# Process children albums (get uri or create)
for folder_path, files_bundle in self.media_data['albums'].items():
with db.atomic():
photos_insert_to_db = []
for f in files_bundle['files']:
photos_insert_to_db.append(
{
'local_path': f,
'local_md5': get_md5(f),
'status': 'pending'
}
)
if photos_insert_to_db:
logger.info("\tInserting to DB: %d",
len(photos_insert_to_db))
for photos_insert_to_db_chunk in chunks(
photos_insert_to_db, 300):
Photo.insert_many(photos_insert_to_db_chunk).execute()
for files_chunk in chunks(files_bundle['files'], 300):
Photo.update(
ext_album_key=files_bundle['album_uri']
).where(
(Photo.local_path << files_chunk)
).execute()
def upload_photo(photo_item):
api = SmugmugAPI()
attempts = 5
img_path, album_uri = photo_item
while attempts:
try:
file_name = os.path.basename(img_path)
headers = {
'User-Agent': 'Safari',
'X-Smug-ResponseType': 'JSON',
'X-Smug-Version': 'v2',
'Content-Type': guess_type(file_name)[0],
'X-Smug-AlbumUri': album_uri,
'X-Smug-FileName': file_name,
'Content-Length': str(path.getsize(img_path)),
'Content-MD5': get_md5(img_path),
'X-Smug-Keywords': get_keywords(img_path),
}
if DEBUG:
logger.debug(["Uploading:", img_path, 'to:', album_uri])
with open(img_path, "rb") as f:
data = f.read()
response = api.r.post('http://upload.smugmug.com/',
headers=headers,
data=data,
header_auth=True)
r = json.loads(response.content) or {}
del data
if r.get('stat') == 'ok':
logger.info('\t\tPhoto uploaded: %s', file_name)
field_data = {
"status": "uploaded",
"ext_key": r.get('Image').get('ImageUri'),
"ext_album_key": album_uri,
}
with db.atomic():
Photo.update(**field_data).where(
(Photo.local_path == img_path)).execute()
break
else:
logger.exception(
"Something goes wrong while uploading image")
try:
log = "\n".join([
str(response.content),
str(headers),
str(response.headers),
str(response.status_code),
str(response.reason)
])
except Exception:
logger.exception(
"Something goes wrong while uploading image")
log = None
field_data = {"status": "failed", "log": log}
with db.atomic():
Photo.update(**field_data).where(
(Photo.local_path == img_path)).execute()
raise Exception("stat is not OK")
except Exception:
sleep(5)
attempts -= 1
field_data = {
"status": "failed",
}
with db.atomic():
Photo.update(**field_data).where(
(Photo.local_path == img_path)).execute()