def update_ignore_folders():
"""Get sub-folders in path and return them."""
if request.method == 'GET':
ignore_folders = [{
"name": folder.name
} for folder in IgnoreFolder.select().where(
(IgnoreFolder.type == 'USER') & (IgnoreFolder.removed == False))]
sync_db.close()
return jsonify(ignore_folders)
if request.method == 'PUT':
folder = request.get_json()['folder']
new_ignore = IgnoreFolder(name=folder, type='USER')
db_queue.put([new_ignore, 'save'])
sync_db.close()
return Response(status=200)
if request.method == 'DELETE':
# Flag as removed so sync loop updates all assets blocked by it.
folder = request.get_json()['folder']
db_folder = IgnoreFolder.get((IgnoreFolder.type == 'USER')
& (IgnoreFolder.name == folder))
db_folder.removed = True
db_queue.put([db_folder, 'save'])
sync_db.close()
return Response(status=200)
return Response('Bad request', status=400)
def update_ignored_assets(self):
"""Find changes to ignore folders and un-flag blocked assets.
Frame.io assets are added to DB even if they match an ignore folder.
Remove the ignore flag, and download_new_assets will pick them up.
Local assets are not added to DB if they match an ignore folder, just
skipped. Reset last_scan will add/upload them.
"""
removed_ignore_folders = self.IgnoreFolder.select().where(
self.IgnoreFolder.removed == True)
active_ignore_folders = [
folder.name for folder in self.IgnoreFolder.select().where(
self.IgnoreFolder.removed == False)
]
all_blocked_folders = self.Asset.select().where(
(self.Asset.ignore == True) & (self.Asset.is_file == False))
for ignore_folder in removed_ignore_folders:
logger.info('Removing ignore folder {}'.format(ignore_folder.name))
blocked_folders = [
f for f in all_blocked_folders if f.name == ignore_folder.name
or self.wildcard_match(f.name, [ignore_folder.name])
]
self.remove_ignore_flag(blocked_folders, active_ignore_folders)
db_queue.put([ignore_folder, 'delete'])
for project in self.Project.select(): # Trigger re-scan of all
project.last_local_scan = 0
db_queue.put([project, 'save'])
def run(self):
while True:
if authenticated_client():
try:
logger.info('Checking for updates')
self.update_projects()
ignore_folders = [
folder.name
for folder in self.IgnoreFolder.select().where(
self.IgnoreFolder.removed == False)
]
for project in self.Project.select().where(
self.Project.sync == True):
self.update_frameio_assets(
project=project, ignore_folders=ignore_folders)
self.update_local_assets(project=project,
ignore_folders=ignore_folders)
if config.SyncSetting.ASSETS_LOCAL_TO_FRAME:
self.upload_new_assets(project)
if config.SyncSetting.ASSETS_FRAMEIO_TO_LOCAL:
self.download_new_assets(project)
self.verify_new_uploads()
except (requests.exceptions.ConnectionError,
requests.exceptions.HTTPError):
logger.info('Could not connect, retrying in {}'.format(
config.SCAN_INTERVAL))
# Delete project from DB if requested by user.
for project in self.Project.select().where(
self.Project.db_delete_requested == True):
self.delete_db_project(project)
# Delete assets to redo sync from scratch if path has been changed.
for project in self.Project.select().where(
self.Project.local_path_changed == True):
logger.info(
'Path changed, deleting and recreating assets in db')
self.delete_assets_from_db(project)
project.local_path_changed = False
project.last_local_scan = 0
project.last_frameio_scan = '2014-02-07T00:00:01.000000+00:00'
db_queue.put([project, 'save'])
# Updated ignored assets if an ignore folder has been removed.
if self.IgnoreFolder.select().where(
self.IgnoreFolder.removed == True):
self.update_ignored_assets()
self.db.close()
sleep(config.SCAN_INTERVAL)
def delete_db_project(self, project):
"""Delete project and its associated assets from DB."""
logger.info('Deleting project {} from DB.'.format(project.name))
for asset in self.Asset.select().where(
self.Asset.project_id == project.project_id):
db_queue.put([asset, 'delete'])
db_queue.put([project, 'delete'])
def remove_ignore_flag(self, assets, ignore_folders):
for asset in assets:
if asset.is_file:
asset.ignore = False
db_queue.put([asset, 'save'])
if not asset.is_file:
if asset.name not in ignore_folders and not self.wildcard_match(
asset.name, ignore_folders):
asset.ignore = False
db_queue.put([asset, 'save'])
children = self.Asset.select().where(
self.Asset.parent_id == asset.asset_id)
self.remove_ignore_flag(children, ignore_folders)
def remove_project(project_id):
"""Flag project so sync thread removes it and all its assets from DB."""
try:
project = Project.get(Project.project_id == project_id)
project.db_delete_requested = True
db_queue.put([project, 'save'])
logger.info('Project {} requested to be deleted from DB'.format(
project.name))
sync_db.close()
return Response(status=200)
except Project.DoesNotExist:
return Response('Bad request', status=400)
def save_tokens(tokens):
"""Save tokens to DB."""
login = Login.select().limit(1).get()
login.token = tokens['access_token']
login.refresh_token = tokens['refresh_token']
login.type = tokens['type']
if tokens['type'] == 'DEVTOKEN':
login.token_expires = 'NEVER'
else:
login.token_expires = time() + 3300 # 5min padding for safety
db_queue.put([login, 'save'])
sync_db.close()
def authenticated_client():
"""Return authenticated frame.io client either from cache or refresh"""
if config.client_expires == 'NEVER':
return config.authenticated_client
if config.client_expires > time():
return config.authenticated_client
login = Login.select().limit(1).get()
if login.token == '': # Not logged in
logger.info('Not logged in')
sync_db.close()
return False
if login.type == 'DEVTOKEN':
logger.info('Dev token login found')
config.authenticated_client = frameioclient.FrameioClient(login.token)
config.client_expires = 'NEVER'
sync_db.close()
return config.authenticated_client
logger.info('Refreshing OAuth token')
tokens = refresh_token(login.refresh_token)
if tokens:
tokens['type'] = 'OAUTH'
save_tokens(tokens)
token = tokens['access_token']
config.authenticated_client = frameioclient.FrameioClient(token)
config.client_expires = time() + 3300 # 5min padding for safety
else:
# Token might be too old, delete it to force use to re-auth
logger.info("Couldn't refresh token, signing out")
login = Login.select().limit(1).get()
login.token = ''
db_queue.put([login, 'save'])
return False
if not sync_db.is_closed():
sync_db.close()
return config.authenticated_client
def create_frameio_folder_tree(self, project, new_folder_path):
"""Step up folder tree in DB to find the closest parent folder on
Frame.io. Then creates new folders on Frame.io down to the
new folder and save all new asset_ids to DB.
:Args:
project (Project): In what project to create
new_folder (string): Path to folder to create
"""
current_path = os.path.dirname(new_folder_path)
while current_path != '': # While not at root
try:
db_asset = self.Asset.get(
self.Asset.project_id == project.project_id,
self.Asset.path == current_path,
self.Asset.on_frameio == True)
break
except self.Asset.DoesNotExist:
# Not found, continue up in tree
current_path = os.path.dirname(current_path)
# Parent folder was found
try:
parent_asset_id = db_asset.asset_id
parent_path = db_asset.path
# No parent folder found, root is closest
except NameError:
parent_asset_id = project.root_asset_id
parent_path = ''
new_tree = os.path.relpath(new_folder_path, parent_path)
for folder in new_tree.split('/'):
asset_id = self.new_frameio_folder(folder, parent_asset_id)
path = os.path.join(parent_path, folder)
asset = self.Asset.get(self.Asset.path == path,
self.Asset.project_id == project.project_id)
asset.asset_id = asset_id
asset.on_frameio = True
db_queue.put([asset, 'save'])
parent_asset_id = asset_id
parent_path = path
def calculate_missing_paths(self, project, parent_id, parent_path,
ignore_folders, ignore):
"""Recurse through DB and add missing paths and if folder should be
ignored.
"""
children = self.Asset.select().where(
(self.Asset.project_id == project.project_id)
& (self.Asset.is_file == False)
& (self.Asset.parent_id == parent_id))
for child in children:
if child.name in ignore_folders or ignore or self.wildcard_match(
child.name, ignore_folders):
ignore = True
if child.path == '':
child.path = os.path.join(parent_path, child.name)
child.ignore = ignore
db_queue.put([child, 'save'])
logger.info('Added path to folder {}'.format(child.path))
self.calculate_missing_paths(project, child.asset_id, child.path,
ignore_folders, ignore)
def verify_new_uploads(self):
"""Get xxhash from Frame.io and compare it to local hash in DB.
Call delete and re-upload if hashes don't match.
"""
new_assets = self.Asset.select().where(
(self.Asset.upload_verified == False))
if len(new_assets) == 0:
return
for asset in new_assets:
if int(time()) - asset.uploaded_at < 100:
continue # Giving Frame.io time to calculate hash.
logger.info('New upload to verify: {}'.format(asset.path))
project = self.Project.get(
self.Project.project_id == asset.project_id)
try:
frameio_asset = authenticated_client().get_asset(
asset.asset_id)
except requests.exceptions.HTTPError:
logger.info('Asset deleted from Frame.io, skipping')
asset.upload_verified = True
db_queue.put([asset, 'save'])
continue
if frameio_asset.get('upload_completed_at') is None:
logger.info('Upload failed')
self.delete_and_reupload(project=project, asset=asset)
else:
try:
frameio_hash = frameio_asset['checksums']['xx_hash']
if frameio_hash != asset.local_xxhash:
logger.info('Hash mismatch')
self.delete_and_reupload(project=project, asset=asset)
else:
logger.info('Upload succeeded')
asset.frameio_xxhash = frameio_hash
asset.upload_verified = True
db_queue.put([asset, 'save'])
except (KeyError, TypeError):
logger.info('No calculated checksum yet')
# Edge cases where Frame.io fails to calculate a checksum.
# Mark as successful anyway.
if (time() - asset.uploaded_at) > 1800:
logger.info("""30 mins since upload and no checksum on
Frame.io, marking as successful anyway""")
asset.upload_verified = True
db_queue.put([asset, 'save'])
def update_projects(self):
"""Get all projects from Frame.io and add new to DB."""
projects = []
try:
for team in authenticated_client().get_all_teams():
# print(f"{team['name']}: {team['id']}")
projects += authenticated_client().get_projects(team['id'])
except (requests.exceptions.ConnectionError,
requests.exceptions.HTTPError):
raise
# try:
# projects += authenticated_client().get_shared_projects() # could take a while to run
# except (requests.exceptions.ConnectionError,
# requests.exceptions.HTTPError):
# raise
for project in projects:
try:
db_project = self.Project.get(
self.Project.project_id == project['id'])
# Check if project has been renamed
if db_project.name != project['name']:
db_project.name = project['name']
db_queue.put([db_project, 'save'])
logger.info('Renamed project {} to {}'.format(
db_project.name, project['name']))
except self.Project.DoesNotExist:
logger.info('New project found: {}'.format(project['name']))
new_project = self.Project(
name=project['name'],
project_id=project['id'],
root_asset_id=project['root_asset_id'],
team_id=project['team_id'],
on_frameio=True)
db_queue.put([new_project, 'save'])
# Check if any projects have been deleted
active_projects = [project['id'] for project in projects]
for db_project in self.Project.select().where(
self.Project.deleted_from_frameio == False):
if db_project.project_id not in active_projects:
db_project.deleted_from_frameio = True
db_project.sync = False
db_queue.put([db_project, 'save'])
logger.info("Project {} has been deleted, "
"turning off sync.".format(db_project.name))
def upload_new_assets(self, project):
"""Upload new local assets to Frame.io and save new asset ids to DB."""
new_folders = self.Asset.select().where(
(self.Asset.on_frameio == False) & (self.Asset.is_file == False)
& (self.Asset.project_id == project.project_id))
new_files = self.Asset.select().where(
(self.Asset.on_frameio == False) & (self.Asset.is_file == True)
& (self.Asset.project_id == project.project_id))
if len(new_folders) == 0 and len(new_files) == 0:
return
for folder in new_folders:
if not os.path.isdir(os.path.join(project.local_path,
folder.path)):
logger.info("Can't find {}, skipping.".format(folder.name))
db_queue.put([folder, 'delete'])
continue
self.create_frameio_folder_tree(project=project,
new_folder_path=folder.path)
for file in new_files:
if not os.path.isfile(os.path.join(project.local_path, file.path)):
logger.info("Can't find {}".format(file.name))
db_queue.put([file, 'delete'])
continue
logger.info('Uploading asset: {}'.format(file.path))
abs_path = os.path.abspath(
os.path.join(project.local_path, file.path))
if os.path.dirname(file.path) == '':
parent_asset_id = project.root_asset_id
else:
parent_asset_id = self.Asset.get(
self.Asset.project_id == project.project_id,
self.Asset.path == os.path.dirname(file.path)).asset_id
new_asset = self.upload_asset(abs_path, parent_asset_id)
logger.info('Upload done')
file.asset_id = new_asset['id']
file.original = new_asset['original']
file.uploaded_at = int(time())
file.on_frameio = True
file.upload_verified = False
db_queue.put([file, 'save'])
def logout():
"""Logout of Frame.io and clear database."""
config.authenticated_client = None
config.client_expires = 0
login = Login.select().limit(1).get()
db_queue.put([login, 'delete'])
Login.create() # Default row
for asset in Asset.select():
db_queue.put([asset, 'delete'])
for project in Project.select():
db_queue.put([project, 'delete'])
sync_db.close()
return Response(status=200)
def update_project(project_id):
"""Update selected project in DB."""
try:
project = Project.get(Project.project_id == project_id)
except Project.DoesNotExist:
return Response('Bad request', status=400)
req = request.get_json()
if req.get('sync') is not None:
if req['sync'] is False:
logger.info('Sync changed to FALSE for {}'.format(project.name))
project.sync = False
db_queue.put([project, 'save'])
else:
logger.info('Sync changed to TRUE for {}'.format(project.name))
project.sync = True
db_queue.put([project, 'save'])
sync_db.close()
return Response(status=200)
if req.get('local_path') is not None:
abs_path = os.path.abspath(req['local_path'])
previous_path = project.local_path
if os.access(abs_path, os.W_OK):
if req.get('sub_folder') != "":
os.makedirs(os.path.join(abs_path, req.get('sub_folder')),
exist_ok=True)
project.local_path = os.path.join(abs_path,
req.get('sub_folder'))
else:
project.local_path = abs_path
logger.info('Local path change to {} for {}'.format(
project.local_path, project.name))
if previous_path != '' and previous_path != project.local_path:
project.local_path_changed = True
db_queue.put([project, 'save'])
sync_db.close()
return Response(status=200)
return Response('Invalid path', status=400)
def delete_and_reupload(self, project, asset):
"""Delete and re-upload asset to Frame.io. Max attempts: 3."""
logger.info('Deleting and re-uploading: {}'.format(asset.name))
if asset.upload_retries == 2:
logger.info(
'Asset already uploaded 3 times. Marking as successful anyway')
asset.upload_verified = True
db_queue.put([asset, 'save'])
return
try:
authenticated_client().delete_asset(asset.asset_id)
except requests.exceptions.HTTPError: # Deleted by user already.
pass
abs_path = os.path.abspath(os.path.join(project.local_path,
asset.path))
if not os.path.isfile(abs_path):
logger.info('{} not found'.format(asset.name))
db_queue.put([asset, 'delete'])
return
if os.path.dirname(asset.path) == '':
parent_asset_id = project.root_asset_id
else:
parent_asset_id = self.Asset.get(
self.Asset.project_id == project.project_id,
self.Asset.path == os.path.dirname(asset.path)).asset_id
new_asset = self.upload_asset(abs_path, parent_asset_id)
asset.asset_id = new_asset['id']
asset.original = new_asset['original']
asset.uploaded_at = int(time())
asset.on_frameio = True
asset.upload_verified = False
asset.upload_retries += 1
db_queue.put([asset, 'save'])
def update_frameio_assets(self, project, ignore_folders):
"""Fetch assets that've been added since last scan and add them to DB.
:Args:
project (DB Project)
ignore_folders (List)
"""
# Always overscan by 10 minutes to help avoid missing assets.
new_scan_timestamp = (datetime.now(timezone.utc) -
timedelta(minutes=10)).isoformat()
try:
account_id = authenticated_client().get_project(
project.project_id)['root_asset']['account_id']
updated_assets = authenticated_client().get_updated_assets(
account_id, project.project_id, project.last_frameio_scan)
except (requests.exceptions.ConnectionError,
requests.exceptions.HTTPError):
raise
# Find assets not already in DB.
new_assets = []
for asset in updated_assets:
try:
self.Asset.get(self.Asset.asset_id == asset['id'])
except self.Asset.DoesNotExist:
new_assets.append(asset)
new_folders = [a for a in new_assets if a['type'] == 'folder']
new_folders.sort(key=lambda a: a['inserted_at']) # Oldest first
new_files = [a for a in new_assets if a['type'] == 'file']
added_folders = 0
added_files = 0
duplicates_folders = 0
duplicates_files = 0
# Filter out duplicate folders with same name/path
new_folders_filtered = []
for folder in new_folders:
if (folder['name'] + folder['parent_id']) not in [
f['name'] + f['parent_id'] for f in new_folders_filtered
]:
new_folders_filtered.append(folder)
for folder in new_folders_filtered:
ignore = False
path = ''
if folder['name'] in ignore_folders or self.wildcard_match(
folder['name'], ignore_folders):
ignore = True
if folder['parent_id'] == project.root_asset_id:
path = folder['name']
else:
try:
parent = self.Asset.get(
self.Asset.asset_id == folder['parent_id'])
if parent.path != '':
path = os.path.join(parent.path, folder['name'])
if parent.ignore:
ignore = True
except self.Asset.DoesNotExist:
pass
# If folder has the same path/name as an existing one, ignore it
try:
self.Asset.get(self.Asset.path == path,
self.Asset.project_id == project.project_id)
ignore = True
duplicates_folders += 1
except self.Asset.DoesNotExist:
pass
new_asset = self.Asset(name=folder['name'],
project_id=project.project_id,
path=path,
asset_id=folder['id'],
parent_id=folder['parent_id'],
ignore=ignore,
on_frameio=True)
db_queue.put([new_asset, 'save'])
added_folders += 1
# If folders are out of order from Frame.io we need to calc paths.
if self.Asset.select().where(self.Asset.path == ''):
self.calculate_missing_paths(project=project,
parent_id=project.root_asset_id,
parent_path='',
ignore_folders=ignore_folders,
ignore=False)
for file in new_files:
if file['upload_completed_at'] is not None:
ignore = False
if file['parent_id'] == project.root_asset_id:
parent_path = ''
else:
try:
parent = self.Asset.get(
self.Asset.asset_id == file['parent_id'])
if parent.path == '':
logger.info(
"Parent to {} path is not set, retry".format(
file['name']))
continue
parent_path = parent.path
if parent.ignore:
ignore = True
except self.Asset.DoesNotExist:
logger.info('Parent to {} not found, retry'.format(
file['name']))
continue
# Only add files with unique path and name.
asset_path = os.path.join(parent_path, file['name'])
try:
self.Asset.get(self.Asset.path == asset_path,
self.Asset.project_id == project.project_id)
duplicates_files += 1
except self.Asset.DoesNotExist:
new_asset = self.Asset(name=file['name'],
project_id=project.project_id,
path=asset_path,
is_file=True,
asset_id=file['id'],
parent_id=file['parent_id'],
original=file['original'],
ignore=ignore,
on_frameio=True)
db_queue.put([new_asset, 'save'])
added_files += 1
if added_folders - duplicates_folders != 0:
logger.info('New folders on Frame.io for project {}'.format(
project.name))
if added_files - duplicates_files != 0:
logger.info('New files on Frame.io for project {}'.format(
project.name))
if len(new_files) == added_files: # All done. Moving up timestamp.
project.last_frameio_scan = new_scan_timestamp
db_queue.put([project, 'save'])
def update_local_assets(self, project, ignore_folders):
"""Scan local storage for assets and creates new ones in DB.
:Args:
project (DB project)
ignore_folders (List)
"""
abs_project_path = os.path.abspath(project.local_path)
if not os.path.isdir(abs_project_path):
logger.info(
'Local folder for {} not found, turning off sync'.format(
project.name))
self.delete_db_project(project)
return
new_scan_time = int(time()) - 500 # Overscan to avoid missing assets.
all_assets_ready = True
new_folders = False
new_files = False
for root, dirs, files in os.walk(abs_project_path, topdown=True):
dirs[:] = [
d for d in dirs
if d not in ignore_folders and not d.startswith(".")
and not self.wildcard_match(d, ignore_folders)
]
for name in files:
full_path = os.path.join(root, name)
if not name.startswith('.'):
try:
create_time = os.path.getctime(full_path)
except FileNotFoundError:
all_assets_ready = False
continue
# Add new file criteria
# - Created since last complete scan (all_assets_ready)
# - Not changed in the last 60 secs
# - Size not 0
if create_time > project.last_local_scan:
if (time() - create_time) < 60 or os.path.getsize(
full_path) == 0:
all_assets_ready = False
continue
path = os.path.relpath(full_path, abs_project_path)
try:
db_asset = self.Asset.get(
self.Asset.project_id == project.project_id,
self.Asset.path == path)
# Already synced
if db_asset.on_local_storage is False:
db_asset.on_local_storage = True
db_queue.put([db_asset, 'save'])
# New file
except self.Asset.DoesNotExist:
try:
file_hash = calculate_hash(full_path)
except FileNotFoundError:
all_assets_ready = False
continue
new_files = True
new_asset = self.Asset(
name=name,
project_id=project.project_id,
path=path,
is_file=True,
on_local_storage=True,
local_xxhash=file_hash)
db_queue.put([new_asset, 'save'])
for name in dirs:
full_path = os.path.join(root, name)
try:
create_time = os.path.getctime(full_path)
except FileNotFoundError:
all_assets_ready = False
continue
if create_time > project.last_local_scan:
path = os.path.relpath(full_path, abs_project_path)
try:
db_asset = self.Asset.get(
self.Asset.project_id == project.project_id,
self.Asset.path == path)
# Already synced
if db_asset.on_local_storage is False:
db_asset.on_local_storage = True
db_queue.put([db_asset, 'save'])
except self.Asset.DoesNotExist:
new_folders = True
new_asset = self.Asset(name=name,
project_id=project.project_id,
on_local_storage=True,
path=path)
db_queue.put([new_asset, 'save'])
if all_assets_ready:
project.last_local_scan = new_scan_time
db_queue.put([project, 'save'])
if new_folders:
logger.info('New local folders for project {}'.format(
project.name))
if new_files:
logger.info('New local files for project {}'.format(project.name))
def delete_assets_from_db(self, project):
"""Delete all assets from DB."""
for asset in self.Asset.select().where(
self.Asset.project_id == project.project_id):
db_queue.put([asset, 'delete'])
def download_new_assets(self, project):
"""Get new assets from DB and download them"""
new_folders = self.Asset.select().where(
(self.Asset.on_local_storage == False)
& (self.Asset.is_file == False)
& (self.Asset.project_id == project.project_id)
& (self.Asset.ignore == False) & (self.Asset.path != ''))
new_files = self.Asset.select().where(
(self.Asset.on_local_storage == False)
& (self.Asset.is_file == True)
& (self.Asset.project_id == project.project_id)
& (self.Asset.ignore == False))
if len(new_folders) == 0 and len(new_files) == 0:
return
for folder in new_folders:
logger.info('Creating local folder: {}'.format(
os.path.join(project.local_path, folder.path)))
os.makedirs(os.path.join(project.local_path, folder.path),
exist_ok=True)
folder.on_local_storage = True
db_queue.put([folder, 'save'])
for file in new_files:
try:
asset = authenticated_client().get_asset(file.asset_id)
except requests.exceptions.HTTPError:
logger.info('File removed from Frame.io')
db_queue.put(file, 'delete')
continue
if asset['checksums'] is None:
logger.info('No checksum for {}'.format(file.name))
# Allow Frame.io some time to calculate hash, retry next loop
asset_uploaded_epoch = parser.parse(
asset['upload_completed_at']).timestamp()
if time() - asset_uploaded_epoch < 300:
logger.info('Waiting for checksum'.format(file.name))
continue
download_folder = os.path.join(project.local_path,
os.path.dirname(file.path))
if os.path.isdir(download_folder):
logger.info('Downloading: {}'.format(file.path))
try:
authenticated_client().download(
asset, download_folder=download_folder, replace=False)
except FileExistsError:
logger.info('{} already exists.'.format(file.path))
# Add local props to new file
file.on_local_storage = True
db_queue.put([file, 'save'])
else:
logger.info('Download folder not found: {}'.format(file.path))
db_queue.put([file, 'delete'])
