def __is_valid_image_settings_update(
configuration,
base_dir,
vgrid_name,
vgrid_path,
extension,
settings_recursive,
):
"""Check if valid image settings update"""
result = True
msg = ''
logger = configuration.logger
# Check for vgrid
(status, vgrid_list) = vgrid_list_vgrids(configuration)
if not status or status and not vgrid_name in vgrid_list:
result = False
msg = "'%s' is _NOT_ workflow enabled." % vgrid_name
# Check for child folder image settings
if result and settings_recursive:
abs_vgrid_path = os.path.join(base_dir,
os.path.join(vgrid_name, vgrid_path))
for path in listdirs_rec(abs_vgrid_path):
try:
image_meta = get_image_file_setting(logger, path,
extension)
except Exception, ex:
image_meta = None
logger.debug(str(traceback.format_exc()))
if image_meta is not None:
result = False
current_vgrid_path = path.replace(base_dir, '', 1)
msg = \
"Settings for extension: '%s' found in path: '%s'." \
% (extension, current_vgrid_path)
msg = '%s Overloading _NOT_ supported' % msg
def update_preview(logger, base_path, filepath):
"""Update preview for image *filepath*"""
filepath_list = filepath.split(os.sep)
path = os.sep.join(filepath_list[0:-1])
filename = filepath_list[-1]
extension = filename.split('.')[-1]
logger.debug("base_path: '%s', path: '%s', filename: '%s', extension: '%s'"
% (base_path, path, filename, extension))
result = False
image = {}
image['base_path'] = base_path
image['path'] = path
image['filename'] = filename
image['setting'] = get_image_file_setting(logger, base_path,
extension)
if image['setting'] is not None:
if fill_data(logger, image) and fill_image_stats(logger, image) \
and fill_md5sum(logger, image) and fill_preview(logger,
image) and write_preview_image(logger, image) \
and add_image_meta_data(logger, image) \
and add_preview_data(logger, image) \
and fill_preview_histogram(logger, image) \
and add_preview_histogram(logger, image):
result = True
else:
logger.info('Skipping update for: %s, %s' % (base_path,
filepath))
logger.debug('result: %s' % str(result))
return result
def update_previews(logger, base_path, extension):
"""Update image previews for *extension* in *base_path*"""
logger.debug("base_path: '%s', extension: '%s'" % (base_path,
extension))
image_setting = get_image_file_setting(logger, base_path, extension)
logger.debug('image_setting: %s' % image_setting)
result = True
status_pending = allowed_settings_status['pending']
status_updating = allowed_settings_status['updating']
status_failed = allowed_settings_status['failed']
status_ready = allowed_settings_status['ready']
if image_setting is not None:
settings_status = image_setting['settings_status']
if settings_status == status_pending:
logger.debug('settings recursive: %s'
% image_setting['settings_recursive'])
image_setting['settings_status'] = status_updating
logger.debug('settings status: %s'
% image_setting['settings_status'])
# Count files to process and set status / update progress
processed_filecount = 0
total_filecount = 0
if image_setting['settings_recursive']:
for (root, _, files) in os.walk(base_path):
for name in files:
if not name.startswith('.') \
and name.endswith('.%s' % extension):
total_filecount += 1
else:
for name in os.listdir(base_path):
if not name.startswith('.') and name.endswith('.%s'
% extension):
total_filecount += 1
image_setting['settings_update_progress'] = '%s/%s' \
% (processed_filecount, total_filecount)
logger.debug('settings_status: %s'
% image_setting['settings_status'])
logger.debug('settings_update_progress: %s'
% image_setting['settings_update_progress'])
update_image_file_setting(logger, base_path, image_setting)
# Process image files
if image_setting['settings_recursive']:
for (root, _, files) in os.walk(base_path):
for name in files:
if not name.startswith('.') \
and name.endswith('.%s' % extension):
path = root.replace(base_path, '',
1).strip('/')
filepath = os.path.join(path, name)
logger.debug('check entry -> path: %s, filepath: %s'
% (path, filepath))
if update_preview(logger, base_path,
filepath):
processed_filecount += 1
image_setting['settings_update_progress'
] = '%s/%s' \
% (processed_filecount,
total_filecount)
logger.debug('settings_update_progress: %s'
% image_setting['settings_update_progress'
])
update_image_file_setting(logger,
base_path, image_setting)
else:
result = False
else:
for name in os.listdir(base_path):
logger.debug('check entry: %s' % name)
if not name.startswith('.') and name.endswith('.%s'
% extension):
filepath = os.path.join(base_path, name)
if os.path.isfile(filepath):
if update_preview(logger, base_path, name):
processed_filecount += 1
image_setting['settings_update_progress'
] = '%s/%s' \
% (processed_filecount,
total_filecount)
logger.debug('settings_update_progress: %s'
% image_setting['settings_update_progress'
])
update_image_file_setting(logger,
base_path, image_setting)
else:
result = False
# Set final update status and progress
image_setting['settings_update_progress'] = None
if result:
image_setting['settings_status'] = status_ready
else:
image_setting['settings_status'] = status_failed
image_setting['settings_update_progress'] = None
logger.debug('status: %s' % image_setting['settings_status'
])
update_image_file_setting(logger, base_path, image_setting)
else:
logger.info("Skipping update for: %s, %s, expected status: 'Pending', found '%s'"
% (base_path, extension, settings_status))
else:
logger.info('Skipping update for: %s, %s -> No image settings found'
% (base_path, extension))
return result
def main(client_id, user_arguments_dict):
"""Main function used by front end"""
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id, op_header=False)
client_dir = client_id_dir(client_id)
defaults = signature()[1]
(validate_status, accepted) = validate_input_and_cert(
user_arguments_dict,
defaults,
output_objects,
client_id,
configuration,
allow_rejects=False,
)
if not validate_status:
WARNING_MSG = str(accepted)
output_objects.append({'object_type': 'warning',
'text': WARNING_MSG})
return (accepted, returnvalues.CLIENT_ERROR)
action = ''.join(accepted['action'])
flags = ''.join(accepted['flags'])
path = ''.join(accepted['path'])
extension = ''.join(accepted['extension'])
# Please note that base_dir must end in slash to avoid access to other
# user dirs when own name is a prefix of another user name
base_dir = os.path.abspath(os.path.join(configuration.user_home,
client_dir)) + os.sep
abs_path = os.path.join(base_dir, path)
settings_dict = load_settings(client_id, configuration)
javascript = None
title_entry = find_entry(output_objects, 'title')
title_entry['text'] = 'FILEMETAIO Management'
title_entry['javascript'] = javascript
output_objects.append({'object_type': 'header',
'text': 'FILEMETAIO Management'})
status = returnvalues.ERROR
if flags == 'i':
if action == 'list':
image_meta = get_image_file_settings(logger, abs_path)
if image_meta is not None:
extension_list = []
settings_status_list = []
settings_progress_list = []
image_count_list = []
for entry in image_meta:
extension_list.append(entry['extension'])
settings_status_list.append(entry['settings_status'
])
settings_progress_list.append(entry['settings_update_progress'
])
image_count_list.append(get_image_file_count(logger,
abs_path, entry['extension']))
output_objects.append({
'object_type': 'image_settings_list',
'extension_list': extension_list,
'settings_status_list': settings_status_list,
'settings_progress_list': settings_progress_list,
'image_count_list': image_count_list,
})
status = returnvalues.OK
else:
status = returnvalues.ERROR
ERROR_MSG = "No image settings found for path: '%s'" \
% path
output_objects.append({'object_type': 'text',
'text': ERROR_MSG})
logger.error('filemetaio.py: %s -> %s' % (action,
ERROR_MSG))
elif action == 'remove_dir':
remove_ext = None
vgrid_name = path.split('/')[0]
if extension != '':
remove_ext = extension
try:
(result, removed_ext_list) = \
remove_image_file_settings(logger, abs_path,
remove_ext)
except Exception, ex:
logger.debug(str(traceback.format_exc()))
if result is not None:
result = returnvalues.OK
else:
result = returnvalues.ERROR
ERROR_MSG = \
'Unable to remove image settings for path: %s' \
% path
output_objects.append({'object_type': 'text',
'text': ERROR_MSG})
logger.error('filemetaio.py: %s -> %s' % (action,
ERROR_MSG))
for removed_ext in removed_ext_list:
abs_last_modified_filepath = \
__get_image_settings_trigger_last_modified_filepath(logger,
abs_path, removed_ext)
# Remove trigger
if delete_file(abs_last_modified_filepath, logger):
# FYSIKER HACK: Sleep 1 to prevent trigger rule/event race
# TODO: Modify events handler to accept trigger action + delete
time.sleep(1)
# Remove old vgrid submit trigger for files
rule_id = __get_image_file_trigger_rule_id(logger,
path, removed_ext)
status = __remove_image_file_trigger(
configuration,
vgrid_name,
path,
extension,
rule_id,
output_objects,
)
if status != returnvalues.OK:
result = status
# Remove old vgrid submit trigger for settings
rule_id = \
__get_image_settings_trigger_rule_id(logger,
path, removed_ext)
status = __remove_image_settings_trigger(
configuration,
vgrid_name,
path,
extension,
rule_id,
output_objects,
)
if status != returnvalues.OK:
result = status
else:
result = returnvalues.ERROR
ERROR_MSG = 'Unable to remove file: %s ' \
% abs_last_modified_filepath
output_objects.append({'object_type': 'text',
'text': ERROR_MSG})
logger.error('filemetaio.py: %s -> %s' % (action,
ERROR_MSG))
elif action == 'get_dir':
image_count = get_image_file_count(logger, abs_path,
extension)
image_meta = get_image_file_setting(logger, abs_path,
extension)
if image_meta is not None:
extension = str(image_meta['extension'])
settings_status = str(image_meta['settings_status'])
settings_update_progress = \
str(image_meta['settings_update_progress'])
settings_recursive = str(image_meta['settings_recursive'
])
image_count = str(image_count)
image_type = str(image_meta['image_type'])
offset = str(image_meta['offset'])
x_dimension = str(image_meta['x_dimension'])
y_dimension = str(image_meta['y_dimension'])
preview_image_extension = \
str(image_meta['preview_image_extension'])
preview_x_dimension = \
str(image_meta['preview_x_dimension'])
preview_y_dimension = \
str(image_meta['preview_y_dimension'])
preview_cutoff_min = str(image_meta['preview_cutoff_min'
])
preview_cutoff_max = str(image_meta['preview_cutoff_max'
])
data_type = str(image_meta['data_type'])
output_objects.append({
'object_type': 'image_setting',
'path': path,
'extension': extension,
'settings_status': settings_status,
'settings_update_progress': settings_update_progress,
'settings_recursive': settings_recursive,
'image_count': image_count,
'image_type': image_type,
'offset': offset,
'x_dimension': x_dimension,
'y_dimension': y_dimension,
'preview_image_extension': preview_image_extension,
'preview_x_dimension': preview_x_dimension,
'preview_y_dimension': preview_y_dimension,
'preview_cutoff_min': preview_cutoff_min,
'preview_cutoff_max': preview_cutoff_max,
'data_type': data_type,
})
status = returnvalues.OK
else:
status = returnvalues.ERROR
ERROR_MSG = \
"No image setting information for path: '%s', extension: '%s'" \
% (path, extension)
output_objects.append({'object_type': 'text',
'text': ERROR_MSG})
logger.error('filemetaio.py: %s -> %s' % (action,
ERROR_MSG))
elif action == 'put_dir':
settings_status = ''.join(accepted['settings_status'])
if ''.join(accepted['settings_recursive']) == 'True':
settings_recursive = True
else:
settings_recursive = False
image_type = ''.join(accepted['image_type'])
data_type = ''.join(accepted['data_type'])
offset = int(''.join(accepted['offset']))
x_dimension = int(''.join(accepted['x_dimension']))
y_dimension = int(''.join(accepted['y_dimension']))
preview_image_extension = \
''.join(accepted['preview_image_extension'])
preview_x_dimension = \
int(''.join(accepted['preview_x_dimension']))
preview_y_dimension = \
int(''.join(accepted['preview_y_dimension']))
preview_cutoff_min = \
float(''.join(accepted['preview_cutoff_min']))
preview_cutoff_max = \
float(''.join(accepted['preview_cutoff_max']))
path_array = path.split('/')
vgrid_name = path_array[0]
vgrid_data_path = '/'.join(path_array[1:])
vgrid_meta_path = os.path.join(vgrid_data_path, __metapath)
vgrid_image_meta_path = os.path.join(vgrid_data_path,
__image_metapath)
OK_MSG = \
"Created/updated settings for image extension: '%s' for path '%s'" \
% (extension, path)
ERROR_MSG = \
"Failed to change settings for image extension: '%s' for path: '%s'" \
% (extension, path)
(is_valid, is_valid_msg) = __is_valid_image_settings_update(
configuration,
base_dir,
vgrid_name,
vgrid_data_path,
extension,
settings_recursive,
)
if is_valid:
status = returnvalues.OK
else:
status = returnvalues.ERROR
output_objects.append({'object_type': 'error_text',
'text': ERROR_MSG})
output_objects.append({'object_type': 'error_text',
'text': is_valid_msg})
logger.error('filemetaio.py: %s -> %s' % (action,
ERROR_MSG))
logger.error('filemetaio.py: %s -> %s' % (action,
is_valid_msg))
# Ensure meta path existence
if status == returnvalues.OK:
makedirs_rec(os.path.join(base_dir,
os.path.join(vgrid_name,
vgrid_meta_path)), configuration)
# Ensure image meta path existence
makedirs_rec(os.path.join(base_dir,
os.path.join(vgrid_name,
vgrid_image_meta_path)), configuration)
try:
add_status = add_image_file_setting(
logger,
abs_path,
extension,
settings_status,
None,
settings_recursive,
image_type,
data_type,
offset,
x_dimension,
y_dimension,
preview_image_extension,
preview_x_dimension,
preview_y_dimension,
preview_cutoff_min,
preview_cutoff_max,
overwrite=True,
)
except Exception, ex:
add_status = False
logger.debug(str(traceback.format_exc()))
if add_status:
status = returnvalues.OK
output_objects.append({'object_type': 'text',
'text': OK_MSG})
else:
status = returnvalues.ERROR
output_objects.append({'object_type': 'error_text',
'text': ERROR_MSG})
logger.error('filemetaio.py: %s -> %s' % (action,
ERROR_MSG))
if status == returnvalues.OK:
# Generate vgrid trigger for files
if settings_recursive:
vgrid_trigger_path = os.path.join(vgrid_data_path,
'*/*.%s' % extension)
else:
vgrid_trigger_path = os.path.join(vgrid_data_path,
'*.%s' % extension)
rule_id = __get_image_file_trigger_rule_id(logger,
path, extension)
rule_dict = {
'rule_id': rule_id,
'vgrid_name': vgrid_name,
'path': vgrid_trigger_path,
'changes': ['created', 'modified', 'deleted',
'moved'],
'run_as': client_id,
'action': 'submit',
'arguments': 'template_from_filemetaio.py',
'templates': [__get_image_update_preview_mrsl_template(path)],
'settle_time': '60s',
'rate_limit': '',
}
# Remove old vgrid submit trigger for files
status = __remove_image_file_trigger(
configuration,
vgrid_name,
path,
extension,
rule_id,
output_objects,
)
if status == returnvalues.OK:
# Add generated vgrid submit trigger for files
(add_status, add_msg) = \
vgrid_add_triggers(configuration, vgrid_name,
[rule_dict])
if add_status:
status = returnvalues.OK
OK_MSG = \
"Created/updated image file trigger for extension: '%s', path '%s'" \
% (extension, path)
output_objects.append({'object_type': 'text',
'text': OK_MSG})
else:
status = returnvalues.ERROR
ERROR_MSG = \
"Failed change image file trigger for extension: '%s', path '%s'" \
% (extension, path)
ERROR_MSG2 = "Makes sure '%s' is a VGrid" \
% vgrid_name
output_objects.append({'object_type': 'error_text',
'text': ERROR_MSG})
output_objects.append({'object_type': 'error_text',
'text': ERROR_MSG2})
logger.error('filemetaio.py: %s -> %s' % (action,
ERROR_MSG))
logger.error('filemetaio.py: %s -> %s' % (action,
ERROR_MSG2))
if status == returnvalues.OK:
# Generate vgrid trigger for settings
vgrid_path = '/'.join(path.split('/')[1:])
vgrid_trigger_filepath = \
__get_image_settings_trigger_last_modified_filepath(logger,
vgrid_path, extension)
rule_id = __get_image_settings_trigger_rule_id(logger,
path, extension)
rule_dict = {
'rule_id': rule_id,
'vgrid_name': vgrid_name,
'path': vgrid_trigger_filepath,
'changes': ['modified', 'deleted'],
'run_as': client_id,
'action': 'submit',
'arguments': 'template_from_filemetaio.py',
'templates': [__get_image_create_previews_mrsl_template(path,
extension)],
'settle_time': '1s',
'rate_limit': '',
}
# Remove old vgrid submit trigger for settings
status = __remove_image_settings_trigger(
configuration,
vgrid_name,
path,
extension,
rule_id,
output_objects,
)
if status == returnvalues.OK:
# Add generated vgrid submit trigger for settings
(add_status, add_msg) = \
vgrid_add_triggers(configuration, vgrid_name,
[rule_dict])
if add_status:
status = returnvalues.OK
OK_MSG = \
"Created/updated old image setting trigger for extension: '%s', path '%s'" \
% (extension, path)
output_objects.append({'object_type': 'text',
'text': OK_MSG})
else:
status = returnvalues.ERROR
ERROR_MSG = \
"Failed change old image setting trigger for extension: '%s', path '%s'" \
% (extension, path)
ERROR_MSG2 = "Makes sure '%s' is a VGrid" \
% vgrid_name
output_objects.append({'object_type': 'error_text',
'text': ERROR_MSG})
output_objects.append({'object_type': 'error_text',
'text': ERROR_MSG2})
logger.error('filemetaio.py: %s -> %s' % (action,
ERROR_MSG))
logger.error('filemetaio.py: %s -> %s' % (action,
ERROR_MSG2))
if status == returnvalues.OK:
# Trigger Trigger (Trigger Happty)
abs_vgrid_trigger_filepath = os.path.join(base_dir,
os.path.join(vgrid_name,
vgrid_trigger_filepath))
# FYSIKER HACK: Sleep 1 to prevent trigger rule/event race
# TODO: Modify events handler to accept add+trigger action
time.sleep(1)
timestamp = time.time()
touch(abs_vgrid_trigger_filepath, timestamp)
msg = \
"Settings for extension: '%s' found in path: '%s'." \
% (extension, current_vgrid_path)
msg = '%s Overloading _NOT_ supported' % msg
# Check for parent folder image settings
if result:
vgrid_path_array = ('%s/%s' % (vgrid_name,
vgrid_path)).split('/')[:-2]
while result and len(vgrid_path_array) > 0:
current_vgrid_path = os.sep.join(vgrid_path_array)
abs_vgrid_path = os.path.join(base_dir, current_vgrid_path)
try:
image_meta = get_image_file_setting(logger,
abs_vgrid_path, extension)
except Exception, ex:
image_meta = None
logger.debug(str(traceback.format_exc()))
if image_meta is not None \
and image_meta['settings_recursive']:
result = False
msg = \
"settings for extension: '%s' found in path: '%s'." \
% (extension, current_vgrid_path)
msg = '%s Overloading _NOT_ supported' % msg
vgrid_path_array = vgrid_path_array[:-1]
# Check image settings status
abs_path = os.path.join(base_dir, os.path.join(vgrid_name,