return False
# Get timestamp for last build and create timestamp for this
# build.
# This is done here to avoid races in the cache
# between mRSL files that are being modified while cache is
# being build.
last_buildtime = 0
if os.path.exists(buildtimestamp_file):
last_buildtime = os.path.getmtime(buildtimestamp_file)
# Touch buildtimestamp file, so the modified time of it is
# updated
touch(buildtimestamp_file)
# Traverse mRSL dir and update cache
for (root, _, files) in os.walk(root_dir, topdown=True):
# skip all dot dirs - they are from repos etc and _not_ jobs
if root.find(os.sep + '.') != -1:
continue
for name in files:
filename = os.path.join(root, name)
# Only files modified since last update is checked
if os.path.getmtime(filename) > last_buildtime:
return False
# Get timestamp for last build and create timestamp for this
# build.
# This is done here to avoid races in the cache
# between mRSL files that are being modified while cache is
# being build.
last_buildtime = 0
if os.path.exists(buildtimestamp_file):
last_buildtime = os.path.getmtime(buildtimestamp_file)
# Touch buildtimestamp file, so the modified time of it is
# updated
touch(buildtimestamp_file, self.__configuration)
# Traverse mRSL dir and update cache
for (root, _, files) in os.walk(root_dir, topdown=True):
# skip all dot dirs - they are from repos etc and _not_ jobs
if root.find(os.sep + '.') != -1:
continue
for name in files:
filename = os.path.join(root, name)
# Only files modified since last update is checked
if os.path.getmtime(filename) > last_buildtime:
def check_mrsl_files(
configuration,
job_queue,
executing_queue,
only_new,
logger,
):
"""Check job files on disk in order to initialize job queue after
(re)start of grid_script.
"""
# We only check files modified since last start if possible
last_start = 0
last_start_file = os.path.join(configuration.mig_system_files,
'grid_script_laststart')
if os.path.exists(last_start_file):
last_start = os.path.getmtime(last_start_file)
check_mrsl_files_start_time = time.time()
# TODO: switch to listdir or glob? all files are in mrsl_files_dir/*/*.mRSL
for (root, _, files) in os.walk(configuration.mrsl_files_dir):
# skip all dot dirs - they are from repos etc and _not_ jobs
if root.find(os.sep + '.') != -1:
continue
# skip all dirs without any recent changes
if only_new and os.path.getmtime(root) < last_start:
logger.info('check mRSL files: skipping unchanged dir: %s' % root)
continue
logger.info('check mRSL files: inspecting %d files in %s' % \
(len(files), root))
file_count = 0
for name in files:
filename = os.path.join(root, name)
file_count += 1
if file_count % 1000 == 0:
logger.info('check mRSL files: %d files in %s checked' % \
(file_count, root))
if os.path.getmtime(filename) < last_start:
if only_new:
#logger.debug('skipping treated mrsl file: %s'
# % filename)
continue
logger.info('parsing possibly outdated mrsl file: %s'
% filename)
job_dict = io.unpickle(filename, logger)
if not job_dict:
logger.error('could not open and unpickle: %s' % filename)
continue
if job_dict['STATUS'] == 'PARSE':
# parse is ok, since mRSL file exists
# tell 'grid_script' and let grid_script put it into the queue
logger.info('Found a file with PARSE status: %s'
% job_dict['JOB_ID'])
job_id = job_dict['JOB_ID']
client_id = job_dict['USER_CERT']
client_dir = client_id_dir(client_id)
message = 'USERJOBFILE %s/%s\n' % (client_dir, job_id)
if not send_message_to_grid_script(message, logger,
configuration):
print 'Fatal error: Could not write to grid stdin'
elif job_dict['STATUS'] == 'QUEUED'\
and not job_queue.get_job_by_id(job_dict['JOB_ID']):
# put in job queue
logger.info('USERJOBFILE: There were %s jobs in the job_queue'
% job_queue.queue_length())
job_queue.enqueue_job(job_dict,
job_queue.queue_length())
logger.info("Now there's %s (QUEUED job %s added)"
% (job_queue.queue_length(),
job_dict['JOB_ID']))
elif job_dict['STATUS'] == 'EXECUTING'\
and not executing_queue.get_job_by_id(job_dict['JOB_ID'
]):
# put in executing queue
logger.info('USERJOBFILE: There were %s jobs in the executing_queue'
% executing_queue.queue_length())
executing_queue.enqueue_job(job_dict,
executing_queue.queue_length())
logger.info("Now there's %s (EXECUTING job %s added)"
% (executing_queue.queue_length(),
job_dict['JOB_ID']))
else:
# logger.debug('Job in %s is already treated' % filename)
continue
# update last_start_file access times. Note the timestamp is not "now" but
# when check_mrsl_files was called to avoid loosing any jobs being parsed
# at the same time as this function is running.
logger.info('setting time of last_start_file %s to %s'
% (last_start_file, check_mrsl_files_start_time))
io.touch(last_start_file, check_mrsl_files_start_time)
check_mrsl_files_end_time = time.time()
logger.info('finished checking for mRSL files in %fs' % \
(check_mrsl_files_end_time-check_mrsl_files_start_time))
def check_mrsl_files(
configuration,
job_queue,
executing_queue,
only_new,
logger,
):
"""Check job files on disk in order to initialize job queue after
(re)start of grid_script.
"""
# We only check files modified since last start if possible
last_start = 0
last_start_file = os.path.join(configuration.mig_system_files,
'grid_script_laststart')
if os.path.exists(last_start_file):
last_start = os.path.getmtime(last_start_file)
check_mrsl_files_start_time = time.time()
# TODO: switch to listdir or glob? all files are in mrsl_files_dir/*/*.mRSL
for (root, _, files) in os.walk(configuration.mrsl_files_dir):
# skip all dot dirs - they are from repos etc and _not_ jobs
if root.find(os.sep + '.') != -1:
continue
# skip all dirs without any recent changes
if only_new and os.path.getmtime(root) < last_start:
logger.info('check mRSL files: skipping unchanged dir: %s' % root)
continue
logger.info('check mRSL files: inspecting %d files in %s' % \
(len(files), root))
file_count = 0
for name in files:
filename = os.path.join(root, name)
file_count += 1
if file_count % 1000 == 0:
logger.info('check mRSL files: %d files in %s checked' % \
(file_count, root))
if os.path.getmtime(filename) < last_start:
if only_new:
#logger.debug('skipping treated mrsl file: %s'
# % filename)
continue
logger.info('parsing possibly outdated mrsl file: %s' %
filename)
job_dict = io.unpickle(filename, logger)
if not job_dict:
logger.error('could not open and unpickle: %s' % filename)
continue
if job_dict['STATUS'] == 'PARSE':
# parse is ok, since mRSL file exists
# tell 'grid_script' and let grid_script put it into the queue
logger.info('Found a file with PARSE status: %s' %
job_dict['JOB_ID'])
job_id = job_dict['JOB_ID']
client_id = job_dict['USER_CERT']
client_dir = client_id_dir(client_id)
message = 'USERJOBFILE %s/%s\n' % (client_dir, job_id)
if not send_message_to_grid_script(message, logger,
configuration):
print 'Fatal error: Could not write to grid stdin'
elif job_dict['STATUS'] == 'QUEUED'\
and not job_queue.get_job_by_id(job_dict['JOB_ID']):
# put in job queue
logger.info(
'USERJOBFILE: There were %s jobs in the job_queue' %
job_queue.queue_length())
job_queue.enqueue_job(job_dict, job_queue.queue_length())
logger.info("Now there's %s (QUEUED job %s added)" %
(job_queue.queue_length(), job_dict['JOB_ID']))
elif job_dict['STATUS'] == 'EXECUTING'\
and not executing_queue.get_job_by_id(job_dict['JOB_ID'
]):
# put in executing queue
logger.info(
'USERJOBFILE: There were %s jobs in the executing_queue' %
executing_queue.queue_length())
executing_queue.enqueue_job(job_dict,
executing_queue.queue_length())
logger.info(
"Now there's %s (EXECUTING job %s added)" %
(executing_queue.queue_length(), job_dict['JOB_ID']))
else:
# logger.debug('Job in %s is already treated' % filename)
continue
# update last_start_file access times. Note the timestamp is not "now" but
# when check_mrsl_files was called to avoid loosing any jobs being parsed
# at the same time as this function is running.
logger.info('setting time of last_start_file %s to %s' %
(last_start_file, check_mrsl_files_start_time))
io.touch(last_start_file,
configuration,
timestamp=check_mrsl_files_start_time)
check_mrsl_files_end_time = time.time()
logger.info('finished checking for mRSL files in %fs' % \
(check_mrsl_files_end_time-check_mrsl_files_start_time))
def _set_last_expire(configuration, proto):
"""Set last expire timestamp"""
last_expired_filepath = os.path.join(
configuration.mig_system_run,
"%s.%s" % (proto, _last_expired_filename))
return touch(last_expired_filepath, configuration)
def main(client_id, user_arguments_dict):
"""Main function used by front end"""
(configuration, logger, output_objects, op_name) = \
initialize_main_variables(client_id, op_header=False)
client_dir = client_id_dir(client_id)
defaults = signature()[1]
(validate_status, accepted) = validate_input_and_cert(
user_arguments_dict,
defaults,
output_objects,
client_id,
configuration,
allow_rejects=False,
)
if not validate_status:
WARNING_MSG = str(accepted)
output_objects.append({'object_type': 'warning',
'text': WARNING_MSG})
return (accepted, returnvalues.CLIENT_ERROR)
action = ''.join(accepted['action'])
flags = ''.join(accepted['flags'])
path = ''.join(accepted['path'])
extension = ''.join(accepted['extension'])
# Please note that base_dir must end in slash to avoid access to other
# user dirs when own name is a prefix of another user name
base_dir = os.path.abspath(os.path.join(configuration.user_home,
client_dir)) + os.sep
abs_path = os.path.join(base_dir, path)
settings_dict = load_settings(client_id, configuration)
javascript = None
title_entry = find_entry(output_objects, 'title')
title_entry['text'] = 'FILEMETAIO Management'
title_entry['javascript'] = javascript
output_objects.append({'object_type': 'header',
'text': 'FILEMETAIO Management'})
status = returnvalues.ERROR
if flags == 'i':
if action == 'list':
image_meta = get_image_file_settings(logger, abs_path)
if image_meta is not None:
extension_list = []
settings_status_list = []
settings_progress_list = []
image_count_list = []
for entry in image_meta:
extension_list.append(entry['extension'])
settings_status_list.append(entry['settings_status'
])
settings_progress_list.append(entry['settings_update_progress'
])
image_count_list.append(get_image_file_count(logger,
abs_path, entry['extension']))
output_objects.append({
'object_type': 'image_settings_list',
'extension_list': extension_list,
'settings_status_list': settings_status_list,
'settings_progress_list': settings_progress_list,
'image_count_list': image_count_list,
})
status = returnvalues.OK
else:
status = returnvalues.ERROR
ERROR_MSG = "No image settings found for path: '%s'" \
% path
output_objects.append({'object_type': 'text',
'text': ERROR_MSG})
logger.error('filemetaio.py: %s -> %s' % (action,
ERROR_MSG))
elif action == 'remove_dir':
remove_ext = None
vgrid_name = path.split('/')[0]
if extension != '':
remove_ext = extension
try:
(result, removed_ext_list) = \
remove_image_file_settings(logger, abs_path,
remove_ext)
except Exception, ex:
logger.debug(str(traceback.format_exc()))
if result is not None:
result = returnvalues.OK
else:
result = returnvalues.ERROR
ERROR_MSG = \
'Unable to remove image settings for path: %s' \
% path
output_objects.append({'object_type': 'text',
'text': ERROR_MSG})
logger.error('filemetaio.py: %s -> %s' % (action,
ERROR_MSG))
for removed_ext in removed_ext_list:
abs_last_modified_filepath = \
__get_image_settings_trigger_last_modified_filepath(logger,
abs_path, removed_ext)
# Remove trigger
if delete_file(abs_last_modified_filepath, logger):
# FYSIKER HACK: Sleep 1 to prevent trigger rule/event race
# TODO: Modify events handler to accept trigger action + delete
time.sleep(1)
# Remove old vgrid submit trigger for files
rule_id = __get_image_file_trigger_rule_id(logger,
path, removed_ext)
status = __remove_image_file_trigger(
configuration,
vgrid_name,
path,
extension,
rule_id,
output_objects,
)
if status != returnvalues.OK:
result = status
# Remove old vgrid submit trigger for settings
rule_id = \
__get_image_settings_trigger_rule_id(logger,
path, removed_ext)
status = __remove_image_settings_trigger(
configuration,
vgrid_name,
path,
extension,
rule_id,
output_objects,
)
if status != returnvalues.OK:
result = status
else:
result = returnvalues.ERROR
ERROR_MSG = 'Unable to remove file: %s ' \
% abs_last_modified_filepath
output_objects.append({'object_type': 'text',
'text': ERROR_MSG})
logger.error('filemetaio.py: %s -> %s' % (action,
ERROR_MSG))
elif action == 'get_dir':
image_count = get_image_file_count(logger, abs_path,
extension)
image_meta = get_image_file_setting(logger, abs_path,
extension)
if image_meta is not None:
extension = str(image_meta['extension'])
settings_status = str(image_meta['settings_status'])
settings_update_progress = \
str(image_meta['settings_update_progress'])
settings_recursive = str(image_meta['settings_recursive'
])
image_count = str(image_count)
image_type = str(image_meta['image_type'])
offset = str(image_meta['offset'])
x_dimension = str(image_meta['x_dimension'])
y_dimension = str(image_meta['y_dimension'])
preview_image_extension = \
str(image_meta['preview_image_extension'])
preview_x_dimension = \
str(image_meta['preview_x_dimension'])
preview_y_dimension = \
str(image_meta['preview_y_dimension'])
preview_cutoff_min = str(image_meta['preview_cutoff_min'
])
preview_cutoff_max = str(image_meta['preview_cutoff_max'
])
data_type = str(image_meta['data_type'])
output_objects.append({
'object_type': 'image_setting',
'path': path,
'extension': extension,
'settings_status': settings_status,
'settings_update_progress': settings_update_progress,
'settings_recursive': settings_recursive,
'image_count': image_count,
'image_type': image_type,
'offset': offset,
'x_dimension': x_dimension,
'y_dimension': y_dimension,
'preview_image_extension': preview_image_extension,
'preview_x_dimension': preview_x_dimension,
'preview_y_dimension': preview_y_dimension,
'preview_cutoff_min': preview_cutoff_min,
'preview_cutoff_max': preview_cutoff_max,
'data_type': data_type,
})
status = returnvalues.OK
else:
status = returnvalues.ERROR
ERROR_MSG = \
"No image setting information for path: '%s', extension: '%s'" \
% (path, extension)
output_objects.append({'object_type': 'text',
'text': ERROR_MSG})
logger.error('filemetaio.py: %s -> %s' % (action,
ERROR_MSG))
elif action == 'put_dir':
settings_status = ''.join(accepted['settings_status'])
if ''.join(accepted['settings_recursive']) == 'True':
settings_recursive = True
else:
settings_recursive = False
image_type = ''.join(accepted['image_type'])
data_type = ''.join(accepted['data_type'])
offset = int(''.join(accepted['offset']))
x_dimension = int(''.join(accepted['x_dimension']))
y_dimension = int(''.join(accepted['y_dimension']))
preview_image_extension = \
''.join(accepted['preview_image_extension'])
preview_x_dimension = \
int(''.join(accepted['preview_x_dimension']))
preview_y_dimension = \
int(''.join(accepted['preview_y_dimension']))
preview_cutoff_min = \
float(''.join(accepted['preview_cutoff_min']))
preview_cutoff_max = \
float(''.join(accepted['preview_cutoff_max']))
path_array = path.split('/')
vgrid_name = path_array[0]
vgrid_data_path = '/'.join(path_array[1:])
vgrid_meta_path = os.path.join(vgrid_data_path, __metapath)
vgrid_image_meta_path = os.path.join(vgrid_data_path,
__image_metapath)
OK_MSG = \
"Created/updated settings for image extension: '%s' for path '%s'" \
% (extension, path)
ERROR_MSG = \
"Failed to change settings for image extension: '%s' for path: '%s'" \
% (extension, path)
(is_valid, is_valid_msg) = __is_valid_image_settings_update(
configuration,
base_dir,
vgrid_name,
vgrid_data_path,
extension,
settings_recursive,
)
if is_valid:
status = returnvalues.OK
else:
status = returnvalues.ERROR
output_objects.append({'object_type': 'error_text',
'text': ERROR_MSG})
output_objects.append({'object_type': 'error_text',
'text': is_valid_msg})
logger.error('filemetaio.py: %s -> %s' % (action,
ERROR_MSG))
logger.error('filemetaio.py: %s -> %s' % (action,
is_valid_msg))
# Ensure meta path existence
if status == returnvalues.OK:
makedirs_rec(os.path.join(base_dir,
os.path.join(vgrid_name,
vgrid_meta_path)), configuration)
# Ensure image meta path existence
makedirs_rec(os.path.join(base_dir,
os.path.join(vgrid_name,
vgrid_image_meta_path)), configuration)
try:
add_status = add_image_file_setting(
logger,
abs_path,
extension,
settings_status,
None,
settings_recursive,
image_type,
data_type,
offset,
x_dimension,
y_dimension,
preview_image_extension,
preview_x_dimension,
preview_y_dimension,
preview_cutoff_min,
preview_cutoff_max,
overwrite=True,
)
except Exception, ex:
add_status = False
logger.debug(str(traceback.format_exc()))
if add_status:
status = returnvalues.OK
output_objects.append({'object_type': 'text',
'text': OK_MSG})
else:
status = returnvalues.ERROR
output_objects.append({'object_type': 'error_text',
'text': ERROR_MSG})
logger.error('filemetaio.py: %s -> %s' % (action,
ERROR_MSG))
if status == returnvalues.OK:
# Generate vgrid trigger for files
if settings_recursive:
vgrid_trigger_path = os.path.join(vgrid_data_path,
'*/*.%s' % extension)
else:
vgrid_trigger_path = os.path.join(vgrid_data_path,
'*.%s' % extension)
rule_id = __get_image_file_trigger_rule_id(logger,
path, extension)
rule_dict = {
'rule_id': rule_id,
'vgrid_name': vgrid_name,
'path': vgrid_trigger_path,
'changes': ['created', 'modified', 'deleted',
'moved'],
'run_as': client_id,
'action': 'submit',
'arguments': 'template_from_filemetaio.py',
'templates': [__get_image_update_preview_mrsl_template(path)],
'settle_time': '60s',
'rate_limit': '',
}
# Remove old vgrid submit trigger for files
status = __remove_image_file_trigger(
configuration,
vgrid_name,
path,
extension,
rule_id,
output_objects,
)
if status == returnvalues.OK:
# Add generated vgrid submit trigger for files
(add_status, add_msg) = \
vgrid_add_triggers(configuration, vgrid_name,
[rule_dict])
if add_status:
status = returnvalues.OK
OK_MSG = \
"Created/updated image file trigger for extension: '%s', path '%s'" \
% (extension, path)
output_objects.append({'object_type': 'text',
'text': OK_MSG})
else:
status = returnvalues.ERROR
ERROR_MSG = \
"Failed change image file trigger for extension: '%s', path '%s'" \
% (extension, path)
ERROR_MSG2 = "Makes sure '%s' is a VGrid" \
% vgrid_name
output_objects.append({'object_type': 'error_text',
'text': ERROR_MSG})
output_objects.append({'object_type': 'error_text',
'text': ERROR_MSG2})
logger.error('filemetaio.py: %s -> %s' % (action,
ERROR_MSG))
logger.error('filemetaio.py: %s -> %s' % (action,
ERROR_MSG2))
if status == returnvalues.OK:
# Generate vgrid trigger for settings
vgrid_path = '/'.join(path.split('/')[1:])
vgrid_trigger_filepath = \
__get_image_settings_trigger_last_modified_filepath(logger,
vgrid_path, extension)
rule_id = __get_image_settings_trigger_rule_id(logger,
path, extension)
rule_dict = {
'rule_id': rule_id,
'vgrid_name': vgrid_name,
'path': vgrid_trigger_filepath,
'changes': ['modified', 'deleted'],
'run_as': client_id,
'action': 'submit',
'arguments': 'template_from_filemetaio.py',
'templates': [__get_image_create_previews_mrsl_template(path,
extension)],
'settle_time': '1s',
'rate_limit': '',
}
# Remove old vgrid submit trigger for settings
status = __remove_image_settings_trigger(
configuration,
vgrid_name,
path,
extension,
rule_id,
output_objects,
)
if status == returnvalues.OK:
# Add generated vgrid submit trigger for settings
(add_status, add_msg) = \
vgrid_add_triggers(configuration, vgrid_name,
[rule_dict])
if add_status:
status = returnvalues.OK
OK_MSG = \
"Created/updated old image setting trigger for extension: '%s', path '%s'" \
% (extension, path)
output_objects.append({'object_type': 'text',
'text': OK_MSG})
else:
status = returnvalues.ERROR
ERROR_MSG = \
"Failed change old image setting trigger for extension: '%s', path '%s'" \
% (extension, path)
ERROR_MSG2 = "Makes sure '%s' is a VGrid" \
% vgrid_name
output_objects.append({'object_type': 'error_text',
'text': ERROR_MSG})
output_objects.append({'object_type': 'error_text',
'text': ERROR_MSG2})
logger.error('filemetaio.py: %s -> %s' % (action,
ERROR_MSG))
logger.error('filemetaio.py: %s -> %s' % (action,
ERROR_MSG2))
if status == returnvalues.OK:
# Trigger Trigger (Trigger Happty)
abs_vgrid_trigger_filepath = os.path.join(base_dir,
os.path.join(vgrid_name,
vgrid_trigger_filepath))
# FYSIKER HACK: Sleep 1 to prevent trigger rule/event race
# TODO: Modify events handler to accept add+trigger action
time.sleep(1)
timestamp = time.time()
touch(abs_vgrid_trigger_filepath, timestamp)