Example #1
0
def generate_user_key(configuration, client_id, key_filename, truncate=False):
    """Generate a new key and save it as key_filename in settings dir"""
    # TODO: switch to paramiko key generation?
    logger = configuration.logger
    key_dir = os.path.join(configuration.user_settings,
                           client_id_dir(client_id), user_keys_dir)
    key_path = os.path.join(key_dir, key_filename)
    makedirs_rec(key_dir, configuration)
    if os.path.exists(key_path) and not truncate:
        logger.error("user key %s already exists!" % key_path)
        return (False, 'user key %s already exists!' % key_filename)
    logger.debug("generating user key %s" % key_path)
    gen_proc = subprocess_popen([
        'ssh-keygen', '-t', default_key_type, '-b',
        '%d' % default_key_bits, '-f', key_path, '-N', '', '-C', key_filename
    ],
                                stdout=subprocess_pipe,
                                stderr=subprocess_pipe)
    exit_code = gen_proc.wait()
    out, err = gen_proc.communicate()
    if exit_code != 0:
        logger.error("user key generation in %s failed: %s %s (%s)" % \
                     (key_path, out, err, exit_code))
        return (False, "user key generation in %s failed!" % key_filename)
    logger.info('done generating user key %s: %s : %s (%s)' % \
                (key_path, out, err, exit_code))
    pub_key = ''
    try:
        pub_fd = open(key_path + '.pub')
        pub_key = pub_fd.read()
        pub_fd.close()
    except Exception, exc:
        logger.error("user key generation %s did not create a pub key: %s" % \
                     (key_path, exc))
        return (False, "user key generation in %s failed!" % key_filename)
Example #2
0
def transfer_result(configuration, client_id, transfer_dict, exit_code,
                    out_msg, err_msg):
    """Update status file from transfer_dict with the result from transfer
    that reurned exit_code, out_msg and err_msg.
    """
    time_stamp = datetime.datetime.now().ctime()
    transfer_id = transfer_dict['transfer_id']
    rel_src = transfer_dict.get("rel_src", False)
    if not rel_src:
        rel_src = ', '.join(transfer_dict['src'])
    res_dir = get_status_dir(configuration, client_id, transfer_id)
    makedirs_rec(res_dir, configuration)
    status_msg = '''%s: %s %s of %s in %s finished with status %s
''' % (time_stamp, transfer_dict['protocol'], transfer_dict['action'], rel_src,
       transfer_dict['transfer_id'], exit_code)
    out_msg = '%s:\n%s\n' % (time_stamp, out_msg)
    err_msg = '%s:\n%s\n' % (time_stamp, err_msg)
    status = True
    for (ext, msg) in [("status", status_msg), ("stdout", out_msg),
                       ("stderr", err_msg)]:
        path = os.path.join(res_dir, "%s.%s" % (transfer_id, ext))
        try:
            if os.path.exists(path):
                status_fd = open(path, "a")
            else:
                status_fd = open(path, "w")
            status_fd.write(msg)
            status_fd.close()
        except Exception, exc:
            logger.error("writing status file %s for %s failed: %s" %
                         (path, blind_pw(transfer_dict), exc))
            status = False
Example #3
0
def list_frozen_archives(configuration, client_id):
    """Find all frozen_archives owned by user"""
    logger = configuration.logger
    frozen_list = []
    dir_content = []

    try:
        dir_content = os.listdir(configuration.freeze_home)
    except Exception:
        if not makedirs_rec(configuration.freeze_home, configuration):
            logger.error(
                'freezefunctions.py: not able to create directory %s'
                % configuration.freeze_home)
            return (False, "archive setup is broken")
        dir_content = []

    for entry in dir_content:

        # Skip dot files/dirs

        if entry.startswith('.'):
            continue
        if is_frozen_archive(entry, configuration):

            # entry is a frozen archive - check ownership

            (meta_status, meta_out) = get_frozen_meta(entry, configuration)
            if meta_status and meta_out['CREATOR'] == client_id:
                frozen_list.append(entry)
        else:
            logger.warning(
                '%s in %s is not a directory, move it?'
                % (entry, configuration.freeze_home))
    return (True, frozen_list)
Example #4
0
def modify_data_transfers(configuration,
                          client_id,
                          transfer_dict,
                          action,
                          transfers=None):
    """Modify data transfers with given action and transfer_dict for client_id.
    In practice this a shared helper to add or remove transfers from the saved
    data transfers. The optional transfers argument can be used to pass an
    already loaded dictionary of saved transfers to avoid reloading.
    """
    logger = configuration.logger
    transfer_id = transfer_dict['transfer_id']
    if transfers is None:
        (load_status, transfers) = load_data_transfers(configuration,
                                                       client_id)
        if not load_status:
            logger.error("modify_data_transfers failed in load: %s" % \
                         transfers)
            return (load_status, transfers)

    if action == "create":
        now = datetime.datetime.now()
        transfer_dict.update({
            'created_timestamp': now,
            'updated_timestamp': now,
            'owner': client_id,
        })
        transfers[transfer_id] = transfer_dict
    elif action == "modify":
        transfer_dict['updated_timestamp'] = datetime.datetime.now()
        transfers[transfer_id].update(transfer_dict)
    elif action == "delete":
        del transfers[transfer_id]
    else:
        return (False, "Invalid action %s on data transfers" % action)

    try:
        transfers_path = os.path.join(configuration.user_settings,
                                      client_id_dir(client_id),
                                      datatransfers_filename)
        dump(transfers, transfers_path)
        res_dir = get_status_dir(configuration, client_id, transfer_id)
        makedirs_rec(res_dir, configuration)
    except Exception, err:
        logger.error("modify_data_transfers failed: %s" % err)
        return (False, 'Error updating data transfers: %s' % err)
Example #5
0
    def setUp(self):
        self.created_workflows = []
        self.username = '******'
        self.test_vgrid = default_vgrid
        if not os.environ.get('MIG_CONF', False):
            os.environ['MIG_CONF'] = '/home/mig/mig/server/MiGserver.conf'
        self.configuration = get_configuration_object()
        self.logger = self.configuration.logger
        # Ensure that the vgrid_files_home exist
        vgrid_file_path = os.path.join(self.configuration.vgrid_files_home,
                                       self.test_vgrid)
        if not os.path.exists(vgrid_file_path):
            self.assertTrue(
                makedirs_rec(vgrid_file_path,
                             self.configuration,
                             accept_existing=True))
        # Ensure that the mrsl_files home exists
        mrsl_file_path = os.path.join(self.configuration.mrsl_files_dir,
                                      self.username)
        if not os.path.exists(mrsl_file_path):
            self.assertTrue(
                makedirs_rec(mrsl_file_path,
                             self.configuration,
                             accept_existing=True))
        self.assertTrue(os.path.exists(vgrid_file_path))

        self.configuration.workflows_db_home = this_path
        self.configuration.workflows_db = \
            os.path.join(this_path, 'test_sessions_db.pickle')
        self.configuration.workflows_db_lock = \
            os.path.join(this_path, 'test_sessions_db.lock')
        self.assertTrue(
            reset_workflows(self.configuration, vgrid=self.test_vgrid))
        created = touch_workflow_sessions_db(self.configuration, force=True)
        self.assertTrue(created)
        self.session_id = create_workflow_session_id(self.configuration,
                                                     self.username)
        self.assertIsNot(self.session_id, False)
        self.assertIsNotNone(self.session_id)

        self.workflow_sessions_db = load_workflow_sessions_db(
            self.configuration)
        self.assertIn(self.session_id, self.workflow_sessions_db)
        self.workflow_session = self.workflow_sessions_db.get(
            self.session_id, None)
        self.assertIsNotNone(self.workflow_session)
Example #6
0
def __transfer_log(configuration, client_id, msg, level='info'):
    """Wrapper to send a single msg to transfer log file of client_id"""
    status_dir = get_status_dir(configuration, client_id)
    log_path = os.path.join(status_dir, configuration.site_transfer_log)
    makedirs_rec(os.path.dirname(log_path), configuration)
    transfers_logger = logging.getLogger('background-transfer')
    transfers_logger.setLevel(logging.INFO)
    handler = logging.handlers.RotatingFileHandler(
        log_path,
        maxBytes=transfers_log_size,
        backupCount=transfers_log_cnt - 1)
    formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
    handler.setFormatter(formatter)
    transfers_logger.addHandler(handler)
    if level == 'error':
        transfers_logger.error(msg)
    elif level == 'warning':
        transfers_logger.warning(msg)
    else:
        transfers_logger.info(msg)
    handler.flush()
    handler.close()
    transfers_logger.removeHandler(handler)
Example #7
0
def main(client_id, user_arguments_dict):
    """Main function used by front end"""

    (configuration, logger, output_objects, op_name) = \
        initialize_main_variables(client_id, op_header=False)
    defaults = signature()[1]
    title_entry = find_entry(output_objects, 'title')
    label = "%s" % configuration.site_vgrid_label
    title_entry['text'] = '%s Workflows' % label
    # NOTE: Delay header entry here to include vgrid_name
    (validate_status, accepted) = validate_input_and_cert(
        user_arguments_dict,
        defaults,
        output_objects,
        client_id,
        configuration,
        allow_rejects=False,
    )
    if not validate_status:
        return (accepted, returnvalues.CLIENT_ERROR)

    vgrid_name = accepted['vgrid_name'][-1]
    operation = accepted['operation'][-1]
    flags = ''.join(accepted['flags'][-1])

    if not vgrid_is_owner_or_member(vgrid_name, client_id, configuration):
        output_objects.append({
            'object_type':
            'error_text',
            'text':
            '''You must be an owner or member of %s vgrid to
access the workflows.''' % vgrid_name
        })
        return (output_objects, returnvalues.CLIENT_ERROR)

    if not operation in allowed_operations:
        output_objects.append({
            'object_type':
            'error_text',
            'text':
            '''Operation must be one of %s.''' % ', '.join(allowed_operations)
        })
        return (output_objects, returnvalues.OK)

    if operation in show_operations:

        # jquery support for tablesorter (and unused confirmation dialog)
        # table initially sorted by 0 (last update / date)

        refresh_call = 'ajax_workflowjobs("%s", "%s")' % (vgrid_name, flags)
        table_spec = {
            'table_id': 'workflowstable',
            'sort_order': '[[0,1]]',
            'refresh_call': refresh_call
        }
        (add_import, add_init,
         add_ready) = man_base_js(configuration, [table_spec])
        if operation == "show":
            add_ready += '%s;' % refresh_call
        add_ready += '''
              /* Init variables helper as foldable but closed and with individual
              heights */
              $(".variables-accordion").accordion({
                                           collapsible: true,
                                           active: false,
                                           heightStyle: "content"
                                          });
              /* fix and reduce accordion spacing */
              $(".ui-accordion-header").css("padding-top", 0)
                                       .css("padding-bottom", 0).css("margin", 0);
              /* NOTE: requires managers CSS fix for proper tab bar height */
              $(".workflow-tabs").tabs();
              $("#logarea").scrollTop($("#logarea")[0].scrollHeight);
        '''
        title_entry['script']['advanced'] += add_import
        title_entry['script']['init'] += add_init
        title_entry['script']['ready'] += add_ready
        output_objects.append({
            'object_type': 'html_form',
            'text': man_base_html(configuration)
        })

        output_objects.append({
            'object_type':
            'header',
            'text':
            '%s Workflows for %s' % (label, vgrid_name)
        })

    logger.info('vgridworkflows %s %s' % (vgrid_name, operation))

    # Iterate through jobs and list details for each

    trigger_jobs = []
    log_content = ''

    if operation in list_operations:
        trigger_job_dir = os.path.join(
            configuration.vgrid_home,
            os.path.join(vgrid_name,
                         '.%s.jobs' % configuration.vgrid_triggers))
        trigger_job_pending_dir = os.path.join(trigger_job_dir,
                                               'pending_states')
        trigger_job_final_dir = os.path.join(trigger_job_dir, 'final_states')

        if makedirs_rec(trigger_job_pending_dir, configuration) \
                and makedirs_rec(trigger_job_final_dir, configuration):
            abs_vgrid_dir = '%s/' \
                % os.path.abspath(os.path.join(configuration.vgrid_files_home,
                                               vgrid_name))
            for filename in os.listdir(trigger_job_pending_dir):
                trigger_job_filepath = \
                    os.path.join(trigger_job_pending_dir, filename)
                trigger_job = unpickle(trigger_job_filepath, logger)
                serverjob_filepath = \
                    os.path.join(configuration.mrsl_files_dir,
                                 os.path.join(
                                     client_id_dir(trigger_job['owner']),
                                     '%s.mRSL' % trigger_job['jobid']))
                serverjob = unpickle(serverjob_filepath, logger)
                if serverjob:
                    if serverjob['STATUS'] in pending_states:
                        trigger_event = trigger_job['event']
                        trigger_rule = trigger_job['rule']
                        trigger_action = trigger_event['event_type']
                        trigger_time = time.ctime(trigger_event['time_stamp'])
                        trigger_path = '%s %s' % \
                                       (trigger_event['src_path'].replace(
                                        abs_vgrid_dir, ''),
                                           trigger_event['dest_path'].replace(
                                        abs_vgrid_dir, ''))
                        job = {
                            'object_type': 'trigger_job',
                            'job_id': trigger_job['jobid'],
                            'rule_id': trigger_rule['rule_id'],
                            'path': trigger_path,
                            'action': trigger_action,
                            'time': trigger_time,
                            'status': serverjob['STATUS']
                        }
                        if not job['rule_id'].startswith(img_trigger_prefix) \
                                or verbose(flags):
                            trigger_jobs.append(job)
                    elif serverjob['STATUS'] in final_states:
                        src_path = os.path.join(trigger_job_pending_dir,
                                                filename)
                        dest_path = os.path.join(trigger_job_final_dir,
                                                 filename)
                        move_file(src_path, dest_path, configuration)
                    else:
                        logger.error(
                            'Trigger job: %s, unknown state: %s' %
                            (trigger_job['jobid'], serverjob['STATUS']))

        log_content = read_trigger_log(configuration, vgrid_name, flags)

    if operation in show_operations:

        # Always run as rule creator to avoid users being able to act on behalf
        # of ANY other user using triggers (=exploit)

        extra_fields = [
            ('path', None),
            ('match_dirs', ['False', 'True']),
            ('match_recursive', ['False', 'True']),
            ('changes', [keyword_all] + valid_trigger_changes),
            ('action', [keyword_auto] + valid_trigger_actions),
            ('arguments', None),
            ('run_as', client_id),
        ]

        # NOTE: we do NOT show saved template contents - see addvgridtriggers

        optional_fields = [('rate_limit', None), ('settle_time', None)]

        # Only include system triggers in verbose mode
        if verbose(flags):
            system_filter = []
        else:
            system_filter = [('rule_id', '%s_.*' % img_trigger_prefix)]
        (init_status,
         oobjs) = vgrid_add_remove_table(client_id,
                                         vgrid_name,
                                         'trigger',
                                         'vgridtrigger',
                                         configuration,
                                         extra_fields + optional_fields,
                                         filter_items=system_filter)
        if not init_status:
            output_objects.append({
                'object_type': 'error_text',
                'text': 'failed to load triggers: %s' % oobjs
            })
            return (output_objects, returnvalues.SYSTEM_ERROR)

        # Generate variable helper values for a few concrete samples for help
        # text
        vars_html = ''
        dummy_rule = {'run_as': client_id, 'vgrid_name': vgrid_name}
        samples = [('input.txt', 'modified'), ('input/image42.raw', 'changed')]
        for (path, change) in samples:
            vgrid_path = os.path.join(vgrid_name, path)
            vars_html += "<b>Expanded variables when %s is %s:</b><br/>" % \
                (vgrid_path, change)
            expanded = get_path_expand_map(vgrid_path, dummy_rule, change)
            for (key, val) in expanded.items():
                vars_html += "    %s: %s<br/>" % (key, val)
        commands_html = ''
        commands = get_usage_map(configuration)
        for usage in commands.values():
            commands_html += "    %s<br/>" % usage

        helper_html = """
<div class='variables-accordion'>
<h4>Help on available trigger variable names and values</h4>
<p>
Triggers can use a number of helper variables on the form +TRIGGERXYZ+ to
dynamically act on targets. Some of the values are bound to the rule owner the
%s while the remaining ones are automatically expanded for the particular
trigger target as shown in the following examples:<br/>
%s
</p>
<h4>Help on available trigger commands and arguments</h4>
<p>
It is possible to set up trigger rules that basically run any operation with a
side effect you could manually do on %s. I.e. like submitting/cancelling
a job, creating/moving/deleting a file or directory and so on. When you select
'command' as the action for a trigger rule, you have the following commands at
your disposal:<br/>
%s
</p>
</div>
""" % (label, vars_html, configuration.short_title, commands_html)

        # Make page with manage triggers tab and active jobs and log tab

        output_objects.append({
            'object_type':
            'html_form',
            'text':
            '''
    <div id="wrap-tabs" class="workflow-tabs">
<ul>
<li><a href="#manage-tab">Manage Triggers</a></li>
<li><a href="#jobs-tab">Active Trigger Jobs</a></li>
</ul>
'''
        })

        # Display existing triggers and form to add new ones

        output_objects.append({
            'object_type': 'html_form',
            'text': '''
<div id="manage-tab">
'''
        })

        output_objects.append({
            'object_type': 'sectionheader',
            'text': 'Manage Triggers'
        })
        output_objects.extend(oobjs)
        output_objects.append({
            'object_type': 'html_form',
            'text': helper_html
        })

        if configuration.site_enable_crontab:
            output_objects.append({
                'object_type':
                'html_form',
                'text':
                '''
<p>You can combine these workflows with the personal '''
            })
            output_objects.append({
                'object_type': 'link',
                'destination': 'crontab.py',
                'class': 'crontablink iconspace',
                'text': 'schedule task'
            })
            output_objects.append({
                'object_type':
                'html_form',
                'text':
                '''
facilities in case you want to trigger flows at given times rather than only
in reaction to file system events.</p>
'''
            })
        output_objects.append({
            'object_type': 'html_form',
            'text': '''
</div>
'''
        })

        # Display active trigger jobs and recent logs for this vgrid

        output_objects.append({
            'object_type': 'html_form',
            'text': '''
    <div id="jobs-tab">
    '''
        })
        output_objects.append({
            'object_type': 'sectionheader',
            'text': 'Active Trigger Jobs'
        })
        output_objects.append({
            'object_type': 'table_pager',
            'entry_name': 'job',
            'default_entries': default_pager_entries
        })

    output_objects.append({
        'object_type': 'trigger_job_list',
        'trigger_jobs': trigger_jobs
    })

    if operation in show_operations:
        output_objects.append({
            'object_type': 'sectionheader',
            'text': 'Trigger Log'
        })

    output_objects.append({
        'object_type': 'trigger_log',
        'log_content': log_content
    })
    if operation in show_operations:
        output_objects.append({
            'object_type': 'html_form',
            'text': '''
</div>
'''
        })

        output_objects.append({
            'object_type': 'html_form',
            'text': '''
</div>
'''
        })
    return (output_objects, returnvalues.OK)
Example #8
0
            'object_type':
            'error_text',
            'text':
            'Errors parsing upload files: %s' %
            '\n '.join(["%s %s" % pair for pair in upload_rejected])
        })
        for (rel_path, err) in upload_rejected:
            uploaded.append({
                'object_type': 'uploadfile',
                'name': rel_path,
                'size': -1,
                "error": "upload rejected: %s" % err
            })
        return (output_objects, returnvalues.CLIENT_ERROR)

    if not makedirs_rec(cache_dir, configuration):
        output_objects.append({
            'object_type': 'error_text',
            'text': "Problem creating temporary upload dir"
        })
        return (output_objects, returnvalues.SYSTEM_ERROR)

    if action == "status" and not upload_files:
        # Default to entire cache dir
        upload_files = [(os.path.join(upload_tmp_dir, i), '')
                        for i in os.listdir(cache_dir)]
    elif not upload_files:
        logger.error('Rejecting upload with: %s' % upload_files)
        output_objects.append({
            'object_type': 'error_text',
            'text': 'No files included to upload!'
Example #9
0
    # IMPORTANT: path must be expanded to abs for proper chrooting
    abs_dest = os.path.abspath(dest)

    # Don't use abs_path in output as it may expose underlying
    # fs layout.

    relative_dest = abs_dest.replace(dst_base, '')
    if not valid_user_path(configuration, abs_dest, dst_base, True):
        logger.warning('%s tried to %s restricted path %s ! (%s)'
                       % (client_id, op_name, abs_dest, dst))
        output_objects.append(
            {'object_type': 'error_text', 'text':
             "Invalid destination (%s expands to an illegal path)" % dst})
        return (output_objects, returnvalues.CLIENT_ERROR)
    # We must make sure target dir exists if called in import X mode
    if (share_id or freeze_id) and not makedirs_rec(abs_dest, configuration):
        logger.error('could not create import destination dir: %s' % abs_dest)
        output_objects.append(
            {'object_type': 'error_text', 'text':
             'cannot import to "%s" : file in the way?' % relative_dest})
        return (output_objects, returnvalues.SYSTEM_ERROR)
    if not check_write_access(abs_dest, parent_dir=True):
        logger.warning('%s called without write access: %s'
                       % (op_name, abs_dest))
        output_objects.append(
            {'object_type': 'error_text', 'text':
             'cannot copy to "%s": inside a read-only location!'
             % relative_dest})
        return (output_objects, returnvalues.CLIENT_ERROR)
    if share_id and not force(flags) and not check_empty_dir(abs_dest):
        logger.warning('%s called %s sharelink import with non-empty dst: %s'
Example #10
0
def main(client_id, user_arguments_dict):
    """Main function used by front end"""

    (configuration, logger, output_objects, op_name) = \
        initialize_main_variables(client_id, op_header=False)
    defaults = signature()[1]
    (validate_status, accepted) = validate_input_and_cert(
        user_arguments_dict,
        defaults,
        output_objects,
        client_id,
        configuration,
        allow_rejects=False,
        )
    if not validate_status:
        return (accepted, returnvalues.CLIENT_ERROR)

    vgrid_name = accepted['vgrid_name'][-1]

    if not vgrid_is_owner_or_member(vgrid_name, client_id,
                                    configuration):
        output_objects.append({'object_type': 'error_text',
                              'text': '''You must be an owner or member of %s vgrid to
access the workflows.'''
                               % vgrid_name})
        return (output_objects, returnvalues.CLIENT_ERROR)

    title_entry = find_entry(output_objects, 'title')
    title_entry['text'] = '%s Workflows' \
        % configuration.site_vgrid_label
    title_entry['style'] = themed_styles(configuration)
    title_entry['javascript'] = \
        '''
<script type="text/javascript" src="/images/js/jquery.js"></script>
<script type="text/javascript" src="/images/js/jquery.tablesorter.js"></script>
<script type="text/javascript" src="/images/js/jquery.tablesorter.pager.js"></script>
<script type="text/javascript" src="/images/js/jquery.tablesorter.widgets.js"></script>
<script type="text/javascript" src="/images/js/jquery-ui.js"></script>

<script type="text/javascript">
$(document).ready(function() {

          $("#logarea").scrollTop($("#logarea")[0].scrollHeight);

          // table initially sorted by 0 (last update / date) 
          var sortOrder = [[0,1]];

          // use image path for sorting if there is any inside
          var imgTitle = function(contents) {
              var key = $(contents).find("a").attr("class");
              if (key == null) {
                  key = $(contents).html();
              }
              return key;
          }

          $("#workflowstable").tablesorter({widgets: ["zebra", "saveSort"],
                                        sortList:sortOrder,
                                        textExtraction: imgTitle
                                        })
                               .tablesorterPager({ container: $("#pager"),
                                        size: %s
                                        });
     }
);
</script>
''' \
        % default_pager_entries

    output_objects.append({'object_type': 'html_form',
                          'text': '''
 <div id="confirm_dialog" title="Confirm" style="background:#fff;">
  <div id="confirm_text"><!-- filled by js --></div>
   <textarea cols="72" rows="10" id="confirm_input" style="display:none;"></textarea>
 </div>
'''})

    output_objects.append({'object_type': 'header',
                          'text': '%s Workflows for %s'
                          % (configuration.site_vgrid_label,
                          vgrid_name)})

    logger.info('vgridworkflows %s' % vgrid_name)

    # Display active trigger jobs for this vgrid

    output_objects.append({'object_type': 'sectionheader',
                          'text': 'Active Trigger Jobs'})
    html = '<table><thead><tr>'
    html += '<th>Job ID</th>'
    html += '<th>Rule</th>'
    html += '<th>Path</th>'
    html += '<th>Change</th>'
    html += '<th>Time</th>'
    html += '<th>Status</th>'
    html += '</tr></thead>'
    html += '<tbody>'

    trigger_job_dir = os.path.join(configuration.vgrid_home,
                                   os.path.join(vgrid_name, '.%s.jobs'
                                   % configuration.vgrid_triggers))
    trigger_job_pending_dir = os.path.join(trigger_job_dir,
            'pending_states')
    trigger_job_final_dir = os.path.join(trigger_job_dir, 'final_states'
            )

    if makedirs_rec(trigger_job_pending_dir, logger) \
        and makedirs_rec(trigger_job_final_dir, logger):
        abs_vgrid_dir = '%s/' \
            % os.path.abspath(os.path.join(configuration.vgrid_files_home,
                              vgrid_name))
        for filename in os.listdir(trigger_job_pending_dir):
            trigger_job_filepath = \
                os.path.join(trigger_job_pending_dir, filename)
            trigger_job = unpickle(trigger_job_filepath, logger)
            serverjob_filepath = \
                os.path.join(configuration.mrsl_files_dir,
                             os.path.join(client_id_dir(trigger_job['owner'
                             ]), '%s.mRSL' % trigger_job['jobid']))
            serverjob = unpickle(serverjob_filepath, logger)
            if serverjob:
                if serverjob['STATUS'] in pending_states:
                    trigger_event = trigger_job['event']
                    trigger_rule = trigger_job['rule']
                    trigger_action = trigger_event['event_type']
                    trigger_time = time.ctime(trigger_event['time_stamp'
                            ])
                    trigger_path = '%s %s' % (trigger_event['src_path'
                            ].replace(abs_vgrid_dir, ''),
                            trigger_event['dest_path'
                            ].replace(abs_vgrid_dir, ''))
                    html += \
                        '<tr><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td></td><td>%s</td>' \
                        % (trigger_job['jobid'], trigger_rule['rule_id'
                           ], trigger_path, trigger_action, trigger_time,
                           serverjob['STATUS'])
                elif serverjob['STATUS'] in final_states:
                    src_path = os.path.join(trigger_job_pending_dir,
                            filename)
                    dest_path = os.path.join(trigger_job_final_dir,
                            filename)
                    move_file(src_path, dest_path, configuration)
                else:
                    logger.error('Trigger job: %s, unknown state: %s'
                                 % (trigger_job['jobid'],
                                 serverjob['STATUS']))
    html += '</tbody>'
    html += '</table>'
    output_objects.append({'object_type': 'html_form', 'text': html})

    # Display active trigger jobs for this vgrid

    output_objects.append({'object_type': 'sectionheader',
                          'text': 'Trigger Log'})
    log_content = read_trigger_log(configuration, vgrid_name)
    output_objects.append({'object_type': 'html_form',
                          'text': '''
 <div class="form_container">
 <textarea id="logarea" rows=10 readonly="readonly">%s</textarea>
 </div>
 '''
                          % log_content})

    output_objects.append({'object_type': 'sectionheader',
                          'text': 'Manage Triggers'})

    # Always run as rule creator to avoid users being able to act on behalf
    # of ANY other user using triggers (=exploit)

    extra_fields = [
        ('path', None),
        ('match_dirs', ['False', 'True']),
        ('match_recursive', ['False', 'True']),
        ('changes', [keyword_all] + valid_trigger_changes),
        ('action', [keyword_auto] + valid_trigger_actions),
        ('arguments', None),
        ('run_as', client_id),
        ]

    # NOTE: we do NOT show saved template contents - see addvgridtriggers

    optional_fields = [('rate_limit', None), ('settle_time', None)]

    (status, oobjs) = vgrid_add_remove_table(
        client_id,
        vgrid_name,
        'trigger',
        'vgridtrigger',
        configuration,
        extra_fields,
        optional_fields,
        )
    output_objects.extend(oobjs)

    if not status:
        return (output_objects, returnvalues.SYSTEM_ERROR)

    return (output_objects, returnvalues.OK)
Example #11
0
def run_transfer(configuration, client_id, transfer_dict):
    """Actual data transfer built from transfer_dict on behalf of client_id"""

    logger.debug('run transfer for %s: %s' %
                 (client_id, blind_pw(transfer_dict)))
    transfer_id = transfer_dict['transfer_id']
    action = transfer_dict['action']
    protocol = transfer_dict['protocol']
    status_dir = get_status_dir(configuration, client_id, transfer_id)
    cmd_map = get_cmd_map()
    if not protocol in cmd_map[action]:
        raise ValueError('unsupported protocol: %s' % protocol)

    client_dir = client_id_dir(client_id)
    makedirs_rec(status_dir, configuration)

    # Please note that base_dir must end in slash to avoid access to other
    # user dirs when own name is a prefix of another user name

    base_dir = os.path.abspath(
        os.path.join(configuration.user_home, client_dir)) + os.sep
    # TODO: we should refactor to move command extraction into one function
    command_pattern = cmd_map[action][protocol]
    target_helper_list = []
    key_path = transfer_dict.get("key", "")
    if key_path:
        # Use key with given name from settings dir
        settings_base_dir = os.path.abspath(
            os.path.join(configuration.user_settings, client_dir)) + os.sep
        key_path = os.path.join(settings_base_dir, user_keys_dir,
                                key_path.lstrip(os.sep))
        # IMPORTANT: path must be expanded to abs for proper chrooting
        key_path = os.path.abspath(key_path)
        if not valid_user_path(configuration, key_path, settings_base_dir):
            logger.error('rejecting illegal directory traversal for %s (%s)' %
                         (key_path, blind_pw(transfer_dict)))
            raise ValueError("user provided a key outside own settings!")
    rel_src_list = transfer_dict['src']
    rel_dst = transfer_dict['dst']
    compress = transfer_dict.get("compress", False)
    exclude = transfer_dict.get("exclude", [])
    if transfer_dict['action'] in ('import', ):
        logger.debug('setting abs dst for action %(action)s' % transfer_dict)
        src_path_list = transfer_dict['src']
        dst_path = os.path.join(base_dir, rel_dst.lstrip(os.sep))
        dst_path = os.path.abspath(dst_path)
        for src in rel_src_list:
            abs_dst = os.path.join(dst_path, src.lstrip(os.sep))
            # IMPORTANT: path must be expanded to abs for proper chrooting
            abs_dst = os.path.abspath(abs_dst)
            # Reject illegal directory traversal and hidden files
            if not valid_user_path(configuration, abs_dst, base_dir, True):
                logger.error(
                    'rejecting illegal directory traversal for %s (%s)' %
                    (abs_dst, blind_pw(transfer_dict)))
                raise ValueError("user provided a destination outside home!")
            if src.endswith(os.sep):
                target_helper_list.append(
                    (get_lftp_target(True, False, exclude),
                     get_rsync_target(True, False, exclude, compress)))
            else:
                target_helper_list.append(
                    (get_lftp_target(True, True, exclude),
                     get_rsync_target(True, True, exclude, compress)))
        makedirs_rec(dst_path, configuration)
    elif transfer_dict['action'] in ('export', ):
        logger.debug('setting abs src for action %(action)s' % transfer_dict)
        dst_path = transfer_dict['dst']
        src_path_list = []
        for src in rel_src_list:
            src_path = os.path.join(base_dir, src.lstrip(os.sep))
            # IMPORTANT: path must be expanded to abs for proper chrooting
            src_path = os.path.abspath(src_path)
            # Reject illegal directory traversal and hidden files
            if not valid_user_path(configuration, src_path, base_dir, True):
                logger.error(
                    'rejecting illegal directory traversal for %s (%s)' %
                    (src, blind_pw(transfer_dict)))
                raise ValueError("user provided a source outside home!")
            src_path_list.append(src_path)
            if src.endswith(os.sep) or os.path.isdir(src):
                target_helper_list.append(
                    (get_lftp_target(False, False, exclude),
                     get_rsync_target(False, False, exclude, compress)))
            else:
                target_helper_list.append(
                    (get_lftp_target(False, True, exclude),
                     get_rsync_target(False, True, exclude, compress)))
    else:
        raise ValueError('unsupported action for %(transfer_id)s: %(action)s' %
                         transfer_dict)
    run_dict = transfer_dict.copy()
    run_dict['log_path'] = os.path.join(status_dir, 'transfer.log')
    # Use private known hosts file for ssh transfers as explained above
    # NOTE: known_hosts containing '=' silently leads to rest getting ignored!
    #       use /dev/null to skip host key verification completely for now.
    #run_dict['known_hosts'] = os.path.join(base_dir, '.ssh', 'known_hosts')
    run_dict['known_hosts'] = '/dev/null'
    # Make sure password is set to empty string as default
    run_dict['password'] = run_dict.get('password', '')
    # TODO: this is a bogus cert path for now - we don't support ssl certs
    run_dict['cert'] = run_dict.get('cert', '')
    # IMPORTANT: must be implicit proto or 'ftp://' (not ftps://) and similarly
    #            webdav(s) must use explicit http(s) instead. In both cases we
    #            replace protocol between cmd selection and lftp path expansion
    if run_dict['protocol'] == 'ftps':
        run_dict['orig_proto'] = run_dict['protocol']
        run_dict['protocol'] = 'ftp'
        logger.info(
            'force %(orig_proto)s to %(protocol)s for %(transfer_id)s' %
            run_dict)
    elif run_dict['protocol'].startswith('webdav'):
        run_dict['orig_proto'] = run_dict['protocol']
        run_dict['protocol'] = run_dict['protocol'].replace('webdav', 'http')
        logger.info(
            'force %(orig_proto)s to %(protocol)s for %(transfer_id)s' %
            run_dict)
    if key_path:
        rel_key = run_dict['key']
        rel_cert = run_dict['cert']
        run_dict['key'] = key_path
        run_dict['cert'] = key_path.replace(rel_key, rel_cert)
        run_dict['ssh_auth'] = get_ssh_auth(True, run_dict)
        run_dict['ssl_auth'] = get_ssl_auth(True, run_dict)
    else:
        # Extract encrypted password
        password_digest = run_dict.get('password_digest', '')
        if password_digest:
            _, _, _, payload = password_digest.split("$")
            unscrambled = unscramble_digest(configuration.site_digest_salt,
                                            payload)
            _, _, password = unscrambled.split(":")
            run_dict['password'] = password
        run_dict['ssh_auth'] = get_ssh_auth(False, run_dict)
        run_dict['ssl_auth'] = get_ssl_auth(False, run_dict)
    run_dict['rel_dst'] = rel_dst
    run_dict['dst'] = dst_path
    run_dict['lftp_buf_size'] = run_dict.get('lftp_buf_size',
                                             lftp_buffer_bytes)
    run_dict['lftp_sftp_block_size'] = run_dict.get('sftp_sftp_block_size',
                                                    lftp_sftp_block_bytes)
    status = 0
    for (src, rel_src, target_helper) in zip(src_path_list, rel_src_list,
                                             target_helper_list):
        (lftp_target, rsync_target) = target_helper
        logger.debug('setting up %(action)s for %(src)s' % run_dict)
        if run_dict['protocol'] == 'sftp' and not os.path.isabs(src):
            # NOTE: lftp interprets sftp://FQDN/SRC as absolute path /SRC
            #       We force relative paths into user home with a tilde.
            #       The resulting sftp://FQDN/~/SRC looks funky but works.
            run_dict['src'] = "~/%s" % src
        else:
            # All other paths are probably absolute or auto-chrooted anyway
            run_dict['src'] = src
        run_dict['rel_src'] = rel_src
        run_dict['lftp_args'] = ' '.join(lftp_target[0]) % run_dict
        run_dict['lftp_excludes'] = ' '.join(lftp_target[1])
        # src and dst may actually be reversed for lftp, but for symmetry ...
        run_dict['lftp_src'] = lftp_target[2][0] % run_dict
        run_dict['lftp_dst'] = lftp_target[2][1] % run_dict
        run_dict['rsync_args'] = ' '.join(rsync_target[0]) % run_dict
        # Preserve excludes on list form for rsync, where it matters
        run_dict[RSYNC_EXCLUDES_LIST] = rsync_target[1]
        run_dict['rsync_src'] = rsync_target[2][0] % run_dict
        run_dict['rsync_dst'] = rsync_target[2][1] % run_dict
        blind_dict = blind_pw(run_dict)
        logger.debug('expanded vars to %s' % blind_dict)
        # NOTE: Make sure NOT to break rsync excludes on list form as they
        # won't work if concatenated to a single string in command_list!
        command_list, blind_list = [], []
        for i in command_pattern:
            if i == RSYNC_EXCLUDES_LIST:
                command_list += run_dict[RSYNC_EXCLUDES_LIST]
                blind_list += run_dict[RSYNC_EXCLUDES_LIST]
            else:
                command_list.append(i % run_dict)
                blind_list.append(i % blind_dict)
        command_str = ' '.join(command_list)
        blind_str = ' '.join(blind_list)
        logger.info('run %s on behalf of %s' % (blind_str, client_id))
        transfer_proc = subprocess_popen(command_list,
                                         stdout=subprocess_pipe,
                                         stderr=subprocess_pipe)
        # Save transfer_proc.pid for use in clean up during shutdown
        # in that way we can resume pretty smoothly in next run.
        sub_pid = transfer_proc.pid
        logger.info('%s %s running transfer process %s' %
                    (client_id, transfer_id, sub_pid))
        add_sub_pid(configuration, sub_pid_map, client_id, transfer_id,
                    sub_pid)
        out, err = transfer_proc.communicate()
        exit_code = transfer_proc.wait()
        status |= exit_code
        del_sub_pid(configuration, sub_pid_map, client_id, transfer_id,
                    sub_pid)
        logger.info('done running transfer %s: %s' % (transfer_id, blind_str))
        logger.debug('raw output is: %s' % out)
        logger.debug('raw error is: %s' % err)
        logger.debug('result was %s' % exit_code)
        if not transfer_result(configuration, client_id, run_dict, exit_code,
                               out.replace(base_dir, ''),
                               err.replace(base_dir, '')):
            logger.error('writing transfer status for %s failed' % transfer_id)

    logger.debug('done handling transfers in %(transfer_id)s' % transfer_dict)
    transfer_dict['exit_code'] = status
    if status == 0:
        transfer_dict['status'] = 'DONE'
    else:
        transfer_dict['status'] = 'FAILED'
Example #12
0
def main(client_id, user_arguments_dict):
    """Main function used by front end"""

    (configuration, logger, output_objects, op_name) = \
        initialize_main_variables(client_id, op_header=False)
    client_dir = client_id_dir(client_id)
    defaults = signature()[1]

    (validate_status, accepted) = validate_input_and_cert(
        user_arguments_dict,
        defaults,
        output_objects,
        client_id,
        configuration,
        allow_rejects=False,
        )

    if not validate_status:
        WARNING_MSG = str(accepted)
        output_objects.append({'object_type': 'warning',
                              'text': WARNING_MSG})
        return (accepted, returnvalues.CLIENT_ERROR)

    action = ''.join(accepted['action'])
    flags = ''.join(accepted['flags'])
    path = ''.join(accepted['path'])
    extension = ''.join(accepted['extension'])

    # Please note that base_dir must end in slash to avoid access to other
    # user dirs when own name is a prefix of another user name

    base_dir = os.path.abspath(os.path.join(configuration.user_home,
                               client_dir)) + os.sep
    abs_path = os.path.join(base_dir, path)

    settings_dict = load_settings(client_id, configuration)
    javascript = None

    title_entry = find_entry(output_objects, 'title')
    title_entry['text'] = 'FILEMETAIO Management'
    title_entry['javascript'] = javascript
    output_objects.append({'object_type': 'header',
                          'text': 'FILEMETAIO Management'})
    status = returnvalues.ERROR

    if flags == 'i':
        if action == 'list':
            image_meta = get_image_file_settings(logger, abs_path)
            if image_meta is not None:
                extension_list = []
                settings_status_list = []
                settings_progress_list = []
                image_count_list = []
                for entry in image_meta:
                    extension_list.append(entry['extension'])
                    settings_status_list.append(entry['settings_status'
                            ])
                    settings_progress_list.append(entry['settings_update_progress'
                            ])
                    image_count_list.append(get_image_file_count(logger,
                            abs_path, entry['extension']))

                output_objects.append({
                    'object_type': 'image_settings_list',
                    'extension_list': extension_list,
                    'settings_status_list': settings_status_list,
                    'settings_progress_list': settings_progress_list,
                    'image_count_list': image_count_list,
                    })
                status = returnvalues.OK
            else:
                status = returnvalues.ERROR
                ERROR_MSG = "No image settings found for path: '%s'" \
                    % path
                output_objects.append({'object_type': 'text',
                        'text': ERROR_MSG})
                logger.error('filemetaio.py: %s -> %s' % (action,
                             ERROR_MSG))
        elif action == 'remove_dir':
            remove_ext = None
            vgrid_name = path.split('/')[0]

            if extension != '':
                remove_ext = extension
            try:
                (result, removed_ext_list) = \
                    remove_image_file_settings(logger, abs_path,
                        remove_ext)
            except Exception, ex:
                logger.debug(str(traceback.format_exc()))

            if result is not None:
                result = returnvalues.OK
            else:
                result = returnvalues.ERROR
                ERROR_MSG = \
                    'Unable to remove image settings for path: %s' \
                    % path
                output_objects.append({'object_type': 'text',
                        'text': ERROR_MSG})
                logger.error('filemetaio.py: %s -> %s' % (action,
                             ERROR_MSG))

            for removed_ext in removed_ext_list:
                abs_last_modified_filepath = \
                    __get_image_settings_trigger_last_modified_filepath(logger,
                        abs_path, removed_ext)

                # Remove trigger

                if delete_file(abs_last_modified_filepath, logger):

                    # FYSIKER HACK: Sleep 1 to prevent trigger rule/event race
                    # TODO: Modify events handler to accept trigger action + delete

                    time.sleep(1)

                    # Remove old vgrid submit trigger for files

                    rule_id = __get_image_file_trigger_rule_id(logger,
                            path, removed_ext)
                    status = __remove_image_file_trigger(
                        configuration,
                        vgrid_name,
                        path,
                        extension,
                        rule_id,
                        output_objects,
                        )
                    if status != returnvalues.OK:
                        result = status

                    # Remove old vgrid submit trigger for settings

                    rule_id = \
                        __get_image_settings_trigger_rule_id(logger,
                            path, removed_ext)
                    status = __remove_image_settings_trigger(
                        configuration,
                        vgrid_name,
                        path,
                        extension,
                        rule_id,
                        output_objects,
                        )
                    if status != returnvalues.OK:
                        result = status
                else:
                    result = returnvalues.ERROR
                    ERROR_MSG = 'Unable to remove file: %s ' \
                        % abs_last_modified_filepath
                    output_objects.append({'object_type': 'text',
                            'text': ERROR_MSG})
                    logger.error('filemetaio.py: %s -> %s' % (action,
                                 ERROR_MSG))
        elif action == 'get_dir':

            image_count = get_image_file_count(logger, abs_path,
                    extension)
            image_meta = get_image_file_setting(logger, abs_path,
                    extension)

            if image_meta is not None:
                extension = str(image_meta['extension'])
                settings_status = str(image_meta['settings_status'])
                settings_update_progress = \
                    str(image_meta['settings_update_progress'])
                settings_recursive = str(image_meta['settings_recursive'
                        ])
                image_count = str(image_count)
                image_type = str(image_meta['image_type'])
                offset = str(image_meta['offset'])
                x_dimension = str(image_meta['x_dimension'])
                y_dimension = str(image_meta['y_dimension'])
                preview_image_extension = \
                    str(image_meta['preview_image_extension'])
                preview_x_dimension = \
                    str(image_meta['preview_x_dimension'])
                preview_y_dimension = \
                    str(image_meta['preview_y_dimension'])
                preview_cutoff_min = str(image_meta['preview_cutoff_min'
                        ])
                preview_cutoff_max = str(image_meta['preview_cutoff_max'
                        ])
                data_type = str(image_meta['data_type'])

                output_objects.append({
                    'object_type': 'image_setting',
                    'path': path,
                    'extension': extension,
                    'settings_status': settings_status,
                    'settings_update_progress': settings_update_progress,
                    'settings_recursive': settings_recursive,
                    'image_count': image_count,
                    'image_type': image_type,
                    'offset': offset,
                    'x_dimension': x_dimension,
                    'y_dimension': y_dimension,
                    'preview_image_extension': preview_image_extension,
                    'preview_x_dimension': preview_x_dimension,
                    'preview_y_dimension': preview_y_dimension,
                    'preview_cutoff_min': preview_cutoff_min,
                    'preview_cutoff_max': preview_cutoff_max,
                    'data_type': data_type,
                    })
                status = returnvalues.OK
            else:
                status = returnvalues.ERROR
                ERROR_MSG = \
                    "No image setting information for path: '%s', extension: '%s'" \
                    % (path, extension)
                output_objects.append({'object_type': 'text',
                        'text': ERROR_MSG})
                logger.error('filemetaio.py: %s -> %s' % (action,
                             ERROR_MSG))
        elif action == 'put_dir':
            settings_status = ''.join(accepted['settings_status'])
            if ''.join(accepted['settings_recursive']) == 'True':
                settings_recursive = True
            else:
                settings_recursive = False
            image_type = ''.join(accepted['image_type'])
            data_type = ''.join(accepted['data_type'])
            offset = int(''.join(accepted['offset']))
            x_dimension = int(''.join(accepted['x_dimension']))
            y_dimension = int(''.join(accepted['y_dimension']))
            preview_image_extension = \
                ''.join(accepted['preview_image_extension'])
            preview_x_dimension = \
                int(''.join(accepted['preview_x_dimension']))
            preview_y_dimension = \
                int(''.join(accepted['preview_y_dimension']))
            preview_cutoff_min = \
                float(''.join(accepted['preview_cutoff_min']))
            preview_cutoff_max = \
                float(''.join(accepted['preview_cutoff_max']))

            path_array = path.split('/')
            vgrid_name = path_array[0]
            vgrid_data_path = '/'.join(path_array[1:])
            vgrid_meta_path = os.path.join(vgrid_data_path, __metapath)
            vgrid_image_meta_path = os.path.join(vgrid_data_path,
                    __image_metapath)

            OK_MSG = \
                "Created/updated settings for image extension: '%s' for path '%s'" \
                % (extension, path)
            ERROR_MSG = \
                "Failed to change settings for image extension: '%s' for path: '%s'" \
                % (extension, path)

            (is_valid, is_valid_msg) = __is_valid_image_settings_update(
                configuration,
                base_dir,
                vgrid_name,
                vgrid_data_path,
                extension,
                settings_recursive,
                )

            if is_valid:
                status = returnvalues.OK
            else:
                status = returnvalues.ERROR
                output_objects.append({'object_type': 'error_text',
                        'text': ERROR_MSG})
                output_objects.append({'object_type': 'error_text',
                        'text': is_valid_msg})
                logger.error('filemetaio.py: %s -> %s' % (action,
                             ERROR_MSG))
                logger.error('filemetaio.py: %s -> %s' % (action,
                             is_valid_msg))

            # Ensure meta path existence

            if status == returnvalues.OK:
                makedirs_rec(os.path.join(base_dir,
                             os.path.join(vgrid_name,
                             vgrid_meta_path)), configuration)

                # Ensure image meta path existence

                makedirs_rec(os.path.join(base_dir,
                             os.path.join(vgrid_name,
                             vgrid_image_meta_path)), configuration)

                try:
                    add_status = add_image_file_setting(
                        logger,
                        abs_path,
                        extension,
                        settings_status,
                        None,
                        settings_recursive,
                        image_type,
                        data_type,
                        offset,
                        x_dimension,
                        y_dimension,
                        preview_image_extension,
                        preview_x_dimension,
                        preview_y_dimension,
                        preview_cutoff_min,
                        preview_cutoff_max,
                        overwrite=True,
                        )
                except Exception, ex:
                    add_status = False
                    logger.debug(str(traceback.format_exc()))

                if add_status:
                    status = returnvalues.OK
                    output_objects.append({'object_type': 'text',
                            'text': OK_MSG})
                else:
                    status = returnvalues.ERROR
                    output_objects.append({'object_type': 'error_text',
                            'text': ERROR_MSG})
                    logger.error('filemetaio.py: %s -> %s' % (action,
                                 ERROR_MSG))

            if status == returnvalues.OK:

                # Generate vgrid trigger for files

                if settings_recursive:
                    vgrid_trigger_path = os.path.join(vgrid_data_path,
                            '*/*.%s' % extension)
                else:
                    vgrid_trigger_path = os.path.join(vgrid_data_path,
                            '*.%s' % extension)

                rule_id = __get_image_file_trigger_rule_id(logger,
                        path, extension)
                rule_dict = {
                    'rule_id': rule_id,
                    'vgrid_name': vgrid_name,
                    'path': vgrid_trigger_path,
                    'changes': ['created', 'modified', 'deleted',
                                'moved'],
                    'run_as': client_id,
                    'action': 'submit',
                    'arguments': 'template_from_filemetaio.py',
                    'templates': [__get_image_update_preview_mrsl_template(path)],
                    'settle_time': '60s',
                    'rate_limit': '',
                    }

                # Remove old vgrid submit trigger for files

                status = __remove_image_file_trigger(
                    configuration,
                    vgrid_name,
                    path,
                    extension,
                    rule_id,
                    output_objects,
                    )

            if status == returnvalues.OK:

                # Add generated vgrid submit trigger for files

                (add_status, add_msg) = \
                    vgrid_add_triggers(configuration, vgrid_name,
                        [rule_dict])
                if add_status:
                    status = returnvalues.OK
                    OK_MSG = \
                        "Created/updated image file trigger for extension: '%s', path '%s'" \
                        % (extension, path)
                    output_objects.append({'object_type': 'text',
                            'text': OK_MSG})
                else:
                    status = returnvalues.ERROR
                    ERROR_MSG = \
                        "Failed change image file trigger for extension: '%s', path '%s'" \
                        % (extension, path)
                    ERROR_MSG2 = "Makes sure '%s' is a VGrid" \
                        % vgrid_name
                    output_objects.append({'object_type': 'error_text',
                            'text': ERROR_MSG})
                    output_objects.append({'object_type': 'error_text',
                            'text': ERROR_MSG2})
                    logger.error('filemetaio.py: %s -> %s' % (action,
                                 ERROR_MSG))
                    logger.error('filemetaio.py: %s -> %s' % (action,
                                 ERROR_MSG2))

            if status == returnvalues.OK:

                # Generate vgrid trigger for settings

                vgrid_path = '/'.join(path.split('/')[1:])
                vgrid_trigger_filepath = \
                    __get_image_settings_trigger_last_modified_filepath(logger,
                        vgrid_path, extension)

                rule_id = __get_image_settings_trigger_rule_id(logger,
                        path, extension)
                rule_dict = {
                    'rule_id': rule_id,
                    'vgrid_name': vgrid_name,
                    'path': vgrid_trigger_filepath,
                    'changes': ['modified', 'deleted'],
                    'run_as': client_id,
                    'action': 'submit',
                    'arguments': 'template_from_filemetaio.py',
                    'templates': [__get_image_create_previews_mrsl_template(path,
                                  extension)],
                    'settle_time': '1s',
                    'rate_limit': '',
                    }

                # Remove old vgrid submit trigger for settings

                status = __remove_image_settings_trigger(
                    configuration,
                    vgrid_name,
                    path,
                    extension,
                    rule_id,
                    output_objects,
                    )

            if status == returnvalues.OK:

                # Add generated vgrid submit trigger for settings

                (add_status, add_msg) = \
                    vgrid_add_triggers(configuration, vgrid_name,
                        [rule_dict])
                if add_status:
                    status = returnvalues.OK
                    OK_MSG = \
                        "Created/updated old image setting trigger for extension: '%s', path '%s'" \
                        % (extension, path)
                    output_objects.append({'object_type': 'text',
                            'text': OK_MSG})
                else:
                    status = returnvalues.ERROR
                    ERROR_MSG = \
                        "Failed change old image setting trigger for extension: '%s', path '%s'" \
                        % (extension, path)
                    ERROR_MSG2 = "Makes sure '%s' is a VGrid" \
                        % vgrid_name
                    output_objects.append({'object_type': 'error_text',
                            'text': ERROR_MSG})
                    output_objects.append({'object_type': 'error_text',
                            'text': ERROR_MSG2})
                    logger.error('filemetaio.py: %s -> %s' % (action,
                                 ERROR_MSG))
                    logger.error('filemetaio.py: %s -> %s' % (action,
                                 ERROR_MSG2))

            if status == returnvalues.OK:

                # Trigger Trigger (Trigger Happty)

                abs_vgrid_trigger_filepath = os.path.join(base_dir,
                        os.path.join(vgrid_name,
                        vgrid_trigger_filepath))

                # FYSIKER HACK: Sleep 1 to prevent trigger rule/event race
                # TODO: Modify events handler to accept add+trigger action

                time.sleep(1)
                timestamp = time.time()
                touch(abs_vgrid_trigger_filepath, timestamp)