Example #1
0
    def do_fileprocess_validation(self, store, context_gus, filepath ):

        plugin_type = u'fileprocess'

        profile_iface = PluginProfiles(store)
        profile_associated = profile_iface.get_profiles_by_contexts([ context_gus ] )

        plugin_found = False
        validate_file = False

        for p_cfg in profile_associated:

            # Plugin FileProcess
            if p_cfg['plugin_type'] != plugin_type:
                continue

            plugin_found = True
            print "processing", filepath, "using the profile", p_cfg['profile_gus'], "configured for", p_cfg['plugin_name']

            plugin = PluginManager.instance_plugin(p_cfg['plugin_name'])
            validate_file = plugin.do_fileprocess(filepath, p_cfg['admin_settings'])

        if not plugin_found:
            # FileProcess profile has been not configured, file accepted by default
            validate_file = True

        return validate_file
Example #2
0
    def new(self, profile_dict):
        """
        @profile_dict need to contain the keys: 'plugin_type', 'plugin_name',
            'admin_settings',
        """

        store = self.getStore()
        newone = PluginProfiles()

        try:

            # below, before _import_dict, are checked profile_type, and plugin_name
            # both of them can't be updated, are chosen at creation time,

            if not unicode(profile_dict['plugin_type']) in valid_plugin_types:
                raise InvalidInputFormat("plugin_type not recognized")

            if not PluginManager.plugin_exists(profile_dict['plugin_name']):
                raise InvalidInputFormat("plugin_name not recognized between available plugins")

            newone._import_dict(profile_dict)

            # the name can be updated, but need to be checked to be UNIQUE
            if store.find(PluginProfiles, PluginProfiles.profile_name == newone.profile_name).count() >= 1:
                raise ProfileNameConflict

            plugin_info = PluginManager.get_plugin(profile_dict['plugin_name'])

            # initialize the three plugin_* fields, inherit by Plugin code
            newone.plugin_name = unicode(plugin_info['plugin_name'])
            newone.plugin_type = unicode(plugin_info['plugin_type'])
            newone.plugin_description = unicode(plugin_info['plugin_description'])
            newone.admin_fields = dict(plugin_info['admin_fields'])

            # Admin-only plugins are listed here, and they have not receiver_fields
            if newone.plugin_type in [ u'fileprocess' ]:
                newone.receiver_fields = []
            else:
                newone.receiver_fields = plugin_info['receiver_fields']

        except KeyError, e:
            raise InvalidInputFormat("Profile creation failed (missing %s)" % e)
Example #3
0
    def dump_models(self, expected):

        expected_dict = { 'itip' : InternalTip,
                          'wtip' : WhistleblowerTip,
                          'rtip' : ReceiverTip,
                          'receivers' : Receiver,
                          'comment' : Comment,
                          'profiles' : PluginProfiles,
                          'rcfg' : ReceiverConfs,
                          'file' : File,
                          'submission' : Submission,
                          'contexts' : Context }

        outputDict = {}
        self.returnCode(200)

        store = self.getStore()

        if expected in ['count', 'all']:

            for key, object in expected_dict.iteritems():
                info_list = object(store).get_all()

                if expected == 'all':
                    outputDict.update({key : info_list})

                outputDict.update({("%s_elements" % key) : len(info_list) })

            self.returnData(outputDict)
            return self.prepareRetVals()

        # XXX plugins is not dumped with all or count!
        if expected == 'plugins':

            info_list = PluginManager.get_all()
            outputDict.update({expected : info_list, ("%s_elements" % expected) : len(info_list) })

            self.returnData(outputDict)
            return self.prepareRetVals()

        if expected_dict.has_key(expected):

            info_list = expected_dict[expected](store).get_all()
            outputDict.update({expected : info_list, ("%s_elements" % expected) : len(info_list) })

            self.returnData(outputDict)
            return self.prepareRetVals()

        raise InvalidInputFormat("Not acceptable '%s'" % expected)
Example #4
0
    def get(self, *uriargs):
        """
        Parameters: None
        Response: adminPluginList
        Errors: None

        This handler is one of the few that do not operate versus the database model, but in
        the filesystem. Checks the plugin presents in the appropriate directory and return
        a list with name and properties.
        """

        plugin_descriptive_list = yield PluginManager.get_all()
        # TODO output validation - adminPluginList

        self.set_status(200)
        self.write(json.dumps(plugin_descriptive_list))
        self.finish()
Example #5
0
    def comment_notification(self):

        plugin_type = u'notification'
        store = self.getStore()

        comment_iface = Comment(store)
        internaltip_iface = InternalTip(store)
        receivercfg_iface = ReceiverConfs(store)
        profile_iface = PluginProfiles(store)

        not_notified_comments = comment_iface.get_comment_by_mark(marker=u'not notified')

        for comment in not_notified_comments:

            receivers_list = internaltip_iface.get_receivers_by_itip(comment['internaltip_id'])

            # needed to obtain context!
            itip_info = internaltip_iface.get_single(comment['internaltip_id'])

            for receiver_info in receivers_list:

                receiver_conf = receivercfg_iface.get_active_conf(receiver_info['receiver_gus'],
                    itip_info['context_gus'], plugin_type)

                if receiver_conf is None:
                    # TODO applicative log, database tracking of queue
                    continue

                # Ok, we had a valid an appropriate receiver configuration for the notification task
                related_profile = profile_iface.get_single(receiver_conf['profile_gus'])

                settings_dict = { 'admin_settings' : related_profile['admin_settings'],
                                  'receiver_settings' : receiver_conf['receiver_settings']}

                plugin = PluginManager.instance_plugin(related_profile['plugin_name'])

                return_code = plugin.do_notify(settings_dict, u'comment', comment)

                if return_code:
                    print "Notification of comment successful for user", receiver_conf['receiver_gus']
                else:
                    print "Notification of comment failed for user", receiver_conf['receiver_gus']

            # remind: comment are not guarantee until Task manager is not developed
            comment_iface.flip_mark(comment['comment_id'], u'notified')
Example #6
0
    def tip_notification(self):

        plugin_type = u'notification'
        store = self.getStore()

        receivertip_iface = ReceiverTip(store)
        receivercfg_iface = ReceiverConfs(store)
        profile_iface = PluginProfiles(store)

        not_notified_tips = receivertip_iface.get_tips_by_notification_mark(u'not notified')

        for single_tip in not_notified_tips:

        # from a single tip, we need to extract the receiver, and then, having
        # context + receiver, find out which configuration setting has active

            receivers_map = receivertip_iface.get_receivers_by_tip(single_tip['tip_gus'])

            receiver_info = receivers_map['actor']

            receiver_conf = receivercfg_iface.get_active_conf(receiver_info['receiver_gus'],
               single_tip['context_gus'], plugin_type)

            if receiver_conf is None:
               print "Receiver", receiver_info['receiver_gus'],\
               "has not an active notification settings in context", single_tip['context_gus'], "for", plugin_type
                # TODO separate key in answer
               continue

            # Ok, we had a valid an appropriate receiver configuration for the notification task
            related_profile = profile_iface.get_single(receiver_conf['profile_gus'])

            settings_dict = { 'admin_settings' : related_profile['admin_settings'],
                             'receiver_settings' : receiver_conf['receiver_settings']}

            plugin = PluginManager.instance_plugin(related_profile['plugin_name'])

            updated_tip = receivertip_iface.update_notification_date(single_tip['tip_gus'])
            return_code = plugin.do_notify(settings_dict, u'tip', updated_tip)

            if return_code:
               receivertip_iface.flip_mark(single_tip['tip_gus'], u'notified')
            else:
               receivertip_iface.flip_mark(single_tip['tip_gus'], u'unable to notify')
    def operation(self):
        """
        Goal of this function is to check all the new files and validate
        thru the configured SystemSettings

        possible marker in the file are:
            'not processed', 'ready', 'blocked', 'stored'
        defined in File._marker

        """
        plugin_type = u'fileprocess'

        file_iface = File()
        profile_iface = PluginProfiles()

        not_processed_file = yield file_iface.get_file_by_marker(file_iface._marker[0])

        print "FileProcess", not_processed_file

        for single_file in not_processed_file:

            profile_associated = yield profile_iface.get_profiles_by_contexts([ single_file['context_gus'] ] )

            for p_cfg in profile_associated:

                if p_cfg['plugin_type'] != plugin_type:
                    continue

                print "processing", single_file['file_name'], "using the profile", p_cfg['profile_gus'], "configured for", p_cfg['plugin_name']
                plugin = PluginManager.instance_plugin(p_cfg['plugin_name'])

                try:
                    tempfpath = os.path.join(config.advanced.submissions_dir, single_file['file_gus'])
                except AttributeError:
                    # XXX hi level danger Log - no directory present to perform file analysis
                    continue

                return_code = plugin.do_fileprocess(tempfpath, p_cfg['admin_settings'])

                # Todo Log/stats in both cases
                if return_code:
                    yield file_iface.flip_mark(single_file['file_gus'], file_iface._marker[1]) # ready
                else:
                    yield file_iface.flip_mark(single_file['file_gus'], file_iface._marker[2]) # blocked
Example #8
0
    def fileprocess(self):

        plugin_type = u'fileprocess'

        store = self.getStore()

        file_iface = File(store)
        profile_iface = PluginProfiles(store)

        not_processed_file = file_iface.get_file_by_marker(file_iface._marker[0])

        for single_file in not_processed_file:

            profile_associated = profile_iface.get_profiles_by_contexts([ single_file['context_gus'] ] )

            for p_cfg in profile_associated:

                if p_cfg['plugin_type'] != plugin_type:
                    continue

                print "processing", single_file['file_name'], "using the profile", p_cfg['profile_gus'], "configured for", p_cfg['plugin_name']
                plugin = PluginManager.instance_plugin(p_cfg['plugin_name'])

                try:
                    tempfpath = os.path.join(config.advanced.submissions_dir, single_file['file_gus'])
                except AttributeError:
                    # XXX hi level danger Log - no directory present to perform file analysis
                    continue

                return_code = plugin.do_fileprocess(tempfpath, p_cfg['admin_settings'])

                # Todo Log/stats in both cases
                if return_code:
                    file_iface.flip_mark(single_file['file_gus'], file_iface._marker[1]) # ready
                else:
                    file_iface.flip_mark(single_file['file_gus'], file_iface._marker[2]) # blocked
Example #9
0
    def get(self, what, *uriargs):
        """
        Parameters: None
        Response: Unknown
        Errors: None

        /dump/overview GET should return up to all the tables of GLBackend
        """

        expected = [ 'itip', 'wtip', 'rtip', 'receivers', 'comment',
                     'profiles', 'rcfg', 'file', 'submission', 'contexts', 'plugins', 'all', 'count' ]

        outputDict = {}

        if what == 'receivers' or what == 'all' or what == 'count':
            receiver_iface = Receiver()
            receiver_list = yield receiver_iface.get_all()

            if what != 'count':
                outputDict.update({ 'receivers_elements' : len(receiver_list), 'receivers' : receiver_list})
            else:
                outputDict.update({ 'receivers_elements' : len(receiver_list)})

        if what == 'itip' or what == 'all' or what == 'count':
            itip_iface = InternalTip()
            itip_list = yield itip_iface.get_all()

            if what != 'count':
                outputDict.update({ 'internaltips_elements' : len(itip_list), 'internaltips' : itip_list })
            else:
                outputDict.update({ 'internaltips_elements' : len(itip_list)})

        if what == 'rtip' or what == 'all' or what == 'count':
            rtip_iface = ReceiverTip()
            rtip_list = yield rtip_iface.get_all()

            if what != 'count':
                outputDict.update({ 'rtip_elements' : len(rtip_list), 'receivers_tips' : rtip_list })
            else:
                outputDict.update({ 'rtip_elements' : len(rtip_list)})

        if what == 'wtip' or what == 'all' or what == 'count':
            wtip_iface = WhistleblowerTip()
            wtip_list = yield wtip_iface.get_all()

            if what != 'count':
                outputDict.update({ 'wtip_elements' : len(wtip_list), 'whistleblower_tips' : wtip_list })
            else:
                outputDict.update({ 'wtip_elements' : len(wtip_list)})

        if what == 'comment' or what == 'all' or what == 'count':
            comment_iface = Comment()
            comment_list = yield comment_iface.get_all()

            if what != 'count':
                outputDict.update({ 'comment_elements' : len(comment_list), 'comments' : comment_list })
            else:
                outputDict.update({ 'comment_elements' : len(comment_list)})

        if what == 'profiles' or what == 'all' or what == 'count':
            profile_iface = PluginProfiles()
            profile_list = yield profile_iface.get_all()

            if what != 'count':
                outputDict.update({ 'profiles_elements' : len(profile_list), 'profiles' : profile_list })
            else:
                outputDict.update({ 'profiles_elements' : len(profile_list)})

        if what == 'plugins' or what == 'all' or what == 'count':
            plugin_list = yield PluginManager.get_all()

            if what != 'count':
                outputDict.update({ 'plugins_elements' : len(plugin_list), 'plugins' : plugin_list })
            else:
                outputDict.update({ 'plugins_elements' : len(plugin_list) })

        if what == 'rcfg' or what == 'all' or what == 'count':
            rconf_iface = ReceiverConfs()
            rconf_list = yield rconf_iface.get_all()

            if what != 'count':
                outputDict.update({ 'rcfg_elements' : len(rconf_list), 'settings' : rconf_list })
            else:
                outputDict.update({ 'rcfg_elements' : len(rconf_list)})

        if what == 'submission' or what == 'all' or what == 'count':
            submission_iface = Submission()
            submission_list = yield submission_iface.get_all()

            if what != 'count':
                outputDict.update({ 'submission_elements' : len(submission_list), 'submissions' : submission_list })
            else:
                outputDict.update({ 'submission_elements' : len(submission_list)})

        if what == 'file' or what == 'all' or what == 'count':
            file_iface = File()
            file_list = yield file_iface.get_all()

            if what != 'count':
                outputDict.update({ 'file_elements' : len(file_list), 'files' : file_list })
            else:
                outputDict.update({ 'file_elements' : len(file_list)})

        if what == 'contexts' or what == 'all' or what == 'count':
            context_iface = Context()
            context_list = yield context_iface.get_all()

            if what != 'count':
                outputDict.update({ 'contexts_elements' : len(context_list), 'contexts' : context_list })
            else:
                outputDict.update({ 'contexts_elements' : len(context_list)})


        if not what in expected:
            self.set_status(405)
        else:
            self.set_status(200)
            self.write(outputDict)

        self.finish()
    def operation(self):
        """
        Goal of this function is to check all the:
            Tips
            Comment
            Folder
            System Event

        marked as 'not notified' and perform notification.
        Notification plugin chose if perform a communication or not,
        Then became marked as:
            'notification ignored', or
            'notified'

        Every notification plugin NEED have a checks to verify
        if notification has been correctly performed. If not (eg: wrong
        login/password, network errors) would be marked as:
        'unable to be notified', and a retry logic is in TODO
        """
        plugin_type = u'notification'

        log.debug("[D]", self.__class__, 'operation', datetime.today().ctime())

        receivertip_iface = ReceiverTip()
        receivercfg_iface = ReceiverConfs()
        profile_iface = PluginProfiles()

        # TODO digest check missing it's required refactor the scheduler using a dedicated Storm table
        not_notified_tips = yield receivertip_iface.get_tips_by_notification_mark(u'not notified')

        for single_tip in not_notified_tips:

            # from a single tip, we need to extract the receiver, and then, having
            # context + receiver, find out which configuration setting has active

            receivers_map = yield receivertip_iface.get_receivers_by_tip(single_tip['tip_gus'])

            receiver_info = receivers_map['actor']

            receiver_conf = yield receivercfg_iface.get_active_conf(receiver_info['receiver_gus'],
                single_tip['context_gus'], plugin_type)

            if receiver_conf is None:
                print "Receiver", receiver_info['receiver_gus'], \
                    "has not an active notification settings in context", single_tip['context_gus'], "for", plugin_type

                # TODO applicative log, database tracking of queue
                continue

            # Ok, we had a valid an appropriate receiver configuration for the notification task
            related_profile = yield profile_iface.get_single(receiver_conf['profile_gus'])

            settings_dict = { 'admin_settings' : related_profile['admin_settings'],
                              'receiver_settings' : receiver_conf['receiver_settings']}

            plugin = PluginManager.instance_plugin(related_profile['plugin_name'])

            updated_tip = yield receivertip_iface.update_notification_date(single_tip['tip_gus'])
            return_code = plugin.do_notify(settings_dict, u'tip', updated_tip)

            if return_code:
                yield receivertip_iface.flip_mark(single_tip['tip_gus'], u'notified')
            else:
                yield receivertip_iface.flip_mark(single_tip['tip_gus'], u'unable to notify')


        # Comment Notification here it's just an incomplete version, that never would supports
        # digest or retry, until Task manager queue is implemented

        comment_iface = Comment()
        internaltip_iface = InternalTip()

        not_notified_comments = yield comment_iface.get_comment_by_mark(marker=u'not notified')

        for comment in not_notified_comments:

            receivers_list = yield internaltip_iface.get_receivers_by_itip(comment['internaltip_id'])

            # needed to obtain context!
            itip_info = yield internaltip_iface.get_single(comment['internaltip_id'])

            for receiver_info in receivers_list:

                receiver_conf = yield receivercfg_iface.get_active_conf(receiver_info['receiver_gus'],
                    itip_info['context_gus'], plugin_type)

                if receiver_conf is None:
                    # TODO applicative log, database tracking of queue
                    continue

                # Ok, we had a valid an appropriate receiver configuration for the notification task
                related_profile = yield profile_iface.get_single(receiver_conf['profile_gus'])

                settings_dict = { 'admin_settings' : related_profile['admin_settings'],
                                  'receiver_settings' : receiver_conf['receiver_settings']}

                plugin = PluginManager.instance_plugin(related_profile['plugin_name'])

                return_code = plugin.do_notify(settings_dict, u'comment', comment)

                if return_code:
                    print "Notification of comment successful for user", receiver_conf['receiver_gus']
                else:
                    print "Notification of comment failed for user", receiver_conf['receiver_gus']

            # remind: comment are not guarantee until Task manager is not developed
            yield comment_iface.flip_mark(comment['comment_id'], u'notified')