def do_fileprocess_validation(self, store, context_gus, filepath ): plugin_type = u'fileprocess' profile_iface = PluginProfiles(store) profile_associated = profile_iface.get_profiles_by_contexts([ context_gus ] ) plugin_found = False validate_file = False for p_cfg in profile_associated: # Plugin FileProcess if p_cfg['plugin_type'] != plugin_type: continue plugin_found = True print "processing", filepath, "using the profile", p_cfg['profile_gus'], "configured for", p_cfg['plugin_name'] plugin = PluginManager.instance_plugin(p_cfg['plugin_name']) validate_file = plugin.do_fileprocess(filepath, p_cfg['admin_settings']) if not plugin_found: # FileProcess profile has been not configured, file accepted by default validate_file = True return validate_file
def new(self, profile_dict): """ @profile_dict need to contain the keys: 'plugin_type', 'plugin_name', 'admin_settings', """ store = self.getStore() newone = PluginProfiles() try: # below, before _import_dict, are checked profile_type, and plugin_name # both of them can't be updated, are chosen at creation time, if not unicode(profile_dict['plugin_type']) in valid_plugin_types: raise InvalidInputFormat("plugin_type not recognized") if not PluginManager.plugin_exists(profile_dict['plugin_name']): raise InvalidInputFormat("plugin_name not recognized between available plugins") newone._import_dict(profile_dict) # the name can be updated, but need to be checked to be UNIQUE if store.find(PluginProfiles, PluginProfiles.profile_name == newone.profile_name).count() >= 1: raise ProfileNameConflict plugin_info = PluginManager.get_plugin(profile_dict['plugin_name']) # initialize the three plugin_* fields, inherit by Plugin code newone.plugin_name = unicode(plugin_info['plugin_name']) newone.plugin_type = unicode(plugin_info['plugin_type']) newone.plugin_description = unicode(plugin_info['plugin_description']) newone.admin_fields = dict(plugin_info['admin_fields']) # Admin-only plugins are listed here, and they have not receiver_fields if newone.plugin_type in [ u'fileprocess' ]: newone.receiver_fields = [] else: newone.receiver_fields = plugin_info['receiver_fields'] except KeyError, e: raise InvalidInputFormat("Profile creation failed (missing %s)" % e)
def dump_models(self, expected): expected_dict = { 'itip' : InternalTip, 'wtip' : WhistleblowerTip, 'rtip' : ReceiverTip, 'receivers' : Receiver, 'comment' : Comment, 'profiles' : PluginProfiles, 'rcfg' : ReceiverConfs, 'file' : File, 'submission' : Submission, 'contexts' : Context } outputDict = {} self.returnCode(200) store = self.getStore() if expected in ['count', 'all']: for key, object in expected_dict.iteritems(): info_list = object(store).get_all() if expected == 'all': outputDict.update({key : info_list}) outputDict.update({("%s_elements" % key) : len(info_list) }) self.returnData(outputDict) return self.prepareRetVals() # XXX plugins is not dumped with all or count! if expected == 'plugins': info_list = PluginManager.get_all() outputDict.update({expected : info_list, ("%s_elements" % expected) : len(info_list) }) self.returnData(outputDict) return self.prepareRetVals() if expected_dict.has_key(expected): info_list = expected_dict[expected](store).get_all() outputDict.update({expected : info_list, ("%s_elements" % expected) : len(info_list) }) self.returnData(outputDict) return self.prepareRetVals() raise InvalidInputFormat("Not acceptable '%s'" % expected)
def get(self, *uriargs): """ Parameters: None Response: adminPluginList Errors: None This handler is one of the few that do not operate versus the database model, but in the filesystem. Checks the plugin presents in the appropriate directory and return a list with name and properties. """ plugin_descriptive_list = yield PluginManager.get_all() # TODO output validation - adminPluginList self.set_status(200) self.write(json.dumps(plugin_descriptive_list)) self.finish()
def comment_notification(self): plugin_type = u'notification' store = self.getStore() comment_iface = Comment(store) internaltip_iface = InternalTip(store) receivercfg_iface = ReceiverConfs(store) profile_iface = PluginProfiles(store) not_notified_comments = comment_iface.get_comment_by_mark(marker=u'not notified') for comment in not_notified_comments: receivers_list = internaltip_iface.get_receivers_by_itip(comment['internaltip_id']) # needed to obtain context! itip_info = internaltip_iface.get_single(comment['internaltip_id']) for receiver_info in receivers_list: receiver_conf = receivercfg_iface.get_active_conf(receiver_info['receiver_gus'], itip_info['context_gus'], plugin_type) if receiver_conf is None: # TODO applicative log, database tracking of queue continue # Ok, we had a valid an appropriate receiver configuration for the notification task related_profile = profile_iface.get_single(receiver_conf['profile_gus']) settings_dict = { 'admin_settings' : related_profile['admin_settings'], 'receiver_settings' : receiver_conf['receiver_settings']} plugin = PluginManager.instance_plugin(related_profile['plugin_name']) return_code = plugin.do_notify(settings_dict, u'comment', comment) if return_code: print "Notification of comment successful for user", receiver_conf['receiver_gus'] else: print "Notification of comment failed for user", receiver_conf['receiver_gus'] # remind: comment are not guarantee until Task manager is not developed comment_iface.flip_mark(comment['comment_id'], u'notified')
def tip_notification(self): plugin_type = u'notification' store = self.getStore() receivertip_iface = ReceiverTip(store) receivercfg_iface = ReceiverConfs(store) profile_iface = PluginProfiles(store) not_notified_tips = receivertip_iface.get_tips_by_notification_mark(u'not notified') for single_tip in not_notified_tips: # from a single tip, we need to extract the receiver, and then, having # context + receiver, find out which configuration setting has active receivers_map = receivertip_iface.get_receivers_by_tip(single_tip['tip_gus']) receiver_info = receivers_map['actor'] receiver_conf = receivercfg_iface.get_active_conf(receiver_info['receiver_gus'], single_tip['context_gus'], plugin_type) if receiver_conf is None: print "Receiver", receiver_info['receiver_gus'],\ "has not an active notification settings in context", single_tip['context_gus'], "for", plugin_type # TODO separate key in answer continue # Ok, we had a valid an appropriate receiver configuration for the notification task related_profile = profile_iface.get_single(receiver_conf['profile_gus']) settings_dict = { 'admin_settings' : related_profile['admin_settings'], 'receiver_settings' : receiver_conf['receiver_settings']} plugin = PluginManager.instance_plugin(related_profile['plugin_name']) updated_tip = receivertip_iface.update_notification_date(single_tip['tip_gus']) return_code = plugin.do_notify(settings_dict, u'tip', updated_tip) if return_code: receivertip_iface.flip_mark(single_tip['tip_gus'], u'notified') else: receivertip_iface.flip_mark(single_tip['tip_gus'], u'unable to notify')
def operation(self): """ Goal of this function is to check all the new files and validate thru the configured SystemSettings possible marker in the file are: 'not processed', 'ready', 'blocked', 'stored' defined in File._marker """ plugin_type = u'fileprocess' file_iface = File() profile_iface = PluginProfiles() not_processed_file = yield file_iface.get_file_by_marker(file_iface._marker[0]) print "FileProcess", not_processed_file for single_file in not_processed_file: profile_associated = yield profile_iface.get_profiles_by_contexts([ single_file['context_gus'] ] ) for p_cfg in profile_associated: if p_cfg['plugin_type'] != plugin_type: continue print "processing", single_file['file_name'], "using the profile", p_cfg['profile_gus'], "configured for", p_cfg['plugin_name'] plugin = PluginManager.instance_plugin(p_cfg['plugin_name']) try: tempfpath = os.path.join(config.advanced.submissions_dir, single_file['file_gus']) except AttributeError: # XXX hi level danger Log - no directory present to perform file analysis continue return_code = plugin.do_fileprocess(tempfpath, p_cfg['admin_settings']) # Todo Log/stats in both cases if return_code: yield file_iface.flip_mark(single_file['file_gus'], file_iface._marker[1]) # ready else: yield file_iface.flip_mark(single_file['file_gus'], file_iface._marker[2]) # blocked
def fileprocess(self): plugin_type = u'fileprocess' store = self.getStore() file_iface = File(store) profile_iface = PluginProfiles(store) not_processed_file = file_iface.get_file_by_marker(file_iface._marker[0]) for single_file in not_processed_file: profile_associated = profile_iface.get_profiles_by_contexts([ single_file['context_gus'] ] ) for p_cfg in profile_associated: if p_cfg['plugin_type'] != plugin_type: continue print "processing", single_file['file_name'], "using the profile", p_cfg['profile_gus'], "configured for", p_cfg['plugin_name'] plugin = PluginManager.instance_plugin(p_cfg['plugin_name']) try: tempfpath = os.path.join(config.advanced.submissions_dir, single_file['file_gus']) except AttributeError: # XXX hi level danger Log - no directory present to perform file analysis continue return_code = plugin.do_fileprocess(tempfpath, p_cfg['admin_settings']) # Todo Log/stats in both cases if return_code: file_iface.flip_mark(single_file['file_gus'], file_iface._marker[1]) # ready else: file_iface.flip_mark(single_file['file_gus'], file_iface._marker[2]) # blocked
def get(self, what, *uriargs): """ Parameters: None Response: Unknown Errors: None /dump/overview GET should return up to all the tables of GLBackend """ expected = [ 'itip', 'wtip', 'rtip', 'receivers', 'comment', 'profiles', 'rcfg', 'file', 'submission', 'contexts', 'plugins', 'all', 'count' ] outputDict = {} if what == 'receivers' or what == 'all' or what == 'count': receiver_iface = Receiver() receiver_list = yield receiver_iface.get_all() if what != 'count': outputDict.update({ 'receivers_elements' : len(receiver_list), 'receivers' : receiver_list}) else: outputDict.update({ 'receivers_elements' : len(receiver_list)}) if what == 'itip' or what == 'all' or what == 'count': itip_iface = InternalTip() itip_list = yield itip_iface.get_all() if what != 'count': outputDict.update({ 'internaltips_elements' : len(itip_list), 'internaltips' : itip_list }) else: outputDict.update({ 'internaltips_elements' : len(itip_list)}) if what == 'rtip' or what == 'all' or what == 'count': rtip_iface = ReceiverTip() rtip_list = yield rtip_iface.get_all() if what != 'count': outputDict.update({ 'rtip_elements' : len(rtip_list), 'receivers_tips' : rtip_list }) else: outputDict.update({ 'rtip_elements' : len(rtip_list)}) if what == 'wtip' or what == 'all' or what == 'count': wtip_iface = WhistleblowerTip() wtip_list = yield wtip_iface.get_all() if what != 'count': outputDict.update({ 'wtip_elements' : len(wtip_list), 'whistleblower_tips' : wtip_list }) else: outputDict.update({ 'wtip_elements' : len(wtip_list)}) if what == 'comment' or what == 'all' or what == 'count': comment_iface = Comment() comment_list = yield comment_iface.get_all() if what != 'count': outputDict.update({ 'comment_elements' : len(comment_list), 'comments' : comment_list }) else: outputDict.update({ 'comment_elements' : len(comment_list)}) if what == 'profiles' or what == 'all' or what == 'count': profile_iface = PluginProfiles() profile_list = yield profile_iface.get_all() if what != 'count': outputDict.update({ 'profiles_elements' : len(profile_list), 'profiles' : profile_list }) else: outputDict.update({ 'profiles_elements' : len(profile_list)}) if what == 'plugins' or what == 'all' or what == 'count': plugin_list = yield PluginManager.get_all() if what != 'count': outputDict.update({ 'plugins_elements' : len(plugin_list), 'plugins' : plugin_list }) else: outputDict.update({ 'plugins_elements' : len(plugin_list) }) if what == 'rcfg' or what == 'all' or what == 'count': rconf_iface = ReceiverConfs() rconf_list = yield rconf_iface.get_all() if what != 'count': outputDict.update({ 'rcfg_elements' : len(rconf_list), 'settings' : rconf_list }) else: outputDict.update({ 'rcfg_elements' : len(rconf_list)}) if what == 'submission' or what == 'all' or what == 'count': submission_iface = Submission() submission_list = yield submission_iface.get_all() if what != 'count': outputDict.update({ 'submission_elements' : len(submission_list), 'submissions' : submission_list }) else: outputDict.update({ 'submission_elements' : len(submission_list)}) if what == 'file' or what == 'all' or what == 'count': file_iface = File() file_list = yield file_iface.get_all() if what != 'count': outputDict.update({ 'file_elements' : len(file_list), 'files' : file_list }) else: outputDict.update({ 'file_elements' : len(file_list)}) if what == 'contexts' or what == 'all' or what == 'count': context_iface = Context() context_list = yield context_iface.get_all() if what != 'count': outputDict.update({ 'contexts_elements' : len(context_list), 'contexts' : context_list }) else: outputDict.update({ 'contexts_elements' : len(context_list)}) if not what in expected: self.set_status(405) else: self.set_status(200) self.write(outputDict) self.finish()
def operation(self): """ Goal of this function is to check all the: Tips Comment Folder System Event marked as 'not notified' and perform notification. Notification plugin chose if perform a communication or not, Then became marked as: 'notification ignored', or 'notified' Every notification plugin NEED have a checks to verify if notification has been correctly performed. If not (eg: wrong login/password, network errors) would be marked as: 'unable to be notified', and a retry logic is in TODO """ plugin_type = u'notification' log.debug("[D]", self.__class__, 'operation', datetime.today().ctime()) receivertip_iface = ReceiverTip() receivercfg_iface = ReceiverConfs() profile_iface = PluginProfiles() # TODO digest check missing it's required refactor the scheduler using a dedicated Storm table not_notified_tips = yield receivertip_iface.get_tips_by_notification_mark(u'not notified') for single_tip in not_notified_tips: # from a single tip, we need to extract the receiver, and then, having # context + receiver, find out which configuration setting has active receivers_map = yield receivertip_iface.get_receivers_by_tip(single_tip['tip_gus']) receiver_info = receivers_map['actor'] receiver_conf = yield receivercfg_iface.get_active_conf(receiver_info['receiver_gus'], single_tip['context_gus'], plugin_type) if receiver_conf is None: print "Receiver", receiver_info['receiver_gus'], \ "has not an active notification settings in context", single_tip['context_gus'], "for", plugin_type # TODO applicative log, database tracking of queue continue # Ok, we had a valid an appropriate receiver configuration for the notification task related_profile = yield profile_iface.get_single(receiver_conf['profile_gus']) settings_dict = { 'admin_settings' : related_profile['admin_settings'], 'receiver_settings' : receiver_conf['receiver_settings']} plugin = PluginManager.instance_plugin(related_profile['plugin_name']) updated_tip = yield receivertip_iface.update_notification_date(single_tip['tip_gus']) return_code = plugin.do_notify(settings_dict, u'tip', updated_tip) if return_code: yield receivertip_iface.flip_mark(single_tip['tip_gus'], u'notified') else: yield receivertip_iface.flip_mark(single_tip['tip_gus'], u'unable to notify') # Comment Notification here it's just an incomplete version, that never would supports # digest or retry, until Task manager queue is implemented comment_iface = Comment() internaltip_iface = InternalTip() not_notified_comments = yield comment_iface.get_comment_by_mark(marker=u'not notified') for comment in not_notified_comments: receivers_list = yield internaltip_iface.get_receivers_by_itip(comment['internaltip_id']) # needed to obtain context! itip_info = yield internaltip_iface.get_single(comment['internaltip_id']) for receiver_info in receivers_list: receiver_conf = yield receivercfg_iface.get_active_conf(receiver_info['receiver_gus'], itip_info['context_gus'], plugin_type) if receiver_conf is None: # TODO applicative log, database tracking of queue continue # Ok, we had a valid an appropriate receiver configuration for the notification task related_profile = yield profile_iface.get_single(receiver_conf['profile_gus']) settings_dict = { 'admin_settings' : related_profile['admin_settings'], 'receiver_settings' : receiver_conf['receiver_settings']} plugin = PluginManager.instance_plugin(related_profile['plugin_name']) return_code = plugin.do_notify(settings_dict, u'comment', comment) if return_code: print "Notification of comment successful for user", receiver_conf['receiver_gus'] else: print "Notification of comment failed for user", receiver_conf['receiver_gus'] # remind: comment are not guarantee until Task manager is not developed yield comment_iface.flip_mark(comment['comment_id'], u'notified')