def ScheduleSystemCronFlows(names=None, token=None): """Schedule all the SystemCronFlows found.""" if data_store.RelationalDBReadEnabled(category="cronjobs"): return cronjobs.ScheduleSystemCronJobs(names=names) errors = [] for name in config.CONFIG["Cron.disabled_system_jobs"]: try: cls = registry.FlowRegistry.FlowClassByName(name) except ValueError: errors.append("No such flow: %s." % name) continue if not aff4.issubclass(cls, SystemCronFlow): errors.append("Disabled system cron job name doesn't correspond to " "a flow inherited from SystemCronFlow: %s" % name) if names is None: names = iterkeys(registry.FlowRegistry.FLOW_REGISTRY) for name in names: cls = registry.FlowRegistry.FlowClassByName(name) if not aff4.issubclass(cls, SystemCronFlow): continue cron_args = rdf_cronjobs.CreateCronJobFlowArgs( periodicity=cls.frequency, lifetime=cls.lifetime, allow_overruns=cls.allow_overruns) cron_args.flow_runner_args.flow_name = name if cls.enabled: enabled = name not in config.CONFIG["Cron.disabled_system_jobs"] else: enabled = False job_urn = CronManager.CRON_JOBS_PATH.Add(name) with aff4.FACTORY.Create( job_urn, aff4_type=CronJob, mode="rw", token=token, force_new_version=False) as cron_job: # If the cronjob was already present we don't want to overwrite the # original start_time. existing_cron_args = cron_job.Get(cron_job.Schema.CRON_ARGS) if cron_args != existing_cron_args: cron_job.Set(cron_job.Schema.CRON_ARGS(cron_args)) cron_job.Set(cron_job.Schema.DISABLED(not enabled)) if errors: raise ValueError( "Error(s) while parsing Cron.disabled_system_jobs: %s" % errors)
def ScheduleSystemCronFlows(names=None, token=None): """Schedule all the SystemCronFlows found.""" errors = [] for name in config.CONFIG["Cron.disabled_system_jobs"]: try: cls = registry.FlowRegistry.FlowClassByName(name) except ValueError: errors.append("No such flow: %s." % name) continue if not aff4.issubclass(cls, SystemCronFlow): errors.append( "Disabled system cron job name doesn't correspond to " "a flow inherited from SystemCronFlow: %s" % name) if names is None: names = registry.FlowRegistry.FLOW_REGISTRY.keys() for name in names: cls = registry.FlowRegistry.FlowClassByName(name) if aff4.issubclass(cls, SystemCronFlow): cron_args = rdf_cronjobs.CreateCronJobFlowArgs( periodicity=cls.frequency) cron_args.flow_runner_args.flow_name = name cron_args.lifetime = cls.lifetime cron_args.allow_overruns = cls.allow_overruns cron_args.start_time = GetStartTime(cls) if cls.disabled: disabled = True else: disabled = name in config.CONFIG["Cron.disabled_system_jobs"] manager = GetCronManager() if data_store.RelationalDBReadEnabled(category="cronjobs"): manager.CreateJob(cron_args=cron_args, job_id=name, disabled=disabled) else: manager.CreateJob(cron_args=cron_args, job_id=name, token=token, disabled=disabled) if errors: raise ValueError( "Error(s) while parsing Cron.disabled_system_jobs: %s" % errors)
def CallFallback(self, artifact_name, request_data): classes = iteritems(artifact.ArtifactFallbackCollector.classes) for clsname, fallback_class in classes: if not aff4.issubclass(fallback_class, artifact.ArtifactFallbackCollector): continue if artifact_name in fallback_class.artifacts: if artifact_name in self.state.called_fallbacks: self.Log("Already called fallback class %s for artifact: %s", clsname, artifact_name) else: self.Log("Calling fallback class %s for artifact: %s", clsname, artifact_name) self.CallFlow( clsname, request_data=request_data.ToDict(), artifact_name=artifact_name, next_state="ProcessCollected") # Make sure we only try this once self.state.called_fallbacks.add(artifact_name) return True return False
def GetAllWellKnownFlows(cls, token=None): """Get instances of all well known flows.""" well_known_flows = {} for cls in itervalues(registry.FlowRegistry.FLOW_REGISTRY): if aff4.issubclass(cls, WellKnownFlow) and cls.well_known_session_id: well_known_flow = cls(cls.well_known_session_id, mode="rw", token=token) well_known_flows[cls.well_known_session_id.FlowName()] = well_known_flow return well_known_flows
def GetAllWellKnownFlows(cls, token=None): """Get instances of all well known flows.""" well_known_flows = {} for cls in itervalues(registry.AFF4FlowRegistry.FLOW_REGISTRY): if aff4.issubclass(cls, WellKnownFlow) and cls.well_known_session_id: well_known_flow = cls(cls.well_known_session_id, mode="rw", token=token) well_known_flows[cls.well_known_session_id.FlowName()] = well_known_flow return well_known_flows
def GetRendererForValueOrClass(cls, value, limit_lists=-1): """Returns renderer corresponding to a given value and rendering args.""" if inspect.isclass(value): value_cls = value else: value_cls = value.__class__ cache_key = "%s_%d" % (value_cls.__name__, limit_lists) try: renderer_cls = cls._renderers_cache[cache_key] except KeyError: candidates = [] for candidate in itervalues(ApiValueRenderer.classes): if candidate.value_class: candidate_class = candidate.value_class else: continue if inspect.isclass(value): if aff4.issubclass(value_cls, candidate_class): candidates.append((candidate, candidate_class)) else: if isinstance(value, candidate_class): candidates.append((candidate, candidate_class)) if not candidates: raise RuntimeError("No renderer found for value %s." % value.__class__.__name__) candidates = sorted(candidates, key=lambda candidate: len(candidate[1].mro())) renderer_cls = candidates[-1][0] cls._renderers_cache[cache_key] = renderer_cls return renderer_cls(limit_lists=limit_lists)
def CreateClientObject(self, vfs_fixture): """Make a new client object.""" # First remove the old fixture just in case its still there. aff4.FACTORY.Delete(self.client_id, token=self.token) # Create the fixture at a fixed time. with test_lib.FakeTime(self.age): for path, (aff4_type, attributes) in vfs_fixture: path %= self.args aff4_object = aff4.FACTORY.Create(self.client_id.Add(path), aff4_type, mode="rw", token=self.token) if data_store.RelationalDBWriteEnabled(): data_store.REL_DB.WriteClientMetadata( self.client_id.Basename(), fleetspeak_enabled=False) components = [ component for component in path.split("/") if component ] if components[0:2] == ["fs", "os"]: path_info = rdf_objects.PathInfo() path_info.path_type = rdf_objects.PathInfo.PathType.OS path_info.components = components[2:] if aff4_type in [ aff4_grr.VFSFile, aff4_grr.VFSMemoryFile ]: path_info.directory = False elif aff4_type == aff4_standard.VFSDirectory: path_info.directory = True else: raise ValueError("Incorrect AFF4 type: %s" % aff4_type) data_store.REL_DB.WritePathInfos( client_id=self.client_id.Basename(), path_infos=[path_info]) for attribute_name, value in iteritems(attributes): attribute = aff4.Attribute.PREDICATES[attribute_name] if isinstance(value, (str, unicode)): # Interpolate the value value %= self.args # Is this supposed to be an RDFValue array? if aff4.issubclass(attribute.attribute_type, rdf_protodict.RDFValueArray): rdfvalue_object = attribute() for item in value: new_object = rdfvalue_object.rdf_type.FromTextFormat( utils.SmartStr(item)) rdfvalue_object.Append(new_object) # It is a text serialized protobuf. elif aff4.issubclass(attribute.attribute_type, rdf_structs.RDFProtoStruct): # Use the alternate constructor - we always write protobufs in # textual form: rdfvalue_object = attribute.attribute_type.FromTextFormat( utils.SmartStr(value)) elif aff4.issubclass(attribute.attribute_type, rdfvalue.RDFInteger): rdfvalue_object = attribute(int(value)) else: rdfvalue_object = attribute(value) # If we don't already have a pathspec, try and get one from the stat. if aff4_object.Get(aff4_object.Schema.PATHSPEC) is None: # If the attribute was a stat, it has a pathspec nested in it. # We should add that pathspec as an attribute. if attribute.attribute_type == rdf_client_fs.StatEntry: stat_object = attribute.attribute_type.FromTextFormat( utils.SmartStr(value)) if stat_object.pathspec: pathspec_attribute = aff4.Attribute( "aff4:pathspec", rdf_paths.PathSpec, "The pathspec used to retrieve " "this object from the client.", "pathspec") aff4_object.AddAttribute( pathspec_attribute, stat_object.pathspec) if attribute in ["aff4:content", "aff4:content"]: # For AFF4MemoryStreams we need to call Write() instead of # directly setting the contents.. aff4_object.Write(rdfvalue_object.AsBytes()) else: aff4_object.AddAttribute(attribute, rdfvalue_object) if (isinstance(rdfvalue_object, rdf_client_fs.StatEntry) and rdfvalue_object.pathspec.pathtype != "UNSET"): if data_store.RelationalDBWriteEnabled(): client_id = self.client_id.Basename() path_info = rdf_objects.PathInfo.FromStatEntry( rdfvalue_object) data_store.REL_DB.WritePathInfos( client_id, [path_info]) # Populate the KB from the client attributes. if aff4_type == aff4_grr.VFSGRRClient: kb = rdf_client.KnowledgeBase() artifact.SetCoreGRRKnowledgeBaseValues(kb, aff4_object) aff4_object.Set(aff4_object.Schema.KNOWLEDGE_BASE, kb) # Make sure we do not actually close the object here - we only want to # sync back its attributes, not run any finalization code. aff4_object.Flush() if aff4_type == aff4_grr.VFSGRRClient: index = client_index.CreateClientIndex(token=self.token) index.AddClient(aff4_object)
def BuildTypeDescriptor(self, value_cls): result = ApiRDFValueDescriptor( name=value_cls.__name__, parents=[klass.__name__ for klass in value_cls.__mro__], doc=value_cls.__doc__ or "", kind="STRUCT") for field_desc in value_cls.type_infos: repeated = isinstance(field_desc, rdf_structs.ProtoList) if hasattr(field_desc, "delegate"): field_desc = field_desc.delegate field = ApiRDFValueFieldDescriptor( name=field_desc.name, index=field_desc.field_number, repeated=repeated, dynamic=isinstance(field_desc, rdf_structs.ProtoDynamicEmbedded)) field_type = field_desc.type if field_type is not None: field.type = field_type.__name__ if field_type.context_help_url: # Class attribute context_help_url masks similarly named protobuf # attribute. Using the Set method to set the right attribute. field.Set("context_help_url", field_type.context_help_url) if field_type == rdf_structs.EnumNamedValue: for enum_label in sorted(field_desc.enum, key=field_desc.enum.get): enum_value = field_desc.enum[enum_label] labels = [ rdf_structs.SemanticDescriptor.Labels.reverse_enum[x] for x in enum_value.labels or [] ] field.allowed_values.append( ApiRDFAllowedEnumValueDescriptor( name=enum_label, value=int(enum_value), labels=labels, doc=enum_value.description)) if (field_desc.default is not None and not aff4.issubclass(field_type, rdf_structs.RDFStruct) and hasattr(field_desc, "GetDefault")): field.default = field.GetDefaultValueClass()( field_desc.GetDefault()) if field_desc.description: field.doc = field_desc.description if field_desc.friendly_name: field.friendly_name = field_desc.friendly_name if field_desc.labels: field.labels = [ rdf_structs.SemanticDescriptor.Labels.reverse_enum[x] for x in field_desc.labels ] result.fields.append(field) for processor in self.descriptor_processors: result.fields = processor(self, result.fields) if getattr(value_cls, "union_field", None): result.union_field_name = value_cls.union_field try: result.default = value_cls() except Exception as e: # pylint: disable=broad-except # TODO(user): Some RDFStruct classes can't be constructed using # default constructor (without arguments). Fix the code so that # we can either construct all the RDFStruct classes with default # constructors or know exactly which classes can't be constructed # with default constructors. logging.debug("Can't create default for struct %s: %s", field_type.__name__, e) return result
def CreateClientObject(self, vfs_fixture): """Make a new client object.""" # First remove the old fixture just in case its still there. aff4.FACTORY.Delete(self.client_id, token=self.token) # Create the fixture at a fixed time. with test_lib.FakeTime(self.age): for path, (aff4_type, attributes) in vfs_fixture: path %= self.args aff4_object = aff4.FACTORY.Create( self.client_id.Add(path), aff4_type, mode="rw", token=self.token) if data_store.RelationalDBWriteEnabled(): data_store.REL_DB.WriteClientMetadata( self.client_id.Basename(), fleetspeak_enabled=False) components = [component for component in path.split("/") if component] if components[0:2] == ["fs", "os"]: path_info = rdf_objects.PathInfo() path_info.path_type = rdf_objects.PathInfo.PathType.OS path_info.components = components[2:] if aff4_type in [aff4_grr.VFSFile, aff4_grr.VFSMemoryFile]: path_info.directory = False elif aff4_type == aff4_standard.VFSDirectory: path_info.directory = True else: raise ValueError("Incorrect AFF4 type: %s" % aff4_type) data_store.REL_DB.WritePathInfos( client_id=self.client_id.Basename(), path_infos=[path_info]) for attribute_name, value in iteritems(attributes): attribute = aff4.Attribute.PREDICATES[attribute_name] if isinstance(value, (str, unicode)): # Interpolate the value value %= self.args # Is this supposed to be an RDFValue array? if aff4.issubclass(attribute.attribute_type, rdf_protodict.RDFValueArray): rdfvalue_object = attribute() for item in value: new_object = rdfvalue_object.rdf_type.FromTextFormat( utils.SmartStr(item)) rdfvalue_object.Append(new_object) # It is a text serialized protobuf. elif aff4.issubclass(attribute.attribute_type, rdf_structs.RDFProtoStruct): # Use the alternate constructor - we always write protobufs in # textual form: rdfvalue_object = attribute.attribute_type.FromTextFormat( utils.SmartStr(value)) elif aff4.issubclass(attribute.attribute_type, rdfvalue.RDFInteger): rdfvalue_object = attribute(int(value)) else: rdfvalue_object = attribute(value) # If we don't already have a pathspec, try and get one from the stat. if aff4_object.Get(aff4_object.Schema.PATHSPEC) is None: # If the attribute was a stat, it has a pathspec nested in it. # We should add that pathspec as an attribute. if attribute.attribute_type == rdf_client.StatEntry: stat_object = attribute.attribute_type.FromTextFormat( utils.SmartStr(value)) if stat_object.pathspec: pathspec_attribute = aff4.Attribute( "aff4:pathspec", rdf_paths.PathSpec, "The pathspec used to retrieve " "this object from the client.", "pathspec") aff4_object.AddAttribute(pathspec_attribute, stat_object.pathspec) if attribute in ["aff4:content", "aff4:content"]: # For AFF4MemoryStreams we need to call Write() instead of # directly setting the contents.. aff4_object.Write(rdfvalue_object.AsBytes()) else: aff4_object.AddAttribute(attribute, rdfvalue_object) if (isinstance(rdfvalue_object, rdf_client.StatEntry) and rdfvalue_object.pathspec.pathtype != "UNSET"): if data_store.RelationalDBWriteEnabled(): client_id = self.client_id.Basename() path_info = rdf_objects.PathInfo.FromStatEntry(rdfvalue_object) data_store.REL_DB.WritePathInfos(client_id, [path_info]) # Populate the KB from the client attributes. if aff4_type == aff4_grr.VFSGRRClient: kb = rdf_client.KnowledgeBase() artifact.SetCoreGRRKnowledgeBaseValues(kb, aff4_object) aff4_object.Set(aff4_object.Schema.KNOWLEDGE_BASE, kb) # Make sure we do not actually close the object here - we only want to # sync back its attributes, not run any finalization code. aff4_object.Flush() if aff4_type == aff4_grr.VFSGRRClient: index = client_index.CreateClientIndex(token=self.token) index.AddClient(aff4_object)