def process_saved_artifacts(self): """ Process anything in saved_artifacts that didn't have a match. """ for md5_, value in self.saved_artifacts.iteritems(): (saved_obj, data) = value if saved_obj._XSI_TYPE == 'FileObjectType': #print "Only File found in SA" sample = Sample.from_cybox(saved_obj, [self.source]) db_sample = Sample.objects(md5=md5_).first() if db_sample: # flat out replacing cybox sample object with one from db. # we add the source to track we got a copy from TAXII. # if we have a metadata only doc, the add_file_data below # will generate metadata for us. sample = db_sample sample.add_source(self.source) if data: sample.add_file_data(data) sample.save(username=self.source_instance.analyst) self.samples.append(('Sample', sample.md5))
def __parse_object(self, obs_obj): """ Parse an observable object. :param obs_obj: The observable object to parse. :type obs_obj: CybOX object type. """ properties = obs_obj.properties type_ = properties._XSI_TYPE #would isinstance be preferable? #elif isinstance(defined_obj, # cybox.objects.email_message_object.EmailMessage): #XXX: Need to check the database for an existing Sample or Indicator # and handle accordingly, or risk blowing it away!!!! if type_ == 'FileObjectType': sample = Sample.from_cybox(properties, [self.source]) md5_ = sample.md5 # do we already have this sample? db_sample = Sample.objects(md5=md5_).first() if db_sample: # flat out replacing cybox sample object with one from db. # we add the source to track we got a copy from TAXII. # if we have a metadata only doc, the add_file_data below # will generate metadata for us. sample = db_sample sample.add_source(self.source) if md5_ in self.saved_artifacts: (saved_obj, data) = self.saved_artifacts[md5_] if saved_obj._XSI_TYPE == 'FileObjectType': #print "Only File found in SA" return elif saved_obj._XSI_TYPE == 'ArtifactObjectType': #print "Found matching Artifact in SA" sample.add_file_data(data) sample.save(username=self.source_instance.analyst) self.samples.append(('Sample', sample.md5)) del self.saved_artifacts[md5_] else: #print "Saving File to SA" self.saved_artifacts[md5_] = (properties, None) elif type_ == 'EmailMessageObjectType': # we assume all emails coming in from TAXII are new emails. # there is no way to guarantee we found a dupe in the db. email = Email.from_cybox(properties, [self.source]) email.save(username=self.source_instance.analyst) self.emails.append(('Email', str(email.id))) elif type_ in ['URIObjectType', 'AddressObjectType']: indicator = Indicator.from_cybox(properties, [self.source]) ind_type = indicator.ind_type value = indicator.value db_indicator = Indicator.objects( Q(ind_type=ind_type) & Q(value=value)).first() if db_indicator: # flat out replacing cybox indicator object with one from db. # we add the source to track we got a copy from TAXII. indicator = db_indicator indicator.add_source(self.source) indicator.save(username=self.source_instance.analyst) self.indicators.append(('Indicator', str(indicator.id))) elif type_ == 'ArtifactObjectType': # XXX: Check properties.type_ to see if it is TYPE_FILE, # TYPE_MEMORY, from CybOX definitions. This isn't implemented # yet in Greg's code. Just parse the file blindly for now. #if properties.type_ == 'File': # sample = Sample.from_cybox(properties, [self.source]) #else: # print "XXX: got unknown artifact type %s" % properties.type_ data = base64.b64decode(properties.data) md5_ = md5(data).hexdigest() #print "Found Artifact" if md5_ in self.saved_artifacts: (saved_obj, data) = self.saved_artifacts[md5_] if saved_obj._XSI_TYPE == 'ArtifactObjectType': #print "Only Artifact found in SA" return elif saved_obj._XSI_TYPE == 'FileObjectType': #print "Found matching File in SA" sample = Sample.from_cybox(saved_obj, [self.source]) db_sample = Sample.objects(md5=md5_).first() if db_sample: # flat out replacing cybox sample object with one from db. # we add the source to track we got a copy from TAXII. # if we have a metadata only doc, the add_file_data below # will generate metadata for us. sample = db_sample sample.add_source(self.source) sample.add_file_data(data) sample.save(username=self.source_instance.analyst) self.samples.append(('Sample', sample.md5)) del self.saved_artifacts[md5_] else: #print "Saving Artifact to SA" self.saved_artifacts[md5_] = (properties, data)
def __parse_object(self, obs_obj): """ Parse an observable object. :param obs_obj: The observable object to parse. :type obs_obj: CybOX object type. """ properties = obs_obj.properties type_ = properties._XSI_TYPE #would isinstance be preferable? #elif isinstance(defined_obj, # cybox.objects.email_message_object.EmailMessage): #XXX: Need to check the database for an existing Sample or Indicator # and handle accordingly, or risk blowing it away!!!! if type_ == 'FileObjectType': sample = Sample.from_cybox(properties, [self.source]) md5_ = sample.md5 # do we already have this sample? db_sample = Sample.objects(md5=md5_).first() if db_sample: # flat out replacing cybox sample object with one from db. # we add the source to track we got a copy from TAXII. # if we have a metadata only doc, the add_file_data below # will generate metadata for us. sample = db_sample sample.add_source(self.source) if md5_ in self.saved_artifacts: (saved_obj, data) = self.saved_artifacts[md5_] if saved_obj._XSI_TYPE == 'FileObjectType': #print "Only File found in SA" return elif saved_obj._XSI_TYPE == 'ArtifactObjectType': #print "Found matching Artifact in SA" sample.add_file_data(data) sample.save(username=self.source_instance.analyst) self.samples.append(('Sample', sample.md5)) del self.saved_artifacts[md5_] else: #print "Saving File to SA" self.saved_artifacts[md5_] = (properties, None) elif type_ == 'EmailMessageObjectType': # we assume all emails coming in from TAXII are new emails. # there is no way to guarantee we found a dupe in the db. email = Email.from_cybox(properties, [self.source]) email.save(username=self.source_instance.analyst) self.emails.append(('Email', str(email.id))) elif type_ in ['URIObjectType', 'AddressObjectType']: indicator = Indicator.from_cybox(properties, [self.source]) ind_type = indicator.ind_type value = indicator.value db_indicator = Indicator.objects(Q(ind_type=ind_type) & Q(value=value)).first() if db_indicator: # flat out replacing cybox indicator object with one from db. # we add the source to track we got a copy from TAXII. indicator = db_indicator indicator.add_source(self.source) indicator.save(username=self.source_instance.analyst) self.indicators.append(('Indicator', str(indicator.id))) elif type_ == 'ArtifactObjectType': # XXX: Check properties.type_ to see if it is TYPE_FILE, # TYPE_MEMORY, from CybOX definitions. This isn't implemented # yet in Greg's code. Just parse the file blindly for now. #if properties.type_ == 'File': # sample = Sample.from_cybox(properties, [self.source]) #else: # print "XXX: got unknown artifact type %s" % properties.type_ data = base64.b64decode(properties.data) md5_ = md5(data).hexdigest() #print "Found Artifact" if md5_ in self.saved_artifacts: (saved_obj, data) = self.saved_artifacts[md5_] if saved_obj._XSI_TYPE == 'ArtifactObjectType': #print "Only Artifact found in SA" return elif saved_obj._XSI_TYPE == 'FileObjectType': #print "Found matching File in SA" sample = Sample.from_cybox(saved_obj, [self.source]) db_sample = Sample.objects(md5=md5_).first() if db_sample: # flat out replacing cybox sample object with one from db. # we add the source to track we got a copy from TAXII. # if we have a metadata only doc, the add_file_data below # will generate metadata for us. sample = db_sample sample.add_source(self.source) sample.add_file_data(data) sample.save(username=self.source_instance.analyst) self.samples.append(('Sample', sample.md5)) del self.saved_artifacts[md5_] else: #print "Saving Artifact to SA" self.saved_artifacts[md5_] = (properties, data)