def run(self, state: Dict[str, Any]) -> Dict[str, Any]:
"""Run importer."""
self._info("Running actor importer with state: {0}...", state)
fetch_timestamp = state.get(
self._LATEST_ACTOR_TIMESTAMP, self.default_latest_timestamp
)
new_state = state.copy()
latest_actor_created_timestamp = None
for actors_batch in self._fetch_actors(fetch_timestamp):
if not actors_batch:
break
latest_actor_created_datetime = self._process_actors(actors_batch)
if latest_actor_created_datetime is not None:
latest_actor_created_timestamp = datetime_to_timestamp(
latest_actor_created_datetime
)
new_state[self._LATEST_ACTOR_TIMESTAMP] = latest_actor_created_timestamp
self._set_state(new_state)
latest_actor_timestamp = latest_actor_created_timestamp or fetch_timestamp
self._info(
"Actor importer completed, latest fetch {0}.",
timestamp_to_datetime(latest_actor_timestamp),
)
return {self._LATEST_ACTOR_TIMESTAMP: latest_actor_timestamp}
def run(self, state: Mapping[str, Any]) -> Mapping[str, Any]:
"""Run importer."""
self._info("Running indicator importer with state: {0}...", state)
self._clear_report_fetcher_cache()
fetch_timestamp = state.get(self._LATEST_INDICATOR_TIMESTAMP,
self.default_latest_timestamp)
latest_fetched_indicator_timestamp = None
for indicator_batch in self._fetch_indicators(fetch_timestamp):
if not indicator_batch:
break
if latest_fetched_indicator_timestamp is None:
first_in_batch = indicator_batch[0]
latest_fetched_indicator_timestamp = datetime_to_timestamp(
first_in_batch.published_date)
self._process_indicators(indicator_batch)
state_timestamp = latest_fetched_indicator_timestamp or fetch_timestamp
self._info(
"Indicator importer completed, latest fetch {0}.",
timestamp_to_datetime(state_timestamp),
)
return {self._LATEST_INDICATOR_TIMESTAMP: state_timestamp}
def run(self, state: Mapping[str, Any]) -> Mapping[str, Any]:
"""Run importer."""
self._info(
"Running report importer (update data: {0}, guess malware: {1}) with state: {2}...", # noqa: E501
self.update_existing_data,
self.guess_malware,
state,
)
self._clear_malware_guess_cache()
fetch_timestamp = state.get(self._LATEST_REPORT_TIMESTAMP,
self.default_latest_timestamp)
latest_fetched_report_timestamp = None
for reports_batch in self._fetch_reports(fetch_timestamp):
if not reports_batch:
break
if latest_fetched_report_timestamp is None:
first_in_batch = reports_batch[0]
created_date = first_in_batch.created_date
if created_date is None:
self._error(
"Missing created date for report {0} ({1})",
first_in_batch.name,
first_in_batch.id,
)
break
latest_fetched_report_timestamp = datetime_to_timestamp(
created_date)
self._process_reports(reports_batch)
state_timestamp = latest_fetched_report_timestamp or fetch_timestamp
self._info(
"Report importer completed, latest fetch {0}.",
timestamp_to_datetime(state_timestamp),
)
return {self._LATEST_REPORT_TIMESTAMP: state_timestamp}
def run(self, state: Mapping[str, Any]) -> Mapping[str, Any]:
"""Run importer."""
self._info("Running YARA master importer with state: {0}...", state)
# Ignore the Etag, see the comment below.
# e_tag = state.get(self._E_TAG)
last_modified = state.get(self._LAST_MODIFIED)
if last_modified is not None:
last_modified = timestamp_to_datetime(last_modified)
# TODO: CrowdStrike Etag and Last-Modified fails with HTTP 500.
# download = self._download_yara_master(e_tag, last_modified)
download = self._download_yara_master()
latest_e_tag = download.e_tag
latest_last_modified = download.last_modified
if last_modified is None or (
last_modified is not None
and latest_last_modified is not None
and latest_last_modified > last_modified
):
self._process_content(download.content)
else:
self._info("YARA master not modified, skipping...")
self._info(
"YARA master importer completed, latest download {0} ({1}).",
latest_last_modified,
latest_e_tag,
)
self._clear_report_fetcher_cache()
new_state: Dict[str, Any] = {}
if latest_e_tag is not None:
new_state[self._E_TAG] = latest_e_tag
if latest_last_modified is not None:
new_state[self._LAST_MODIFIED] = datetime_to_timestamp(latest_last_modified)
return new_state
def run(self, state: Dict[str, Any]) -> Dict[str, Any]:
"""Run importer."""
self._info("Running indicator importer with state: {0}...", state)
self._clear_report_fetcher_cache()
fetch_timestamp = state.get(self._LATEST_INDICATOR_TIMESTAMP,
self.default_latest_timestamp)
latest_indicator_published_datetime = None
for indicator_batch in self._fetch_indicators(fetch_timestamp):
if not indicator_batch:
break
latest_batch_published_datetime = self._process_indicators(
indicator_batch)
if latest_batch_published_datetime is not None and (
latest_indicator_published_datetime is None
or latest_batch_published_datetime >
latest_indicator_published_datetime):
latest_indicator_published_datetime = latest_batch_published_datetime
latest_indicator_published_timestamp = fetch_timestamp
if latest_indicator_published_datetime is not None:
latest_indicator_published_timestamp = datetime_to_timestamp(
latest_indicator_published_datetime)
self._info(
"Indicator importer completed, latest fetch {0}.",
timestamp_to_datetime(latest_indicator_published_timestamp),
)
return {
self._LATEST_INDICATOR_TIMESTAMP:
latest_indicator_published_timestamp
}
def run(self, state: Mapping[str, Any]) -> Mapping[str, Any]:
"""Run importer."""
self._info("Running actor importer with state: {0}...", state)
fetch_timestamp = state.get(
self._LATEST_ACTOR_TIMESTAMP, self.default_latest_timestamp
)
latest_fetched_actor_timestamp = None
for actors_batch in self._fetch_actors(fetch_timestamp):
if not actors_batch:
break
if latest_fetched_actor_timestamp is None:
first_in_batch = actors_batch[0]
created_date = first_in_batch.created_date
if created_date is None:
self._error(
"Missing created date for actor {0} ({1})",
first_in_batch.name,
first_in_batch.id,
)
break
latest_fetched_actor_timestamp = datetime_to_timestamp(created_date)
self._process_actors(actors_batch)
state_timestamp = latest_fetched_actor_timestamp or fetch_timestamp
self._info(
"Actor importer completed, latest fetch {0}.",
timestamp_to_datetime(state_timestamp),
)
return {self._LATEST_ACTOR_TIMESTAMP: state_timestamp}
def run(self, state: Dict[str, Any]) -> Dict[str, Any]:
"""Run importer."""
self._info("Running YARA master importer with state: {0}...", state)
# Ignore the Etag, see the comment below.
# e_tag = state.get(self._E_TAG)
last_modified = state.get(self._LAST_MODIFIED)
if last_modified is not None:
last_modified = timestamp_to_datetime(last_modified)
# XXX: Using Etag and Last-Modified results in HTTP 500.
# yara_master = self._fetch_yara_master(e_tag, last_modified)
yara_master = self._fetch_yara_master()
latest_e_tag = yara_master.e_tag
latest_last_modified = yara_master.last_modified
if (last_modified is not None and latest_last_modified is not None
and last_modified >= latest_last_modified):
self._info("YARA master not modified, skipping...")
return state
yara_rules = yara_master.rules
yara_rule_count = len(yara_rules)
self._info(
"YARA master with {0} rules...",
yara_rule_count,
)
new_yara_rules = self._update_existing(yara_rules)
new_yara_rule_count = len(new_yara_rules)
self._info(
"{0} new YARA rules...",
new_yara_rule_count,
)
grouped_yara_rules = self._group_yara_rules_by_report(new_yara_rules)
group_count = len(grouped_yara_rules)
self._info(
"{0} YARA rule groups...",
group_count,
)
for group, rules in grouped_yara_rules:
self._info("YARA rule group: ({0}) {1}", len(rules), group)
failed_count = 0
for yara_rule_group in grouped_yara_rules:
failed = self._process_yara_rule_group(yara_rule_group)
failed_count += failed
success_count = new_yara_rule_count - failed_count
self._info(
"YARA master importer completed (imported: {0}, total: {1}, e_tag: {2}, last_modified: {3})", # noqa: E501
success_count,
new_yara_rule_count,
latest_e_tag,
latest_last_modified,
)
self._clear_report_fetcher_cache()
new_state: Dict[str, Any] = {}
if latest_e_tag is not None:
new_state[self._E_TAG] = latest_e_tag
if latest_last_modified is not None:
new_state[self._LAST_MODIFIED] = datetime_to_timestamp(
latest_last_modified)
return new_state
