class ThreatConnect: """ """ def __init__(self, api_aid, api_sec, api_org, api_url): """ """ # logger self.log_level = { 'debug': logging.DEBUG, 'info': logging.INFO, 'warning': logging.WARNING, 'error': logging.ERROR, 'critical': logging.CRITICAL} self.formatter = logging.Formatter( '%(asctime)s - %(name)s - %(levelname)s - %(message)s (%(funcName)s:%(lineno)d)') self.tcl = tc_logger() # debugging self._memory_monitor = True # credentials self._api_aid = api_aid self._api_sec = api_sec # user defined values self._api_org = api_org self._api_url = api_url self._api_result_limit = 200 # default values self._activity_log = False self._api_request_timeout = 30 self._api_retries = 5 # maximum of 5 minute window self._api_sleep = 59 # seconds self._bulk_on_demand = False self._enable_report = False self._indicators_regex = indicators_regex self._proxies = {'https': None} self._retype = type(re.compile('')) # config items self._report = [] self._verify_ssl = False # initialize request session handle self._session = Session() # instantiate report object self.report = Report() # # Memory Testing # # self._p = psutil.Process(os.getpid()) # self._memory = self._p.memory_info().rss def _api_request_headers(self, ro): """ """ timestamp = int(time.time()) signature = "{0}:{1}:{2}".format(ro.path_url, ro.http_method, timestamp) # python 2.7, does not work on 3.x and not tested on 2.6 # hmac_signature = hmac.new(self._api_sec, signature, digestmod=hashlib.sha256).digest() # authorization = 'TC {0}:{1}'.format(self._api_aid, base64.b64encode(hmac_signature)) # python 3.x hmac_signature = hmac.new(self._api_sec.encode(), signature.encode(), digestmod=hashlib.sha256).digest() authorization = 'TC {0}:{1}'.format(self._api_aid, base64.b64encode(hmac_signature).decode()) ro.add_header('Timestamp', timestamp) ro.add_header('Authorization', authorization) def api_filter_handler(self, resource_obj, filter_objs): """ """ data_set = None if not filter_objs: # build api call (no filters) default_request_object = resource_obj.default_request_object data_set = self.api_response_handler(resource_obj, default_request_object) else: # # process each filter added to the resource object for retrieve # first_run = True # # each resource object can have x filter objects with an operator to join or intersect results # for filter_obj in filter_objs: obj_list = [] # temp storage for results on individual filter objects owners = filter_obj.owners if len(owners) == 0: # handle filters with no owners owners = [self._api_org] # use default org # iterate through all owners for o in owners: self.tcl.debug('owner: {0!s}'.format(o)) if len(filter_obj) > 0: # request object are for api filters for ro in filter_obj: if ro.owner_allowed: ro.set_owner(o) results = self.api_response_handler(resource_obj, ro) if ro.resource_type not in [ResourceType.OWNERS, ResourceType.VICTIMS, ResourceType.BATCH_JOBS]: # TODO: should this be done? # post filter owners for obj in results: if obj.owner_name.upper() != o.upper(): results.remove(obj) obj_list.extend(results) else: ro = filter_obj.default_request_object if ro.owner_allowed: ro.set_owner(o) results = self.api_response_handler(resource_obj, ro) if ro.resource_type not in [ResourceType.OWNERS, ResourceType.VICTIMS]: # TODO: should this be done? # post filter owners for obj in results: if obj.owner_name.upper() != o.upper(): results.remove(obj) obj_list.extend(results) # # post filters # pf_obj_set = set(obj_list) self.tcl.debug('count before post filter: {0:d}'.format(len(obj_list))) for pfo in filter_obj.post_filters: self.tcl.debug('pfo: {0!s}'.format(pfo)) # # Report Entry # report_entry = ReportEntry() report_entry.add_post_filter_object(pfo) # current post filter method filter_method = getattr(resource_obj, pfo.method) # current post filter results post_filter_results = set(filter_method(pfo.filter, pfo.operator, pfo.description)) pf_obj_set = pf_obj_set.intersection(post_filter_results) self.report.add(report_entry) # set obj_list to post_filter results if filter_obj.post_filters_len > 0: obj_list = list(pf_obj_set) self.tcl.debug('count after post filter: {0:d}'.format(len(obj_list))) # no need to join or intersect on first run if first_run: data_set = set(obj_list) first_run = False continue # # depending on the filter type the result will be intersected or joined # if filter_obj.operator is FilterSetOperator.AND: data_set = data_set.intersection(obj_list) elif filter_obj.operator is FilterSetOperator.OR: data_set.update(set(obj_list)) # # only add to report if these results should be tracked (exclude attribute, tags, etc) # self.report.add_filtered_results(len(data_set)) # # after intersection or join add the objects to the resource object # for obj in data_set: resource_obj.add_obj(obj) def api_request(self, ro): """ """ api_response = None fail_msg = None h_content_length = None h_content_type = None start = datetime.now() # # enable activity log # if self._activity_log: ro.enable_activity_log() # # prepare request # url = '{0!s}{1!s}'.format(self._api_url, ro.request_uri) api_request = Request(ro.http_method, url, data=ro.body, params=ro.payload) request_prepped = api_request.prepare() # # generate headers # ro.set_path_url(request_prepped.path_url) self._api_request_headers(ro) request_prepped.prepare_headers(ro.headers) # # Debug # self.tcl.debug('request_object: {0!s}'.format(ro)) self.tcl.debug('url: {0!s}'.format(url)) self.tcl.debug('path url: {0!s}'.format(request_prepped.path_url)) # # api request (gracefully handle temporary communications issues with the API) # for i in range(1, self._api_retries + 1, 1): try: api_response = self._session.send( request_prepped, verify=self._verify_ssl, timeout=self._api_request_timeout, proxies=self._proxies, stream=False) break except exceptions.ReadTimeout as e: self.tcl.error('Error: {0!s}'.format(e)) self.tcl.error('The server may be experiencing delays at the moment.') self.tcl.info('Pausing for {0!s} seconds to give server time to catch up.'.format(self._api_sleep)) time.sleep(self._api_sleep) self.tcl.info('Retry {0!s} ....'.format(i)) if i == self._api_retries: self.tcl.critical('Exiting: {0!s}'.format(e)) raise RuntimeError(e) except exceptions.ConnectionError as e: self.tcl.error('Error: {0!s}'.format(e)) self.tcl.error('Connection Error. The server may be down.') self.tcl.info('Pausing for {0!s} seconds to give server time to catch up.'.format(self._api_sleep)) time.sleep(self._api_sleep) self.tcl.info('Retry {0!s} ....'.format(i)) if i == self._api_retries: self.tcl.critical('Exiting: {0!s}'.format(e)) raise RuntimeError(e) except socket.error as e: self.tcl.critical('Exiting: {0!s}'.format(e)) raise RuntimeError(e) # # header values # if 'content-length' in api_response.headers: h_content_length = api_response.headers['content-length'] if 'content-type' in api_response.headers: h_content_type = api_response.headers['content-type'] # # raise exception on *critical* errors # non_critical_errors = [ b'The MD5 for this File is invalid, a File with this MD5 already exists', # 400 (application/json) b'The SHA-1 for this File is invalid, a File with this SHA-1 already exists', # 400 (application/json) b'The SHA-256 for this File is invalid, a File with this SHA-256 already exists', # 400 (application/json) b'The requested resource was not found', # 404 (application/json) b'Could not find resource for relative', # 500 (text/plain) b'The requested Security Label was not removed - access was denied', # 401 (application/json) ] # # TODO: work out some logic to improve the API error handling, possible area where API could improve # # valid status codes 200, 201, 202 # if api_response.status_code in [400, 401, 403, 500, 503]: if api_response.status_code not in [200, 201, 202]: # check for non critical errors that have bad status codes nce_found = False fail_msg = api_response.content for nce in non_critical_errors: # api_response_dict['message'] not in non_critical_errors: if re.findall(nce, api_response.content): nce_found = True break if ro.failure_callback is not None: ro.failure_callback(api_response.status_code) # raise error on bad status codes that are not defined as nce if not nce_found: self.tcl.critical('Status Code: {0:d}'.format(api_response.status_code)) self.tcl.critical('Failed API Response: {0!s}'.format(api_response.content)) if ro.failure_callback is not None: ro.failure_callback(api_response.status_code) raise RuntimeError(api_response.content) # # set response encoding (best guess) # if api_response.encoding is None: api_response.encoding = api_response.apparent_encoding # # Debug # self.tcl.debug('url: %s', api_response.url) self.tcl.debug('status_code: %s', api_response.status_code) self.tcl.debug('content-length: %s', h_content_length) self.tcl.debug('content-type: %s', h_content_type) # # Report # self.report.add_api_call() # count api calls self.report.add_request_time(datetime.now() - start) self.tcl.debug('Request Time: {0!s}'.format(datetime.now() - start)) if self._enable_report: report_entry = ReportEntry() report_entry.add_request_object(ro) report_entry.set_request_url(api_response.url) report_entry.set_status_code(api_response.status_code) report_entry.set_failure_msg(fail_msg) self.report.add(report_entry) # # return response # return api_response def api_response_handler(self, resource_obj, ro): """ """ # # initialize vars # api_response_dict = {} obj_list = [] # only track filter counts on request from this method ro.enable_track() # # debug # self.tcl.debug('Results Limit: {0!s}'.format(self._api_result_limit)) # only resource supports pagination if ro.resource_pagination: ro.set_result_limit(self._api_result_limit) ro.set_result_start(0) else: ro.set_remaining_results(1) while ro.remaining_results > 0: # # api request # api_response = self.api_request(ro) # self.tcl.debug('Results Content: {0!s}'.format(api_response.content)) self.tcl.debug('Status Code: {0!s}'.format(api_response.status_code)) self.tcl.debug('Content Type: {0!s}'.format(api_response.headers['content-type'])) # # Process API response # if api_response.headers['content-type'] == 'application/json': api_response_dict = api_response.json() # try and free memory for next api request api_response.close() del api_response # doesn't appear to clear memory # # BULK INDICATOR (does not have status) # if 'indicator' in api_response_dict: if ro.resource_type == ResourceType.INDICATORS: data = api_response_dict['indicator'] for item in data: obj_list.append(parse_indicator( item, resource_obj, ro.description, ro.request_uri, self._indicators_regex)) if len(obj_list) % 500 == 0: self.tcl.debug('obj_list len: {0!s}'.format(len(obj_list))) elif api_response_dict['status'] == 'Failure': # handle failed request (404 Resource not Found) if 'message' in api_response_dict: self.tcl.error('{0!s} "{1!s}"'.format(api_response_dict['message'], ro.description)) ro.set_remaining_results(0) continue # # ADVERSARIES # elif ro.resource_type == ResourceType.ADVERSARIES: data = api_response_dict['data']['adversary'] if not isinstance(data, list): data = [data] # for single results to be a list for item in data: obj_list.append( parse_group(item, ResourceType.ADVERSARIES, resource_obj, ro.description, ro.request_uri)) # # INDICATORS # elif ro.resource_type == ResourceType.INDICATORS: data = api_response_dict['data']['indicator'] if not isinstance(data, list): data = [data] # for single results to be a list for item in data: obj_list.append(parse_indicator( item, resource_obj, ro.description, ro.request_uri, self._indicators_regex)) # # ADDRESSES # elif ro.resource_type == ResourceType.ADDRESSES: data = api_response_dict['data']['address'] if not isinstance(data, list): data = [data] # for single results to be a list for item in data: obj_list.append(parse_indicator( item, resource_obj, ro.description, ro.request_uri, self._indicators_regex)) # # DOCUMENTS # elif ro.resource_type == ResourceType.DOCUMENTS: data = api_response_dict['data']['document'] if not isinstance(data, list): data = [data] # for single results to be a list for item in data: obj_list.append( parse_group( item, ResourceType.DOCUMENTS, resource_obj, ro.description, ro.request_uri)) # # EMAILS # elif ro.resource_type == ResourceType.EMAILS: data = api_response_dict['data']['email'] if not isinstance(data, list): data = [data] # for single results to be a list for item in data: obj_list.append( parse_group( item, ResourceType.EMAILS, resource_obj, ro.description, ro.request_uri)) # # EMAIL ADDRESSES # elif ro.resource_type == ResourceType.EMAIL_ADDRESSES: data = api_response_dict['data']['emailAddress'] if not isinstance(data, list): data = [data] # for single results to be a list for item in data: obj_list.append(parse_indicator( item, resource_obj, ro.description, ro.request_uri, self._indicators_regex)) # # GROUPS # elif ro.resource_type == ResourceType.GROUPS: data = api_response_dict['data']['group'] if not isinstance(data, list): data = [data] # for single results to be a list for item in data: obj_list.append( parse_group(item, ResourceType.GROUPS, resource_obj, ro.description, ro.request_uri)) # # FILES # elif ro.resource_type == ResourceType.FILES: data = api_response_dict['data']['file'] if not isinstance(data, list): data = [data] # for single results to be a list for item in data: obj_list.append(parse_indicator( item, resource_obj, ro.description, ro.request_uri, self._indicators_regex)) # # HOSTS # elif ro.resource_type == ResourceType.HOSTS: data = api_response_dict['data']['host'] if not isinstance(data, list): data = [data] # for single results to be a list for item in data: obj_list.append(parse_indicator( item, resource_obj, ro.description, ro.request_uri, self._indicators_regex)) # # DNSResolutions # elif ro.resource_type == ResourceType.DNS_RESOLUTIONS: data = api_response_dict['data']['dnsResolution'] if not isinstance(data, list): data = [data] # for single results to be a list for item in data: if 'addresses' in item: # don't process dns resolutions that have no addresses obj_list.append(parse_dns_resolution(item)) # # INCIDENTS # elif ro.resource_type == ResourceType.INCIDENTS: data = api_response_dict['data']['incident'] if not isinstance(data, list): data = [data] # for single results to be a list for item in data: obj_list.append( parse_group(item, ResourceType.INCIDENTS, resource_obj, ro.description, ro.request_uri)) # # METRICS # # elif ro.resource_type == ResourceType.OWNER_METRICS: # data = api_response_dict['data']['ownerMetric'] # if not isinstance(data, list): # data = [data] # for single results to be a list # for item in data: # obj_list.append( # parse_metrics(item, resource_obj, ro.description, ro.request_uri)) # # MINE # # elif ro.resource_type == ResourceType.OWNER_MINE: # data = api_response_dict['data']['owner'] # if not isinstance(data, list): # data = [data] # for single results to be a list # for item in data: # obj_list.append( # parse_metrics(item, resource_obj, ro.description, ro.request_uri)) # # MEMBERS # # elif ro.resource_type == ResourceType.OWNER_MEMBERS: # data = api_response_dict['data']['user'] # if not isinstance(data, list): # data = [data] # for single results to be a list # for item in data: # obj_list.append( # parse_metrics(item, resource_obj, ro.description, ro.request_uri)) # # OWNERS # elif ro.resource_type == ResourceType.OWNERS: data = api_response_dict['data']['owner'] if not isinstance(data, list): data = [data] # for single results to be a list for item in data: obj_list.append( parse_owner(item, resource_obj, ro.description, ro.request_uri)) # # SIGNATURES # elif ro.resource_type == ResourceType.SIGNATURES: data = api_response_dict['data']['signature'] if not isinstance(data, list): data = [data] # for single results to be a list for item in data: obj_list.append( parse_group(item, ResourceType.SIGNATURES, resource_obj, ro.description, ro.request_uri)) # # TASKS # elif ro.resource_type == ResourceType.TASKS: data = api_response_dict['data']['task'] if not isinstance(data, list): data = [data] # for single results to be a list for item in data: obj_list.append( parse_task(item, ResourceType.TASKS, resource_obj, ro.description, ro.request_uri)) # # THREATS # elif ro.resource_type == ResourceType.THREATS: data = api_response_dict['data']['threat'] if not isinstance(data, list): data = [data] # for single results to be a list for item in data: obj_list.append( parse_group(item, ResourceType.THREATS, resource_obj, ro.description, ro.request_uri)) # # URLS # elif ro.resource_type == ResourceType.URLS: data = api_response_dict['data']['url'] if not isinstance(data, list): data = [data] # for single results to be a list for item in data: obj_list.append(parse_indicator( item, resource_obj, ro.description, ro.request_uri, self._indicators_regex)) # # VICTIMS # elif ro.resource_type == ResourceType.VICTIMS: data = api_response_dict['data']['victim'] if not isinstance(data, list): data = [data] # for single results to be a list for item in data: # victims data comes back with no owner, manually add owner here item['owner'] = ro.owner obj_list.append(parse_victim(item, resource_obj, ro.description, ro.request_uri)) # # BatchJobs # elif ro.resource_type == ResourceType.BATCH_JOBS: data = api_response_dict['data']['batchStatus'] if not isinstance(data, list): data = [data] # for single results to be a list for item in data: # victims data comes back with no owner, manually add owner here item['owner'] = ro.owner obj_list.append(parse_batch_job(item, resource_obj, ro.description, ro.request_uri)) elif api_response.headers['content-type'] == 'text/plain': self.tcl.error('{0!s} "{1!s}"'.format(api_response.content, ro.description)) ro.set_remaining_results(0) continue # add_obj resource_pagination if required if ro.resource_pagination: # get the number of results returned by the api if ro.result_start == 0: ro.set_remaining_results(api_response_dict['data']['resultCount'] - ro.result_limit) else: ro.set_remaining_results(ro.remaining_results - ro.result_limit) # increment the start position ro.set_result_start(ro.result_start + ro.result_limit) else: ro.set_remaining_results(0) self.tcl.debug('Result Count: {0!s}'.format(len(obj_list))) self.report.add_unfiltered_results(len(obj_list)) return obj_list # # api / sdk settings # def result_pagination(self, ro, identifier): data = [] ro.set_result_limit(self._api_result_limit) ro.set_result_start(0) while ro.remaining_results > 0: api_response = self.api_request(ro) if api_response.headers['content-type'] == 'application/json': api_response_dict = api_response.json() if api_response_dict['status'] == 'Success': data.extend(api_response_dict['data'][identifier]) # get the number of results returned by the api if ro.result_start == 0: ro.set_remaining_results(api_response_dict['data']['resultCount'] - ro.result_limit) else: ro.set_remaining_results(ro.remaining_results - ro.result_limit) # increment the start position ro.set_result_start(ro.result_start + ro.result_limit) return data def report_enable(self): """ """ self._enable_report = True def report_disable(self): """ """ self._enable_report = False def set_activity_log(self, data_bool): """ enable or disable api activity log """ if isinstance(data_bool, bool): self._activity_log = data_bool def set_api_request_timeout(self, data_int): """ set timeout value for the requests module """ if isinstance(data_int, int): self._api_request_timeout = data_int else: raise AttributeError(ErrorCodes.e0110.value.format(data_int)) def set_api_retries(self, data): """ set the number of api retries before exception is raised """ if isinstance(data, int): self._api_retries = data else: raise AttributeError(ErrorCodes.e0120.value.format(data)) def set_api_sleep(self, data): """ set the amount of time between retries """ if isinstance(data, int): self._api_sleep = data else: raise AttributeError(ErrorCodes.e0130.value.format(data)) def set_api_result_limit(self, data_int): """ set the number of result to return per api request (500 max) """ if isinstance(data_int, int): self._api_result_limit = data_int else: raise AttributeError(ErrorCodes.e0140.value.format(data_int)) def set_proxies(self, proxy_address, proxy_port, proxy_user=None, proxy_pass=None): """ define proxy server to use with the requests module """ # "http": "http://*****:*****@10.10.1.10:3128/", # accept host with http(s) or without proxy_method = 'http://' if re.match('^http', proxy_address): proxy_method, proxy_host = proxy_address.split('//') proxy_method += '//' proxy_address = proxy_host # TODO: add validation if proxy_user is not None and proxy_pass is not None: self._proxies['https'] = '{0!s}{1!s}:{2!s}@{3!s}:{4!s}'.format( proxy_method, proxy_user, proxy_pass, proxy_address, proxy_port) else: self._proxies['https'] = '{0!s}{1!s}:{2!s}'.format( proxy_method, proxy_address, proxy_port) def get_proxies(self): """ get proxy settings """ return self._proxies def set_tcl_file(self, fqpn, level='info'): """ set the log file destination and log level """ file_path = os.path.dirname(fqpn) if os.access(file_path, os.W_OK): if self.tcl.level > self.log_level[level]: self.tcl.setLevel(self.log_level[level]) fh = logging.FileHandler(fqpn) # fh.set_name('tc_log_file') # not supported in python 2.6 if level in self.log_level.keys(): fh.setLevel(self.log_level[level]) else: fh.setLevel(self.log_level['info']) fh.setFormatter(self.formatter) self.tcl.addHandler(fh) # def set_tcl_level(self, level): # """ """ # if level in self.log_level.keys(): # if self.tcl.level > self.log_level[level]: # self.tcl.setLevel(self.log_level[level]) # self.tcl.handlers[0].setLevel(self.log_level[level]) def set_tcl_console_level(self, level): """ set the console log level """ if level in self.log_level.keys(): if self.tcl.level > self.log_level[level]: self.tcl.setLevel(self.log_level[level]) ch = logging.StreamHandler() # ch.set_name('console') # not supported in python 2.6 ch.setLevel(self.log_level[level]) ch.setFormatter(self.formatter) self.tcl.addHandler(ch) def set_indicator_regex(self, type_enum, compiled_regex): """ overwrite default SDK regex """ self.tcl.debug('overwrite regex for {0!s}'.format(type_enum.name)) if not isinstance(type_enum, IndicatorType): raise AttributeError(ErrorCodes.e0150.value.format(type_enum)) if not isinstance(compiled_regex, list): compiled_regex = [compiled_regex] cr_list = [] for cr in compiled_regex: if isinstance(cr, self._retype): cr_list.append(cr) else: raise AttributeError(ErrorCodes.e0160.value.format(cr)) self._indicators_regex[type_enum.name] = cr_list # # Resources # def adversaries(self): """ return an adversary container object """ return Adversaries(self) def bulk(self): """ return a bulk container object """ return Bulk(self) def bulk_indicators(self, on_demand=False): """ return a bulk indicator container object """ return BulkIndicators(self, on_demand) def documents(self): """ return a document container object """ return Documents(self) def emails(self): """ return an email container object """ return Emails(self) def groups(self): """ return an group container object """ return Groups(self) def incidents(self): """ return an incident container object """ return Incidents(self) def indicators(self): """ return an indicator container object """ return Indicators(self) def owners(self): """ return an owner container object """ return Owners(self) def signatures(self): """ return a signature container object """ return Signatures(self) def tasks(self): """ return a task container object """ return Tasks(self) def threats(self): """ return a threat container object """ return Threats(self) def victims(self): """ return a victim container object """ return Victims(self) def batch_jobs(self): return BatchJobs(self)
class ThreatConnect: """ """ def __init__(self, api_aid=None, api_sec=None, api_org=None, api_url=None, api_token=None, api_token_expires=None): """ """ # logger self.log_level = { 'debug': logging.DEBUG, 'info': logging.INFO, 'warning': logging.WARNING, 'error': logging.ERROR, 'critical': logging.CRITICAL } self.formatter = logging.Formatter( '%(asctime)s - %(name)s - %(levelname)s - %(message)s (%(funcName)s:%(lineno)d)' ) self.tcl = tc_logger() # debugging self._memory_monitor = True # credentials self._api_aid = api_aid self._api_sec = api_sec self._api_token = api_token self._api_token_expires = api_token_expires # user defined values self._api_org = api_org self._api_url = api_url self._api_result_limit = 200 # default values self._activity_log = False self._api_request_timeout = 30 self._api_retries = 5 # maximum of 5 minute window self._api_sleep = 59 # seconds self._bulk_on_demand = False self._enable_report = False self._indicators_regex = indicators_regex self._proxies = {'https': None} self._retype = type(re.compile('')) # config items self._report = [] self._verify_ssl = False # initialize request session handle self._session = Session() # instantiate report object self.report = Report() # save custom types for later self._indicator_parser = IndicatorObjectParser(self) # # Memory Testing # # self._p = psutil.Process(os.getpid()) # self._memory = self._p.memory_info().rss # @property def indicator_parser(self): return self._indicator_parser def _renew_token(self): """ { "success":true, "apiToken":"2:1:-1:1474673195:poZAT:syqtNUKnGn9ZijE5hQ5/D99aD8dIEdgdDCIMbjk2Poc\u003d", "apiTokenExpires":"1474673195" } """ # make api call to get new token url = '{0!s}{1!s}'.format(self._api_url.strip('api'), '/appAuth') payload = {'expiredToken': self._api_token} token_response = self._session.get(url, params=payload, verify=self._verify_ssl, timeout=self._api_request_timeout, proxies=self._proxies, stream=False) if token_response.status_code == 401: if 'application/json' in token_response.headers['content-type']: err_data = token_response.json().get('message') else: err_data = token_response.text err = 'Could not refresh ThreatConnect Token ({}).'.format( err_data) raise RuntimeError(err) # bcs - return new token and set expiration date token_data = token_response.json() self._api_token = token_data['apiToken'] self._api_token_expires = int(token_data['apiTokenExpires']) def _api_request_headers(self, ro): """ """ timestamp = int(time.time()) if self._api_token is not None and self._api_token_expires is not None: window_padding = 15 # bcs - possible configuration option current_time = int(time.time()) - window_padding if (int(self._api_token_expires) < current_time): self._renew_token() authorization = 'TC-Token {0}'.format(self._api_token) elif self._api_aid is not None and self._api_sec is not None: signature = "{0}:{1}:{2}".format(ro.path_url, ro.http_method, timestamp) # python 2.7, does not work on 3.x and not tested on 2.6 # hmac_signature = hmac.new(self._api_sec, signature, digestmod=hashlib.sha256).digest() # authorization = 'TC {0}:{1}'.format(self._api_aid, base64.b64encode(hmac_signature)) # python 3.x hmac_signature = hmac.new(self._api_sec.encode(), signature.encode(), digestmod=hashlib.sha256).digest() authorization = 'TC {0}:{1}'.format( self._api_aid, base64.b64encode(hmac_signature).decode()) ro.add_header('Timestamp', timestamp) ro.add_header('Authorization', authorization) def api_filter_handler(self, resource_obj, filter_objs): """ """ data_set = None if not filter_objs: # build api call (no filters) default_request_object = resource_obj.default_request_object data_set = self.api_response_handler(resource_obj, default_request_object) else: # # process each filter added to the resource object for retrieve # first_run = True # # each resource object can have x filter objects with an operator to join or intersect results # for filter_obj in filter_objs: obj_list = [ ] # temp storage for results on individual filter objects owners = filter_obj.owners if len(owners) == 0: # handle filters with no owners owners = [self._api_org] # use default org # iterate through all owners for o in owners: self.tcl.debug('owner: {0!s}'.format(o)) if len(filter_obj) > 0: # request object are for api filters for ro in filter_obj: if ro.owner_allowed: ro.set_owner(o) if hasattr(filter_obj, 'api_entity'): results = self.api_response_handler( resource_obj, ro, api_entity=filter_obj.api_entity) else: results = self.api_response_handler( resource_obj, ro) if ro.resource_type not in [ ResourceType.OWNERS, ResourceType.VICTIMS, ResourceType.BATCH_JOBS ]: # TODO: should this be done? # post filter owners for obj in results: if obj.owner_name.upper() != o.upper(): results.remove(obj) obj_list.extend(results) else: ro = filter_obj.default_request_object if ro.owner_allowed: ro.set_owner(o) if hasattr(filter_obj, 'api_entity'): results = self.api_response_handler( resource_obj, ro, api_entity=filter_obj.api_entity) else: results = self.api_response_handler( resource_obj, ro) if ro.resource_type not in [ ResourceType.OWNERS, ResourceType.VICTIMS ]: # TODO: should this be done? # post filter owners for obj in results: if obj.owner_name.upper() != o.upper(): results.remove(obj) obj_list.extend(results) # # post filters # pf_obj_set = set(obj_list) self.tcl.debug('count before post filter: {0:d}'.format( len(obj_list))) for pfo in filter_obj.post_filters: self.tcl.debug('pfo: {0!s}'.format(pfo)) # # Report Entry # report_entry = ReportEntry() report_entry.add_post_filter_object(pfo) # current post filter method filter_method = getattr(resource_obj, pfo.method) # current post filter results post_filter_results = set( filter_method(pfo.filter, pfo.operator, pfo.description)) pf_obj_set = pf_obj_set.intersection( post_filter_results) self.report.add(report_entry) # set obj_list to post_filter results if filter_obj.post_filters_len > 0: obj_list = list(pf_obj_set) self.tcl.debug('count after post filter: {0:d}'.format( len(obj_list))) # no need to join or intersect on first run if first_run: data_set = set(obj_list) first_run = False continue # # depending on the filter type the result will be intersected or joined # if filter_obj.operator is FilterSetOperator.AND: data_set = data_set.intersection(obj_list) elif filter_obj.operator is FilterSetOperator.OR: data_set.update(set(obj_list)) # # only add to report if these results should be tracked (exclude attribute, tags, etc) # self.report.add_filtered_results(len(data_set)) # # after intersection or join add the objects to the resource object # for obj in data_set: resource_obj.add_obj(obj) def api_request(self, ro, log=True): """ """ api_response = None fail_msg = None h_content_length = None h_content_type = None start = datetime.now() # # enable activity log # if self._activity_log: ro.enable_activity_log() # # prepare request # url = '{0!s}{1!s}'.format(self._api_url, ro.request_uri) api_request = Request(ro.http_method, url, data=ro.body, params=ro.payload) request_prepped = api_request.prepare() # # generate headers # ro.set_path_url(request_prepped.path_url) self._api_request_headers(ro) request_prepped.prepare_headers(ro.headers) # # Debug # if log: self.tcl.debug('request_object: {0!s}'.format(ro)) self.tcl.debug('url: {0!s}'.format(url)) self.tcl.debug('path url: {0!s}'.format(request_prepped.path_url)) # # api request (gracefully handle temporary communications issues with the API) # for i in range(1, self._api_retries + 1, 1): try: api_response = self._session.send( request_prepped, verify=self._verify_ssl, timeout=self._api_request_timeout, proxies=self._proxies, stream=ro.stream) break except exceptions.ReadTimeout as e: self.tcl.error('Error: {0!s}'.format(e)) self.tcl.error( 'The server may be experiencing delays at the moment.') self.tcl.info( 'Pausing for {0!s} seconds to give server time to catch up.' .format(self._api_sleep)) time.sleep(self._api_sleep) self.tcl.info('Retry {0!s} ....'.format(i)) if i == self._api_retries: self.tcl.critical('Exiting: {0!s}'.format(e)) raise RuntimeError(e) except exceptions.ConnectionError as e: self.tcl.error('Error: {0!s}'.format(e)) self.tcl.error('Connection Error. The server may be down.') self.tcl.info( 'Pausing for {0!s} seconds to give server time to catch up.' .format(self._api_sleep)) time.sleep(self._api_sleep) self.tcl.info('Retry {0!s} ....'.format(i)) if i == self._api_retries: self.tcl.critical('Exiting: {0!s}'.format(e)) raise RuntimeError(e) except socket.error as e: self.tcl.critical('Exiting: {0!s}'.format(e)) raise RuntimeError(e) # # header values # if 'content-length' in api_response.headers: h_content_length = api_response.headers['content-length'] if 'content-type' in api_response.headers: h_content_type = api_response.headers['content-type'] # # raise exception on *critical* errors # non_critical_errors = [ b'The MD5 for this File is invalid, a File with this MD5 already exists', # 400 (application/json) b'The SHA-1 for this File is invalid, a File with this SHA-1 already exists', # 400 (application/json) b'The SHA-256 for this File is invalid, a File with this SHA-256 already exists', # 400 (application/json) b'The requested resource was not found', # 404 (application/json) b'Could not find resource for relative', # 500 (text/plain) b'The requested Security Label was not removed - access was denied', # 401 (application/json) ] # # TODO: work out some logic to improve the API error handling, possible area where API could improve # # valid status codes 200, 201, 202 # if api_response.status_code in [400, 401, 403, 500, 503]: if api_response.status_code not in [200, 201, 202]: # check for non critical errors that have bad status codes nce_found = False fail_msg = api_response.content for nce in non_critical_errors: # api_response_dict['message'] not in non_critical_errors: if re.findall(nce, api_response.content): nce_found = True break if ro.failure_callback is not None: ro.failure_callback(api_response.status_code) # raise error on bad status codes that are not defined as nce if not nce_found: self.tcl.critical('Status Code: {0:d}'.format( api_response.status_code)) self.tcl.critical('Failed API Response: {0!s}'.format( api_response.content)) if ro.failure_callback is not None: ro.failure_callback(api_response.status_code) raise RuntimeError(api_response.content) # # set response encoding (best guess) # if api_response.encoding is None: ## api_response.encoding = api_response.apparent_encoding api_response.encoding = 'utf-8' # using apparent encoding is costly with bulk # # Debug # if log: self.tcl.debug('url: %s', api_response.url) self.tcl.debug('status_code: %s', api_response.status_code) self.tcl.debug('content-length: %s', h_content_length) self.tcl.debug('content-type: %s', h_content_type) # # Report # self.report.add_api_call() # count api calls self.report.add_request_time(datetime.now() - start) if log: self.tcl.debug('Request Time: {0!s}'.format(datetime.now() - start)) if self._enable_report: report_entry = ReportEntry() report_entry.add_request_object(ro) report_entry.set_request_url(api_response.url) report_entry.set_status_code(api_response.status_code) report_entry.set_failure_msg(fail_msg) self.report.add(report_entry) # # return response # return api_response def api_response_handler(self, resource_obj, ro, api_entity=None): """ """ # # initialize vars # api_response_dict = {} obj_list = [] # only track filter counts on request from this method ro.enable_track() # # debug # self.tcl.debug('Results Limit: {0!s}'.format(self._api_result_limit)) # only resource supports pagination if ro.resource_pagination: ro.set_result_limit(self._api_result_limit) ro.set_result_start(0) else: ro.set_remaining_results(1) while ro.remaining_results > 0: # # api request # api_response = self.api_request(ro) # self.tcl.debug('Results Content: {0!s}'.format(api_response.content)) self.tcl.debug('Status Code: {0!s}'.format( api_response.status_code)) self.tcl.debug('Content Type: {0!s}'.format( api_response.headers['content-type'])) # # Process API response # if api_response.headers['content-type'] == 'application/json': api_response_dict = api_response.json() # try and free memory for next api request api_response.close() del api_response # doesn't appear to clear memory # # BULK INDICATOR (does not have status) # if 'indicator' in api_response_dict: if ro.resource_type == ResourceType.INDICATORS: data = api_response_dict['indicator'] for item in data: obj_list.append( parse_typed_indicator( item, resource_obj, ro.description, ro.request_uri, self._indicators_regex, indicator_parser=self.indicator_parser)) if len(obj_list) % 500 == 0: self.tcl.debug('obj_list len: {0!s}'.format( len(obj_list))) elif api_response_dict['status'] == 'Failure': # handle failed request (404 Resource not Found) if 'message' in api_response_dict: self.tcl.error('{0!s} "{1!s}"'.format( api_response_dict['message'], ro.description)) ro.set_remaining_results(0) continue # # ADVERSARIES # elif ro.resource_type == ResourceType.ADVERSARIES: data = api_response_dict['data']['adversary'] if not isinstance(data, list): data = [data] # for single results to be a list for item in data: obj_list.append( parse_group(item, ResourceType.ADVERSARIES, resource_obj, ro.description, ro.request_uri)) # # CAMPAIGNS # elif ro.resource_type == ResourceType.CAMPAIGNS: data = api_response_dict['data']['campaign'] if not isinstance(data, list): data = [data] # for single results to be a list for item in data: obj_list.append( parse_group(item, ResourceType.CAMPAIGNS, resource_obj, ro.description, ro.request_uri)) # # INDICATORS # elif ro.resource_type == ResourceType.INDICATORS: data = api_response_dict['data']['indicator'] if not isinstance(data, list): data = [data] # for single results to be a list for item in data: obj_list.append( parse_typed_indicator( item, resource_obj, ro.description, ro.request_uri, self._indicators_regex, indicator_parser=self.indicator_parser)) # # ADDRESSES # elif ro.resource_type == ResourceType.ADDRESSES: data = api_response_dict['data']['address'] if not isinstance(data, list): data = [data] # for single results to be a list for item in data: obj_list.append( parse_typed_indicator( item, resource_obj, ro.description, ro.request_uri, self._indicators_regex, indicator_parser=self.indicator_parser)) # # DOCUMENTS # elif ro.resource_type == ResourceType.DOCUMENTS: data = api_response_dict['data']['document'] if not isinstance(data, list): data = [data] # for single results to be a list for item in data: obj_list.append( parse_group(item, ResourceType.DOCUMENTS, resource_obj, ro.description, ro.request_uri)) # # EMAILS # elif ro.resource_type == ResourceType.EMAILS: data = api_response_dict['data']['email'] if not isinstance(data, list): data = [data] # for single results to be a list for item in data: obj_list.append( parse_group(item, ResourceType.EMAILS, resource_obj, ro.description, ro.request_uri)) # # EMAIL ADDRESSES # elif ro.resource_type == ResourceType.EMAIL_ADDRESSES: data = api_response_dict['data']['emailAddress'] if not isinstance(data, list): data = [data] # for single results to be a list for item in data: obj_list.append( parse_typed_indicator( item, resource_obj, ro.description, ro.request_uri, self._indicators_regex, indicator_parser=self.indicator_parser)) # # CUSTOM INDICATORS # elif ro.resource_type == ResourceType.CUSTOM_INDICATORS: # api_entity MUST be provided for Custom Indicators data = api_response_dict['data'][api_entity] if not isinstance(data, list): data = [data] # for single results to be a list for item in data: obj_list.append( parse_typed_indicator( item, resource_obj, ro.description, ro.request_uri, self._indicators_regex, indicator_parser=self.indicator_parser)) # # GROUPS # elif ro.resource_type == ResourceType.GROUPS: data = api_response_dict['data']['group'] if not isinstance(data, list): data = [data] # for single results to be a list for item in data: obj_list.append( parse_group(item, ResourceType.GROUPS, resource_obj, ro.description, ro.request_uri)) # # FILES # elif ro.resource_type == ResourceType.FILES: data = api_response_dict['data']['file'] if not isinstance(data, list): data = [data] # for single results to be a list for item in data: obj_list.append( parse_typed_indicator( item, resource_obj, ro.description, ro.request_uri, self._indicators_regex, indicator_parser=self.indicator_parser)) # # HOSTS # elif ro.resource_type == ResourceType.HOSTS: data = api_response_dict['data']['host'] if not isinstance(data, list): data = [data] # for single results to be a list for item in data: obj_list.append( parse_typed_indicator( item, resource_obj, ro.description, ro.request_uri, self._indicators_regex, indicator_parser=self.indicator_parser)) # # DNSResolutions # elif ro.resource_type == ResourceType.DNS_RESOLUTIONS: data = api_response_dict['data']['dnsResolution'] if not isinstance(data, list): data = [data] # for single results to be a list for item in data: if 'addresses' in item: # don't process dns resolutions that have no addresses obj_list.append(parse_dns_resolution(item)) # # INCIDENTS # elif ro.resource_type == ResourceType.INCIDENTS: data = api_response_dict['data']['incident'] if not isinstance(data, list): data = [data] # for single results to be a list for item in data: obj_list.append( parse_group(item, ResourceType.INCIDENTS, resource_obj, ro.description, ro.request_uri)) # # METRICS # # elif ro.resource_type == ResourceType.OWNER_METRICS: # data = api_response_dict['data']['ownerMetric'] # if not isinstance(data, list): # data = [data] # for single results to be a list # for item in data: # obj_list.append( # parse_metrics(item, resource_obj, ro.description, ro.request_uri)) # # MINE # # elif ro.resource_type == ResourceType.OWNER_MINE: # data = api_response_dict['data']['owner'] # if not isinstance(data, list): # data = [data] # for single results to be a list # for item in data: # obj_list.append( # parse_metrics(item, resource_obj, ro.description, ro.request_uri)) # # MEMBERS # # elif ro.resource_type == ResourceType.OWNER_MEMBERS: # data = api_response_dict['data']['user'] # if not isinstance(data, list): # data = [data] # for single results to be a list # for item in data: # obj_list.append( # parse_metrics(item, resource_obj, ro.description, ro.request_uri)) # # OWNERS # elif ro.resource_type == ResourceType.OWNERS: data = api_response_dict['data']['owner'] if not isinstance(data, list): data = [data] # for single results to be a list for item in data: obj_list.append( parse_owner(item, resource_obj, ro.description, ro.request_uri)) # # SIGNATURES # elif ro.resource_type == ResourceType.SIGNATURES: data = api_response_dict['data']['signature'] if not isinstance(data, list): data = [data] # for single results to be a list for item in data: obj_list.append( parse_group(item, ResourceType.SIGNATURES, resource_obj, ro.description, ro.request_uri)) # # TASKS # elif ro.resource_type == ResourceType.TASKS: data = api_response_dict['data']['task'] if not isinstance(data, list): data = [data] # for single results to be a list for item in data: obj_list.append( parse_task(item, ResourceType.TASKS, resource_obj, ro.description, ro.request_uri)) # # THREATS # elif ro.resource_type == ResourceType.THREATS: data = api_response_dict['data']['threat'] if not isinstance(data, list): data = [data] # for single results to be a list for item in data: obj_list.append( parse_group(item, ResourceType.THREATS, resource_obj, ro.description, ro.request_uri)) # # URLS # elif ro.resource_type == ResourceType.URLS: data = api_response_dict['data']['url'] if not isinstance(data, list): data = [data] # for single results to be a list for item in data: obj_list.append( parse_typed_indicator( item, resource_obj, ro.description, ro.request_uri, self._indicators_regex, indicator_parser=self.indicator_parser)) # # VICTIMS # elif ro.resource_type == ResourceType.VICTIMS: data = api_response_dict['data']['victim'] if not isinstance(data, list): data = [data] # for single results to be a list for item in data: # victims data comes back with no owner, manually add owner here item['owner'] = ro.owner obj_list.append( parse_victim(item, resource_obj, ro.description, ro.request_uri)) # # BatchJobs # elif ro.resource_type == ResourceType.BATCH_JOBS: data = api_response_dict['data']['batchStatus'] if not isinstance(data, list): data = [data] # for single results to be a list for item in data: # victims data comes back with no owner, manually add owner here item['owner'] = ro.owner obj_list.append( parse_batch_job(item, resource_obj, ro.description, ro.request_uri)) elif api_response.headers['content-type'] == 'text/plain': self.tcl.error('{0!s} "{1!s}"'.format(api_response.content, ro.description)) ro.set_remaining_results(0) continue # add_obj resource_pagination if required if ro.resource_pagination: # get the number of results returned by the api if ro.result_start == 0: ro.set_remaining_results( api_response_dict['data']['resultCount'] - ro.result_limit) else: ro.set_remaining_results(ro.remaining_results - ro.result_limit) # increment the start position ro.set_result_start(ro.result_start + ro.result_limit) else: ro.set_remaining_results(0) self.tcl.debug('Result Count: {0!s}'.format(len(obj_list))) self.report.add_unfiltered_results(len(obj_list)) return obj_list # # api / sdk settings # def result_pagination(self, ro, identifier): data = [] ro.set_result_limit(self._api_result_limit) ro.set_result_start(0) while ro.remaining_results > 0: api_response = self.api_request(ro) if api_response.headers['content-type'] != 'application/json': break api_response_dict = api_response.json() if api_response_dict['status'] != 'Success': break data.extend(api_response_dict['data'][identifier]) # get the number of results returned by the api if ro.result_start == 0: ro.set_remaining_results( api_response_dict['data']['resultCount'] - ro.result_limit) else: ro.set_remaining_results(ro.remaining_results - ro.result_limit) # increment the start position ro.set_result_start(ro.result_start + ro.result_limit) return data def report_enable(self): """ """ self._enable_report = True def report_disable(self): """ """ self._enable_report = False def set_activity_log(self, data_bool): """ enable or disable api activity log """ if isinstance(data_bool, bool): self._activity_log = data_bool def set_api_request_timeout(self, data_int): """ set timeout value for the requests module """ if isinstance(data_int, int): self._api_request_timeout = data_int else: raise AttributeError(ErrorCodes.e0110.value.format(data_int)) def set_api_retries(self, data): """ set the number of api retries before exception is raised """ if isinstance(data, int): self._api_retries = data else: raise AttributeError(ErrorCodes.e0120.value.format(data)) def set_api_sleep(self, data): """ set the amount of time between retries """ if isinstance(data, int): self._api_sleep = data else: raise AttributeError(ErrorCodes.e0130.value.format(data)) def set_api_result_limit(self, data_int): """ set the number of result to return per api request (500 max) """ if isinstance(data_int, int): self._api_result_limit = data_int else: raise AttributeError(ErrorCodes.e0140.value.format(data_int)) def set_proxies(self, proxy_address, proxy_port, proxy_user=None, proxy_pass=None): """ define proxy server to use with the requests module """ # "http": "http://*****:*****@10.10.1.10:3128/", # accept host with http(s) or without proxy_method = 'http://' if re.match('^http', proxy_address): proxy_method, proxy_host = proxy_address.split('//') proxy_method += '//' proxy_address = proxy_host # TODO: add validation if proxy_user is not None and proxy_pass is not None: self._proxies['https'] = '{0!s}{1!s}:{2!s}@{3!s}:{4!s}'.format( proxy_method, proxy_user, proxy_pass, proxy_address, proxy_port) else: self._proxies['https'] = '{0!s}{1!s}:{2!s}'.format( proxy_method, proxy_address, proxy_port) def get_proxies(self): """ get proxy settings """ return self._proxies def set_tcl_file(self, fqpn, level='info'): """ set the log file destination and log level """ file_path = os.path.dirname(fqpn) if os.access(file_path, os.W_OK): if self.tcl.level > self.log_level[level]: self.tcl.setLevel(self.log_level[level]) if self._api_token is not None: fh = ApiLoggingHandler(fqpn, self) else: fh = FileHandler(fqpn) # fh.set_name('tc_log_file') # not supported in python 2.6 if level in self.log_level.keys(): fh.setLevel(self.log_level[level]) else: fh.setLevel(self.log_level['info']) fh.setFormatter(self.formatter) self.tcl.addHandler(fh) # def set_tcl_level(self, level): # """ """ # if level in self.log_level.keys(): # if self.tcl.level > self.log_level[level]: # self.tcl.setLevel(self.log_level[level]) # self.tcl.handlers[0].setLevel(self.log_level[level]) def set_tcl_console_level(self, level): """ set the console log level """ if level in self.log_level.keys(): if self.tcl.level > self.log_level[level]: self.tcl.setLevel(self.log_level[level]) ch = logging.StreamHandler() # ch.set_name('console') # not supported in python 2.6 ch.setLevel(self.log_level[level]) ch.setFormatter(self.formatter) self.tcl.addHandler(ch) def set_indicator_regex(self, type_enum, compiled_regex): """ overwrite default SDK regex """ self.tcl.debug('overwrite regex for {0!s}'.format(type_enum.name)) if not isinstance(type_enum, IndicatorType): raise AttributeError(ErrorCodes.e0150.value.format(type_enum)) if not isinstance(compiled_regex, list): compiled_regex = [compiled_regex] cr_list = [] for cr in compiled_regex: if isinstance(cr, self._retype): cr_list.append(cr) else: raise AttributeError(ErrorCodes.e0160.value.format(cr)) self._indicators_regex[type_enum.name] = cr_list # # Resources # def adversaries(self): """ return an adversary container object """ self._indicator_parser.init() return Adversaries(self) def bulk(self): """ return a bulk container object """ self._indicator_parser.init() return Bulk(self) def bulk_indicators(self, on_demand=False): """ return a bulk indicator container object """ self._indicator_parser.init() return BulkIndicators(self, on_demand) def campaigns(self): """ return an adversary container object """ self._indicator_parser.init() return Campaigns(self) def documents(self): """ return a document container object """ self._indicator_parser.init() return Documents(self) def emails(self): """ return an email container object """ self._indicator_parser.init() return Emails(self) def groups(self): """ return an group container object """ self._indicator_parser.init() return Groups(self) def incidents(self): """ return an incident container object """ self._indicator_parser.init() return Incidents(self) def indicators(self): """ return an indicator container object """ self._indicator_parser.init() return Indicators(self) def owners(self): """ return an owner container object """ self._indicator_parser.init() return Owners(self) def signatures(self): """ return a signature container object """ self._indicator_parser.init() return Signatures(self) def tasks(self): """ return a task container object """ self._indicator_parser.init() return Tasks(self) def threats(self): """ return a threat container object """ self._indicator_parser.init() return Threats(self) def victims(self): """ return a victim container object """ self._indicator_parser.init() return Victims(self) def batch_jobs(self): """ return a batch container object """ self._indicator_parser.init() return BatchJobs(self)