Esempio n. 1
0
class OtxAdapterControl(object):
    # シングルトン
    __instance = None
    # Scheduler
    _regist_schedule_flag = False

    def __new__(cls, *args, **kwargs):
        if cls.__instance is None:
            cls.__instance = object.__new__(cls)
        return cls.__instance

    def __init__(self):
        # 二重登録チェック
        if OtxAdapterControl._regist_schedule_flag:
            return
        OtxAdapterControl._regist_schedule_flag = True
        self._schedule = CtirsScheduler()
        # schedulerを起動
        # 起動時に1回のみ動作
        # 各job設定ごとに
        otx = None
        try:
            otx = OtxAdapter.objects.get()
            for job in otx.jobs:
                # add_jobするとstatusが変わるためその前に保持
                status = job.status
                self.add_job(job)
                if status == ScheduleJobs.STATUS_IN_OPERATION:
                    self.resume_job(job.id)
                else:
                    self.pause_job(job.id)
        except DoesNotExist:
            # OtxAdapterが存在しない場合はjob追加を行わない
            pass
        # interval の設定があった場合は追加する
        if otx is not None:
            self._interval_job = otx.interval_schedule_job
        else:
            self._interval_job = None
        if self._interval_job is not None:
            if otx.interval_schedule_job.status == ScheduleJobs.STATUS_IN_OPERATION:
                self.add_job(otx.interval_schedule_job)

    # otxからmtimestamp以降のデータを取得する
    def get_otx_stix(self, mtimestamp=None):
        # OTXアダプタの設定を取得
        otx_conf = OtxAdapter.get()
        key = otx_conf.apikey
        # 登録情報を取得
        community = otx_conf.community
        uploader = otx_conf.uploader
        via = Vias.get_via_adapter_otx(uploader)

        # otxから取得
        try:
            proxies = System.get_request_proxies()
            otx = OTXv2(key, proxies)
            slices = otx.getsince(mtimestamp)
        except Exception as e:
            traceback.print_exc()
            raise e

        # last_requested更新
        otx_conf.modify_last_requested()

        count = 0
        # ひとつずつ取得する
        for slice_ in slices:
            try:
                # stix一つごとに登録処理
                stix = StixExport(slice_)
                stix.build()
                content = stix.to_xml()
                # 取得したSTIXを登録
                _regist_stix(content, community, via)
                count += 1
            except Exception as e:
                # エラーが発生した場合はログを表示して処理は実行する
                traceback.print_exc()
        # 件数を返却
        return count

    # job起動用otxからmtimestamp以降のデータを取得する
    def _get_otx_stix_job(self):
        otx = OtxAdapter.objects.get()
        start_time_dt = otx.last_requested
        self.get_otx_stix(mtimestamp=start_time_dt.isoformat())

    # add job
    def add_job(self, schedule_job):
        # スケジューラに job 追加
        self._schedule.add_job(schedule_job, self._get_otx_stix_job)
        # job start
        self._schedule.start_job(schedule_job)
        # mongo の設定に格納する
        otx = OtxAdapter.objects.get()
        if schedule_job.job_type == ScheduleJobs.JOB_INTERVAL:
            # Interval指定の場合は_interval_jobを更新する
            otx.interval_schedule_job = schedule_job
            self._interval_job = schedule_job
        else:
            otx.jobs.append(schedule_job)

    # resume job
    def resume_job(self, job_id):
        schedule_job = ScheduleJobs.objects.get(id=job_id)
        if schedule_job in OtxAdapter.get().jobs:
            if schedule_job.status == ScheduleJobs.STATUS_STOP:
                pass
            else:
                print('already working.')
                return
        else:
            raise Exception('invalid job_id')
        self._schedule.resume_job(schedule_job)

    # stop job
    def pause_job(self, job_id):
        schedule_job = ScheduleJobs.objects.get(id=job_id)
        if schedule_job in OtxAdapter.get().jobs:
            if schedule_job.status == ScheduleJobs.STATUS_IN_OPERATION:
                pass
            else:
                print('not yet start.')
                return
        else:
            raise Exception('invalid job_id')

            return
        self._schedule.pause_job(schedule_job)

    # remove_job job
    def remove_job(self, job_id):
        # OtxAdapterのjobsからjob削除
        otx = OtxAdapter.get()
        otx.remove_job(job_id)
        # スケジューラからjob削除
        schedule_job = ScheduleJobs.objects.get(id=job_id)
        self._schedule.remove_job(schedule_job)
        # mongoのschedule_jobsからschedule_job削除
        schedule_job.remove()

    # remove_job job(interval)
    def remove_interval_job(self):
        if self._interval_job is None:
            # まだ設定されていない
            return
        schedule_job = ScheduleJobs.objects.get(id=self._interval_job.id)
        # スケジューラからjob削除
        self._schedule.remove_job(schedule_job)
        self._interval_job = None

    '''
Esempio n. 2
0
class Client(object):
    _SUBSCRIPTION_ID = 'CTI Repository System.'
    TAXII_2_ACCEPT = 'application/vnd.oasis.taxii+json; version=2.0'
    PLUG_FEST_AUTH_STRING = 'Basic dGVzDoxMjPCow=='
    TAXII_2_PUBLICATION_CONTENT_TYPE = TAXII_2_ACCEPT

    def __init__(self,taxii_name=None,taxii_id=None,community=None,taxii_client=None):
        taxii = None
        if taxii_client is not None:
            taxii = taxii_client
        elif taxii_name is not None:
            taxii = TaxiiClients.objects.get(name=taxii_name)
        elif taxii_id is not None:
            taxii = TaxiiClients.objects.get(id=taxii_id)
            
        self._address = taxii.address
        self._port = taxii.port
        self._path = taxii.path
        self._collection_name = taxii.collection
        self._jobs = taxii.jobs
        self._interval_job = taxii.interval_schedule_job
        self._protocol_version = taxii.protocol_version
        
        #taxii client設定
        self._client = clients.HttpClient()
        if taxii.is_use_cert == True:
            self._auth_type = clients.HttpClient.AUTH_CERT_BASIC
            self._key_file = taxii.key_file
            self._cert_file = taxii.cert_file
        else:
            self._auth_type = clients.HttpClient.AUTH_BASIC
        self._username = taxii.login_id
        self._password = taxii.login_password
        self._ssl = taxii.ssl

        #proxy 設定があれば設定する
        self._proxies = System.get_request_proxies()
        if self._proxies is not None:
            p = urlparse(self._address)
            if p.scheme == 'https':
                self._client.set_proxy(self._proxies['https'])
            else:
                self._client.set_proxy(self._proxies['http'])

        try:
            self._community = taxii.community
        except:
            self._community = None
        self._via = Vias.get_via_taxii_poll(taxii,uploader=taxii.uploader)
        self._taxii = taxii
        self._start = None
        self._end = None
        self._schedule = CtirsScheduler()
        #_scheduleのstartはschedule単位
        #modify/resume/pauseはjob単位
        
        #TAXII Protocol 1.1 なら Authentication 設定を行う
        if self._protocol_version ==  '1.1':
            self.set_taxii_11_authentication()
        
    #TAXII Protocol 1.1 の Authentication 設定を行う
    def set_taxii_11_authentication(self):
        try:
            #auth
            cert_file = None
            key_file = None

            #SSL使用設定
            self._client.set_use_https(self._ssl)
            #認証タイプ
            self._client.set_auth_type(self._auth_type)
            #BASIC認証情報
            auth_credentials_dict = {}
            auth_credentials_dict['username'] = self._username
            auth_credentials_dict['password'] = self._password
            #証明書認証の場合は情報追加
            if self._auth_type == clients.HttpClient.AUTH_CERT_BASIC:
                #certificate/private_keyの一時ファイルを作成
                _,cert_file = tempfile.mkstemp()
                with open(cert_file,'w') as fp:
                    fp.write(self._cert_file)
                _,key_file = tempfile.mkstemp()
                with open(key_file,'w') as fp:
                    fp.write(self._key_file)
                auth_credentials_dict['cert_file'] = cert_file
                auth_credentials_dict['key_file'] = key_file
            #認証情報設定
            self._client.set_auth_credentials(auth_credentials_dict)

        except Exception as e:
            traceback.print_exc()
            raise e
        finally:
            #一時ファイルを削除
            if cert_file is not None:
                os.remove(cert_file)
            if key_file is not None:
                os.remove(key_file)
        

    #TAXII 2.0 用 GET request Header
    def get_taxii_20_get_request_header(self):
        return {
            'Accept'        :   self.TAXII_2_ACCEPT,
            'Authorization' :   self.PLUG_FEST_AUTH_STRING,
        }

    #TAXII 2.0 用 POST request Header
    def get_taxii_20_post_request_header(self):
        return {
            'Accept'        :   self.TAXII_2_ACCEPT,
            'Authorization' :   self.PLUG_FEST_AUTH_STRING,
            'Content-Type' :    self.TAXII_2_PUBLICATION_CONTENT_TYPE,
        }

    #set start time
    def set_start_time(self,start):
        self._start = start

    #set end time
    def set_end_time(self,end):
        self._end = end
        
    #add job
    def add_job(self,schedule_job):
        self._schedule.add_job(schedule_job,self.poll_job)
        self._schedule.start_job(schedule_job)

    #resume job
    def resume_job(self,job_id):
        schedule_job = ScheduleJobs.objects.get(id=job_id)
        #interval job と一緒の場合
        if self._interval_job == schedule_job:
            if schedule_job.status == ScheduleJobs.STATUS_STOP:
                pass
            else:
                print 'already working.'
            return
        #schedule_jobと一緒(cron_jobリストにふくまれている)
        if schedule_job in self._jobs:
            if schedule_job.status == ScheduleJobs.STATUS_STOP:
                pass
            else:
                print 'already working.'
                return
        else:
            raise Exception('invalid job_id')
        self._schedule.resume_job(schedule_job)

    #stop job
    def pause_job(self,job_id):
        schedule_job = ScheduleJobs.objects.get(id=job_id)
        #interval job と一緒の場合
        if self._interval_job == schedule_job:
            if schedule_job.status == ScheduleJobs.STATUS_IN_OPERATION:
                pass
            else:
                print 'not yet start.'
            return
        if schedule_job in self._jobs:
            if schedule_job.status == ScheduleJobs.STATUS_IN_OPERATION:
                pass
            else:
                print 'not yet start.'
                return
        else:
            raise Exception('invalid job_id')
            return
        self._schedule.pause_job(schedule_job)
        
    #remove_job job
    def remove_job(self,job_id):
        schedule_job = ScheduleJobs.objects.get(id=job_id)
        #スケジューラからjob削除
        self._schedule.remove_job(schedule_job)
        #mongoのジョブ情報削除
        schedule_job.remove()

    #remove_job job(interval)
    def remove_interval_job(self):
        if self._interval_job is None:
            #まだ設定されていない
            return
        schedule_job = ScheduleJobs.objects.get(id=self._interval_job.id)
        #スケジューラからjob削除
        self._schedule.remove_job(schedule_job)
        #ScheduleJobs と ScheduleIntervalJobs を削除
        schedule_job.remove()
        self._interval_job = None

    #poll for job
    def poll_job(self):
        if self._taxii.last_requested is not None:
            self.set_start_time(self._taxii.last_requested.replace(tzinfo=pytz.utc))
            self.poll()

    #poll entry
    def poll(self):
        if self._protocol_version == '2.0':
            return self.poll_20()
        else:
            return self.poll_11()

    #poll(version 1.1) entry
    def poll_11(self):
        #request xml作成
        poll_parameters = tm11.PollParameters()
        poll_request = tm11.PollRequest(
            message_id=tm11.generate_message_id(),
            collection_name=self._collection_name,
            exclusive_begin_timestamp_label=self._start,
            inclusive_end_timestamp_label=self._end,
            poll_parameters=poll_parameters,
        )
        last_requested = datetime.datetime.now(pytz.utc)
        
        #from xml.dom.minidom import parseString
        #print( parseString(poll_request.to_xml()).toprettyxml(encoding='utf-8'))

        #request
        http_resp = self._client.call_taxii_service2(
            self._address,
            self._path,
            const.VID_TAXII_XML_11,
            poll_request.to_xml(),
            port=self._port)
        
        #taxii_message抽出
        taxii_message = libtaxii.get_message_from_http_response(http_resp, poll_request.message_id)
        
        #content_blocks が None or 存在しない場合は登録ししない
        try:
            if taxii_message.content_blocks is None:
                return 0
        except AttributeError:
            return 0

        #content_blocksごとに登録処理
        count = 0
        for cb in taxii_message.content_blocks:
            #stixファイルを一時ファイルに出力
            _,stix_file_path = tempfile.mkstemp(suffix='.xml')
            with open(stix_file_path,'wb+') as fp:
                #cb.contentがstixの中身
                fp.write(cb.content)
            #登録
            try:
                from ctirs.core.stix.regist import regist
                if self._community is not None:
                    regist(stix_file_path,self._community,self._via)
                count += 1
            except:
                #taxiiの場合は途中で失敗しても処理を継続する
                traceback.print_exc()
            
        #last_requested更新
        self._taxii.last_requested = last_requested
        self._taxii.save()
        
        return count
    
    def debug_print(self,msg):
        print msg
        
    #poll, push 共通 base_url 取得
    def get_taxii20_base_url(self):
        self.debug_print('>>> get_taxii20_base_url enter')

        ######### Basic TXS Get
        url = '%s:%d%s/' % (self._address,self._port,self._path)
        headers = {
            "Accept": "application/vnd.oasis.taxii+json; version=2.0",
        }
        self.debug_print('get_taxii20_base_url: URL: %s' % (url))
        self.debug_print('get_taxii20_base_url: _username: %s' % (self._username))
        self.debug_print('get_taxii20_base_url: _password: %s' % (self._password))
        self.debug_print('get_taxii20_base_url: request headers:%s' % (json.dumps(headers,indent=4)))
        try:
            resp = requests.get(
                url,
                headers = headers,
                auth = (self._username,self._password),
                verify = False,
                proxies = self._proxies
            )
        except Exception as e:
            traceback.print_exc()
            raise e
        self.debug_print('get_taxii20_base_url: response status_code:%s' % (resp.status_code))

        self.debug_print(resp.text)
        j = resp.json()
        self.debug_print('----')
        self.debug_print(json.dumps(j,indent=4))
        self.debug_print('----')
        
        base_url = None
        if j.has_key('default') == True:
            base_url = j['default']
        else:
            #default がない場合は url を使う
            base_url = url
        self.debug_print('get_taxii20_base_url: base_url:%s' % (base_url))
        return base_url

    #poll, push 共通 objects_url 取得
    def get_taxii20_objects_url(self):
        #base_url 取得
        base_url = self.get_taxii20_base_url()
        if base_url is None:
            self.debug_print('>>> get_taxii20_objects_url: No base_url')
            return None
        if base_url[-1] != '/':
            base_url = base_url + '/'
        url = base_url + 'collections/' + self._collection_name + '/objects/'
        return url


    #poll(version 2.0) entry
    def poll_20(self):
        self.debug_print('>>> poll_20:enter')

        url = self.get_taxii20_objects_url()
        if url is None:
            self.debug_print('>>> poll_20: No base_url')
            return 0
        headers = {
            'Accept'        :   'application/vnd.oasis.stix+json; version=2.0',
        }
        self.debug_print('poll_20: URL: %s' % (url))
        self.debug_print('poll_20: request headers:%s' % (json.dumps(headers,indent=4)))

        resp = requests.get(
            url,
            headers = headers,
            auth = (self._username,self._password),
            verify = False,
            proxies = self._proxies
        )
        self.debug_print('poll_20: response status_code:%s' % (resp.status_code))
        #self.debug_print('poll_20: response headers')
        #for k,v in resp.headers.iteritems():
        #    self.debug_print('%s: %s' % (k,v))
        #self.debug_print('----')
        #self.debug_print('poll_20: request resp.json():%s' % (json.dumps(resp.json(),indent=4)))
            
        #stix のリストで返却される
        count = 0
        stixes = []
        js =resp.json() 
        #list で返却される場合と単体の場合がある
        if isinstance(js, list) == True:
            stixes = js
        else:
            stixes = [js]
        for j in stixes:
            #1つずつ file に保存して regist する
            #stixファイルを一時ファイルに出力
            _,stix_file_path = tempfile.mkstemp(suffix='.json')
            with open(stix_file_path,'wb+') as fp:
                fp.write(json.dumps(j,indent=4))
            #登録
            try:
                from ctirs.core.stix.regist import regist
                if self._community is not None:
                    regist(stix_file_path,self._community,self._via)
                count += 1
            except:
                #taxiiの場合は途中で失敗しても処理を継続する
                traceback.print_exc()
                pass
        
        last_requested = datetime.datetime.now(pytz.utc)
        self._taxii.last_requested = last_requested
        self._taxii.save()
        return count

    #push entry
    def push(self,stix_file_doc):
        self.debug_print('>>> push: enter')
        if self._protocol_version == '2.0':
            self.push_20(stix_file_doc)
        else:
            self.push_11(stix_file_doc)

    #push(version 1.1) entry
    def push_11(self,stix_file_doc):
        self.debug_print('>>> push_11: enter')
        #stix_file_doc の versionが 2.0 ならスライド
        if stix_file_doc.version == '2.0':
            try:
                content = stix_file_doc.get_slide_1_x()
            except Exception as e:
                traceback.print_exc()
                raise e

        else:
            with open(stix_file_doc.origin_path,'r') as fp:
                content = fp.read()

        #Subscription Information xml作成
        subscription_information = tm11.SubscriptionInformation(
            collection_name = self._collection_name,
            subscription_id = 'subscripiton_id')
        cb = tm11.ContentBlock(const.CB_STIX_XML_11,content)
        im = tm11.InboxMessage(
            message_id = tm11.generate_message_id(),
            destination_collection_names = [self._collection_name],
            subscription_information = subscription_information,
            content_blocks = [cb])
        try:
            #push 
            self._address='10.0.3.100'
            #self.debug_print('>>> push_11: self._address:%s' %(self._address))
            #self.debug_print('>>> push_11: self._path:%s' %(self._path))
            #self.debug_print('>>> push_11: self._port:%d' %(self._port))
            #self.debug_print('>>> push_11: self._collection_name:%s' %(self._collection_name))
            #self.debug_print('>>> push_11: verify_server:%s' %(str(self._client.verify_server)))
            #self.debug_print('>>> push_11: use_https:%s' %(str(self._client.use_https)))
            http_resp = self._client.call_taxii_service2(
                self._address,
                self._path,
                const.VID_TAXII_XML_11,
                im.to_xml(),
                port=self._port)
            taxii_message = libtaxii.get_message_from_http_response(http_resp,im.message_id)
            if taxii_message.status_type != 'SUCCESS':
                self.debug_print('taxii_message.status_type is not SUCCESS')
                self.debug_print(taxii_message.to_xml(pretty_print=True))
                raise Exception('taxii_message.status_type is not SUCCESS')
        except Exception as e:
            traceback.print_exc()
            self.debug_print(e)
            raise e

    #push(version 2.0) entry
    def push_20(self,stix_file_doc):
        self.debug_print('>>> push_20: enter')
        #stix_file_doc の versionが 2.0以外 ならelevate
        if stix_file_doc.version != '2.0':
            content = stix_file_doc.get_elevate_2_x()
        else:
            with open(stix_file_doc.origin_path,'r') as fp:
                content = fp.read()

        '''
        #Data1/Write Collection
        url = '%s:%d%s' % (self._address,self._port,self._path)
        #print '>>>push_20'
        print '>>>push_20:url : %s' % (url)

        #TEST_DATA_1_ID = '91a7b528-80eb-42ed-a74d-bd5a2116'
        #TEST_DATA_1_VERIFICATION_URL = 'https://10.1.1.10/api1/collections/91a7b528-80eb-42ed-a74d-bd5a2116/'
        #TEST_DATA_1_PUBLICATION_URL = 'https://10.1.1.10/api1/collections/91a7b528-80eb-42ed-a74d-bd5a2116/objects'
        verification_url = url + 'collections/' + self._collection_name +'/'
        #print '>>>push_20:verification_url : %s' % (verification_url)
        #verification_url = url + 'collections/' + '91a7b528-80eb-42ed-a74d-bd5a2116' +'/'
        publication_url = verification_url + 'objects/'
        print '>>>push_20:publication_url : %s' % (publication_url)
        #TEST_DATA_1_TITLE = 'Write Collection 1'
        #TEST_DATA_1_DESCRIPTION = 'This is Write collection 1'
        #TEST_DATA_1_CAN_READ = False
        #TEST_DATA_1_CAN_WRITE = True
        #TEST_DATA_1_MEDIA_TYPES = [ "application/vnd.oasis.stix+json; version=2.0" ]
        
        #Data2/Read Collection
        #TEST_DATA_2_ID = '91a7b528-80eb-42ed-a74d-bd526120'
        #TEST_DATA_2_VERIFICATION_URL = 'https://10.1.1.10/api1/collections/91a7b528-80eb-42ed-a74d-bd526120/'
        #TEST_DATA_2_PUBLICATION_URL = 'https://10.1.1.10/api1/collections/91a7b528-80eb-42ed-a74d-bd526120/objects'
        #TEST_DATA_2_TITLE = 'Read Collection 1'
        #TEST_DATA_2_DESCRIPTION = 'This is read collection 1'
        #TEST_DATA_2_CAN_READ = True
        #TEST_DATA_2_CAN_WRITE = False
        #TEST_DATA_2_MEDIA_TYPES = [ "application/vnd.oasis.stix+json; version=2.0" ]

        #Data3/Read_Write Collection
        #TEST_DATA_3_ID = '91a7b528-80eb-42ed-a74d-bd5a2118'
        #TEST_DATA_3_VERIFICATION_URL = 'https://10.1.1.10/api1/collections/91a7b528-80eb-42ed-a74d-bd5a2118/'
        #TEST_DATA_3_PUBLICATION_URL = 'https://10.1.1.10/api1/collections/91a7b528-80eb-42ed-a74d-bd5a2118/objects'
        #TEST_DATA_3_TITLE = 'Read-Write Collection 1'
        #TEST_DATA_3_DESCRIPTION = 'This is read-write collection 1'
        #TEST_DATA_3_CAN_READ = True
        #TEST_DATA_3_CAN_WRITE = True
        #TEST_DATA_3_MEDIA_TYPES = [ "application/vnd.oasis.stix+json; version=2.0" ]     
        '''

        '''
        #Verify Collection Information
        headers = {
             "Accept": "application/vnd.oasis.taxii+json; version=2.0",
        }
        print 'taxii_push_20: verification:request headers:%s' % (json.dumps(headers,indent=4))
        print 'taxii_push_20: verification:request url:%s' % (verification_url)
        #print 'taxii_push_20: verification:request headers:%s' % (json.dumps(self.get_taxii_20_get_request_header(),indent=4))
        resp = requests.post(
            verification_url,
            #headers = self.get_taxii_20_get_request_header(),
            headers = headers,
            auth = (self._username,self._password),
            verify = False
            proxies = self._proxies
        )
        print resp
        print resp.status_code
        print resp.headers
        
        #if resp.status_code != 200 and resp.status_code != 202:
        if resp.status_code != 200:
            #error
            print 'taxii_push_20: verification:response_code:%d. Exit.' % (resp.status_code)
            return
        '''

        '''
        try:
            #check response parameter
            j = resp.json()
            #if j['id'] != TEST_DATA_1_ID:
            if j['id'] != self._collection_name:
                print 'taxii_push_20: verification: Invalid ID(%s). Exit.' % (j['id'])
                return
            if j['title'] != TEST_DATA_1_TITLE:
                print 'taxii_push_20: verification: Invalid title(%s). Exit.' % (j['title'])
                return
            if j['description'] != TEST_DATA_1_DESCRIPTION:
                print 'taxii_push_20: verification: Invalid description(%s). Exit.' % (j['description'])
                return
            if j['can_read'] != TEST_DATA_1_CAN_READ:
                print 'taxii_push_20: verification: Invalid can_read(%s). Exit.' % (str(j['can_read']))
                return
            if j['can_write'] != TEST_DATA_1_CAN_WRITE:
                print 'taxii_push_20: verification: Invalid can_write(%s). Exit.' % (str(j['can_write']))
                return
            if j['media_types'] != TEST_DATA_1_MEDIA_TYPES:
                print 'taxii_push_20: verification: Invalid media_types(%s). Exit.' % (str(j['media_types']))
                return
        except:
            traceback.print_exc()
            return
        '''
        
        '''
        INDICATOR_DATA = {
            "type": "bundle",
            "id": "bundle--5d0092c5-5f74-4287-9642-33f4c354e56d",
            "spec_version": "2.0",
            "objects": [
                {
                  "type": "identity",
                  "name": "ACME Corp, Inc.",
                  "identity_class": "organization",
                  "id": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff"
                },
                {
                  "type": "indicator",
                  "id": "indicator--8e2e2d2b-17d4-4cbf-938f-98ee46b3cd3f",
                  "created_by_ref": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff",
                  "created": "2016-04-06T20:03:48.000Z",
                  "modified": "2016-04-06T20:03:48.000Z",
                  "labels": [
                      "Malicious IP"
                      ],
                  "name": "Bad IP1",
                  "description": "This indicator should be monitored for malicious activity",
                  "pattern": "[ ipv4-addr:value: = '198.51.100.1' ]",
                  "valid_from": "2016-01-01T00:00:00Z"
                  }
            ]
        }
        '''

        #Publication
        #print 'taxii_push_20: publication:request headers:%s' % (json.dumps(self.get_taxii_20_post_request_header(),indent=4))
        url = self.get_taxii20_objects_url()
        self.debug_print('taxii_push_20: url:%s' % (url))
        headers = {
             "Accept": "application/vnd.oasis.taxii+json; version=2.0",
             "Content-Type": "application/vnd.oasis.stix+json; version=2.0"
        }
        self.debug_print('taxii_push_20: publication:request headers:%s' % (json.dumps(headers,indent=4)))
        resp = requests.post(
            url,
            headers = headers,
            auth = (self._username,self._password),
            data = content,
            verify = False,
            proxies = self._proxies
        )
        self.debug_print('taxii_push_20: resp.status_code: %s' % (resp.status_code))

        if resp.status_code != 202:
            #error
            self.debug_print('taxii_push_20: publication: response_code:%d. Exit.' % (resp.status_code))
            raise Exception('Invalid http response: %s' % (resp.status_code))

        try:
            #check response parameter
            j = resp.json()
            #self.debug_print(json.dumps(j,indent=4))
            if j['status'] != 'complete':
                self.debug_print('taxii_push_20: publication: Invalid status(%s). Exit.' % (j['status']))
                return
            if j['failure_count'] != 0:
                self.debug_print('taxii_push_20: publication: Invalid failure_count(%d). Exit.' % (j['failure_count']))
                return
            if j['pending_count'] != 0:
                self.debug_print('taxii_push_20: publication: Invalid pending_count(%d). Exit.' % (j['pending_count']))
                return 
        except Exception as e:
            traceback.print_exc()
            raise e
Esempio n. 3
0
class MispAdapterDownloadControl(object):
    # シングルトン
    __instance = None
    # Scheduler
    _regist_schedule_flag = False

    def __new__(cls, *args, **kwargs):
        if cls.__instance is None:
            cls.__instance = object.__new__(cls)
        return cls.__instance

    def __init__(self):
        # 二重登録チェック
        if MispAdapterDownloadControl._regist_schedule_flag:
            return
        MispAdapterDownloadControl._regist_schedule_flag = True
        self._schedule = CtirsScheduler()
        # schedulerを起動
        # 起動時に1回のみ動作
        # 各job設定ごとに
        misp = None
        try:
            misp = MispAdapter.objects.get()
            for job in misp.jobs:
                # add_jobするとstatusが変わるためその前に保持
                status = job.status
                self.add_job(job)
                if status == ScheduleJobs.STATUS_IN_OPERATION:
                    self.resume_job(job.id)
                else:
                    self.pause_job(job.id)
        except DoesNotExist:
            # OtxAdapterが存在しない場合はjob追加を行わない
            pass
        # interval の設定があった場合は追加する
        if misp is not None:
            self._interval_job = misp.interval_schedule_job
        else:
            self._interval_job = None
        if self._interval_job is not None:
            if misp.interval_schedule_job.status == ScheduleJobs.STATUS_IN_OPERATION:
                self.add_job(misp.interval_schedule_job)

    # misp から from_dt から to_dt までのデータを取得する
    def get_misp_stix(self, from_dt=None, to_dt=None):
        # misp アダプタの設定を取得
        misp_conf = MispAdapter.get()
        url = misp_conf.url
        apikey = misp_conf.apikey
        published_only = misp_conf.published_only
        if misp_conf.stix_version.startswith('1.'):
            stix_version = 'stix'
        else:
            stix_version = 'stix2'
        # 登録情報を取得
        community = misp_conf.community
        uploader = misp_conf.uploader
        via = Vias.get_via_adapter_misp(uploader)

        # mispから取得
        try:
            if url[-1] != '/':
                url += '/'
            url = url + 'events/restSearch'
            md = MISPDownloader(url, apikey)
            stix_packages = md.get(from_dt=from_dt,
                                   to_dt=to_dt,
                                   published_only=published_only,
                                   stix_version=stix_version)
        except Exception as e:
            traceback.print_exc()
            raise e

        # last_requested更新
        misp_conf.modify_last_requested()

        if stix_packages is None:
            return 0

        count = 0
        # ひとつずつ取得する
        for stix_package in stix_packages:
            try:
                if misp_conf.stix_version.startswith('1.'):
                    regist_flag = self._regist_12(stix_package, community, via)
                elif misp_conf.stix_version.startswith('2.'):
                    regist_flag = self._regist_20(stix_package, community, via)
                if regist_flag:
                    count += 1
            except Exception:
                # エラーが発生した場合はログを表示して処理は実行する
                traceback.print_exc()

        # 件数を返却
        return count

    def _regist_12(self, stix_package, community, via):
        try:
            StixFiles.objects.get(package_id=stix_package.id_)
            return False
        except DoesNotExist:
            _regist_stix(stix_package.to_xml(), community, via)
            return True

    def _regist_20(self, stix_package, community, via):
        try:
            StixFiles.objects.get(package_id=stix_package['id'])
            return False
        except DoesNotExist:
            _regist_stix(json.dumps(stix_package, indent=4), community, via)
            return True

    # job起動用 misp から last_requested 以降のデータを取得する
    def _get_misp_stix_job(self):
        misp = MispAdapter.objects.get()
        start_time_dt = misp.last_requested
        self.get_misp_stix(from_dt=start_time_dt)

    # add job
    def add_job(self, schedule_job):
        # スケジューラに job 追加
        self._schedule.add_job(schedule_job, self._get_misp_stix_job)
        # job start
        self._schedule.start_job(schedule_job)
        # mongo の設定に格納する
        misp = MispAdapter.objects.get()
        if schedule_job.job_type == ScheduleJobs.JOB_INTERVAL:
            # Interval指定の場合は_interval_jobを更新する
            misp.interval_schedule_job = schedule_job
            self._interval_job = schedule_job
        else:
            misp.jobs.append(schedule_job)

    # resume job
    def resume_job(self, job_id):
        schedule_job = ScheduleJobs.objects.get(id=job_id)
        if schedule_job in MispAdapter.get().jobs:
            if schedule_job.status == ScheduleJobs.STATUS_STOP:
                pass
            else:
                print('already working.')
                return
        else:
            raise Exception('invalid job_id')
        self._schedule.resume_job(schedule_job)

    # stop job
    def pause_job(self, job_id):
        schedule_job = ScheduleJobs.objects.get(id=job_id)
        if schedule_job in MispAdapter.get().jobs:
            if schedule_job.status == ScheduleJobs.STATUS_IN_OPERATION:
                pass
            else:
                print('not yet start.')
                return
        else:
            raise Exception('invalid job_id')

            return
        self._schedule.pause_job(schedule_job)

    # remove_job job
    def remove_job(self, job_id):
        # MispAdapter の jobs から job 削除
        misp = MispAdapter.get()
        misp.remove_job(job_id)
        # スケジューラから job 削除
        schedule_job = ScheduleJobs.objects.get(id=job_id)
        self._schedule.remove_job(schedule_job)
        # mongo の schedule_jobs から schedule_job 削除
        schedule_job.remove()

    def remove_interval_job(self):
        if self._interval_job is None:
            # まだ設定されていない
            return
        schedule_job = ScheduleJobs.objects.get(id=self._interval_job.id)
        # スケジューラからjob削除
        self._schedule.remove_job(schedule_job)
        self._interval_job = None
Esempio n. 4
0
class iSightAdapterControl(object):
    #固定値
    ACCEPT_VERSION = '2.5'
    API_URL = 'api.isightpartners.com'
    ACCEPT_JSON = 'application/json'
    ACCEPT_XML = 'application/stix'

    #シングルトン
    __instance = None
    #Scheduler
    _regist_schedule_flag = False

    def __new__(cls, *args, **kwargs):
        if cls.__instance is None:
            cls.__instance = object.__new__(cls)
        return cls.__instance

    def __init__(self):
        #二重登録チェック
        if iSightAdapterControl._regist_schedule_flag == True:
            return
        iSightAdapterControl._regist_schedule_flag = True
        self._schedule = CtirsScheduler()
        #schedulerを起動
        #起動時に1回のみ動作
        #各job設定ごとに
        isight = None
        try:
            isight = isightAdapter.objects.get()
            for job in isight.jobs:
                #add_jobするとstatusが変わるためその前に保持
                status = job.status
                self.add_job(job)
                if status == ScheduleJobs.STATUS_IN_OPERATION:
                    self.resume_job(job.id)
                else:
                    self.pause_job(job.id)
        except DoesNotExist:
            #isightが存在しない場合はjob追加を行わない
            pass
        #interval の設定があった場合は追加する
        if isight is not None:
            self._interval_job = isight.interval_schedule_job
        else:
            self._interval_job = None
        if self._interval_job is not None:
            if isight.interval_schedule_job.status == ScheduleJobs.STATUS_IN_OPERATION:
                self.add_job(isight.interval_schedule_job)

    #時間文字列はYYYY/MM/DD HH:MM:SSとする
    def get_isight_stix(self, start_time=None, end_time=None):
        #登録情報を取得
        isight_adapter = isightAdapter.get()
        community = isight_adapter.community
        uploader = isight_adapter.uploader
        via = Vias.get_via_adapter_isight(uploader)
        try:
            #範囲内のリストを取得する
            l = self._get_isight_stix_report_list(start_time, end_time)
        except Exception as e:
            traceback.print_exc()
            raise e

        #last_requested更新
        isight_adapter.modify_last_requested()

        #リストの各要素をSTIXで取得してregistする
        count = 0
        for report_id in l:
            try:
                content = self._get_isight_stix_report(report_id)
                #ファイル登録
                #self._regist_isight_stix(content,community,via)
                _regist_stix(content, community, via)
                count += 1
            except Exception as e:
                #エラーが発生した場合はログを表示して処理は実行する
                traceback.print_exc()
        return count

    #job起動用iSIGHT STIX取得
    def _get_isight_stix_job(self):
        isight_adapter = isightAdapter.objects.get()
        start_time_dt = isight_adapter.last_requested
        start_time = calendar.timegm(start_time_dt.timetuple())
        self.get_isight_stix(start_time=start_time)

    #add job
    def add_job(self, schedule_job):
        #スケジューラに job 追加
        self._schedule.add_job(schedule_job, self._get_isight_stix_job)
        #job start
        self._schedule.start_job(schedule_job)
        #mongo の設定に格納する
        isight = isightAdapter.get()
        if schedule_job.job_type == ScheduleJobs.JOB_INTERVAL:
            #Interval指定の場合は_interval_jobを更新する
            isight.interval_schedule_job = schedule_job
            self._interval_job = schedule_job
        else:
            isight.jobs.append(schedule_job)

    #resume job
    def resume_job(self, job_id):
        schedule_job = ScheduleJobs.objects.get(id=job_id)
        if schedule_job in isightAdapter.get().jobs:
            if schedule_job.status == ScheduleJobs.STATUS_STOP:
                pass
            else:
                print 'already working.'
                return
        else:
            raise Exception('invalid job_id')
        self._schedule.resume_job(schedule_job)

    #remove_job job(interval)
    def remove_interval_job(self):
        if self._interval_job is None:
            #まだ設定されていない
            return
        schedule_job = ScheduleJobs.objects.get(id=self._interval_job.id)
        #スケジューラからjob削除
        self._schedule.remove_job(schedule_job)
        self._interval_job = None

    #stop job
    def pause_job(self, job_id):
        schedule_job = ScheduleJobs.objects.get(id=job_id)
        if schedule_job in isightAdapter.get().jobs:
            if schedule_job.status == ScheduleJobs.STATUS_IN_OPERATION:
                pass
            else:
                print 'not yet start.'
                return
        else:
            raise Exception('invalid job_id')
            return
        self._schedule.pause_job(schedule_job)

    #remove_job job
    def remove_job(self, job_id):
        #isightAdapterのjobsからjob削除
        isight_adapter = isightAdapter.get()
        isight_adapter.remove_job(job_id)
        #スケジューラからjob削除
        schedule_job = ScheduleJobs.objects.get(id=job_id)
        self._schedule.remove_job(schedule_job)
        #mongoのschedule_jobsからschedule_job削除
        schedule_job.remove()

    #指定のreport_idのSTIXイメージを取得する
    def _get_isight_stix_report(self, report_id):
        query = '/report/%s' % (report_id)
        headers = self._get_headers(query, accept=self.ACCEPT_XML)
        resp = self._load_data(query, headers)
        if resp.status_code != 200:
            return None
        else:
            return resp.text

    #指定の時間範囲内のレポートリストを取得する
    def _get_isight_stix_report_list(self, start_time=None, end_time=None):
        query = '/report/index?'
        #時間範囲の引数を作成
        query_args = []
        if (start_time is not None):
            query_args.append('startDate=%d' % (start_time))
        if (end_time is not None):
            query_args.append('endDate=%d' % (end_time))
        for arg in query_args:
            query += (arg + '&')
        #最後の&(引数がない場合の?)を削除
        query = query[:-1]

        headers = self._get_headers(query, accept=self.ACCEPT_JSON)
        resp = self._load_data(query, headers)
        print resp.json()
        if resp.status_code != 200:
            return []
        else:
            j = resp.json()
            ret = []
            if j[u'success'] != True:
                return []
            for message in j[u'message']:
                ret.append(message[u'reportId'])
            return ret

    #API通信
    def _load_data(self, query, headers):
        url = 'https://%s%s' % (self.API_URL, query)
        proxies = System.get_request_proxies()
        resp = requests.get(url, headers=headers, proxies=proxies)
        return resp

    #APIで利用するHTTP Header作成
    def _get_headers(self, query, accept='application/json'):
        #各種キーをmongoに格納されている設定から取得
        isight_adapter = isightAdapter.objects.get()
        private_key = str(isight_adapter.private_key)
        public_key = str(isight_adapter.public_key)

        time_stamp = email.Utils.formatdate(localtime=True)
        new_data = query + self.ACCEPT_VERSION + accept + time_stamp
        hashed = hmac.new(private_key, new_data, hashlib.sha256)
        headers = {
            'Accept': accept,
            'Accept-Version': self.ACCEPT_VERSION,
            'X-Auth': public_key,
            'X-Auth-Hash': hashed.hexdigest(),
            'Date': time_stamp
        }
        return headers
Esempio n. 5
0
class Client(object):
    TAXII_2_ACCEPT = 'application/vnd.oasis.taxii+json; version=2.0'
    TAXII_2_PUBLICATION_CONTENT_TYPE = TAXII_2_ACCEPT

    def __init__(self,
                 taxii_name=None,
                 taxii_id=None,
                 community=None,
                 taxii_client=None):
        taxii = None
        if taxii_client is not None:
            taxii = taxii_client
        elif taxii_name is not None:
            taxii = TaxiiClients.objects.get(name=taxii_name)
        elif taxii_id is not None:
            taxii = TaxiiClients.objects.get(id=taxii_id)

        self._address = taxii.address
        self._port = taxii.port
        self._path = taxii.path
        self._collection_name = taxii.collection
        self._jobs = taxii.jobs
        self._interval_job = taxii.interval_schedule_job
        self._protocol_version = taxii.protocol_version

        self._client = clients.HttpClient()
        if taxii.is_use_cert:
            self._auth_type = clients.HttpClient.AUTH_CERT_BASIC
            self._key_file = taxii.key_file
            self._cert_file = taxii.cert_file
        else:
            self._auth_type = clients.HttpClient.AUTH_BASIC
        self._username = taxii.login_id
        self._password = taxii.login_password
        self._ssl = taxii.ssl
        self._client.set_use_https(self._ssl)
        self._client.set_auth_type(self._auth_type)

        self._proxies = System.get_request_proxies()
        if self._proxies is not None:
            p = urlparse(self._address)
            if p.scheme == 'https':
                self._client.set_proxy(self._proxies['https'])
            else:
                self._client.set_proxy(self._proxies['http'])

        try:
            self._community = taxii.community
        except BaseException:
            self._community = None
        self._via = Vias.get_via_taxii_poll(taxii, uploader=taxii.uploader)
        self._taxii = taxii
        self._start = None
        self._end = None
        self._schedule = CtirsScheduler()

    def _get_taxii_20_authorization(self):
        return _basic_auth_str(self._username, self._password)

    def get_taxii_20_get_request_header(self):
        return {
            'Accept': self.TAXII_2_ACCEPT,
            'Authorization': self._get_taxii_20_authorization(),
        }

    # TAXII 2.0 用 POST request Header
    def get_taxii_20_post_request_header(self):
        return {
            'Accept': self.TAXII_2_ACCEPT,
            'Authorization': self._get_taxii_20_authorization(),
            'Content-Type': self.TAXII_2_PUBLICATION_CONTENT_TYPE,
        }

    def set_start_time(self, start):
        self._start = start

    def set_end_time(self, end):
        self._end = end

    def add_job(self, schedule_job):
        self._schedule.add_job(schedule_job, self.poll_job)
        self._schedule.start_job(schedule_job)

    def resume_job(self, job_id):
        schedule_job = ScheduleJobs.objects.get(id=job_id)
        index = -1
        if self._interval_job == schedule_job:
            if schedule_job.status == ScheduleJobs.STATUS_IN_OPERATION:
                return
        if schedule_job in self._jobs:
            index = self._jobs.index(schedule_job)
            if schedule_job.status == ScheduleJobs.STATUS_IN_OPERATION:
                return
        else:
            raise Exception('invalid job_id')
        self._schedule.resume_job(schedule_job)
        if index > -1:
            self._jobs[index] = schedule_job

    def pause_job(self, job_id):
        schedule_job = ScheduleJobs.objects.get(id=job_id)
        index = -1
        if self._interval_job == schedule_job:
            if schedule_job.status == ScheduleJobs.STATUS_STOP:
                return
        if schedule_job in self._jobs:
            index = self._jobs.index(schedule_job)
            if schedule_job.status == ScheduleJobs.STATUS_STOP:
                return
        else:
            raise Exception('invalid job_id')
        self._schedule.pause_job(schedule_job)
        if index > -1:
            self._jobs[index] = schedule_job

    def remove_job(self, job_id):
        schedule_job = ScheduleJobs.objects.get(id=job_id)
        self._schedule.remove_job(schedule_job)
        schedule_job.remove()

    def remove_interval_job(self):
        if self._interval_job is None:
            return
        schedule_job = ScheduleJobs.objects.get(id=self._interval_job.id)
        self._schedule.remove_job(schedule_job)
        schedule_job.remove()
        self._interval_job = None

    def poll_job(self):
        if self._taxii.last_requested is not None:
            self.set_start_time(
                self._taxii.last_requested.replace(tzinfo=pytz.utc))
        self.poll()

    def poll(self):
        if self._protocol_version == '2.0':
            return self.poll_20()
        else:
            return self.poll_11()

    def _get_auth_credentials_dict(self):
        auth_credentials_dict = {}
        auth_credentials_dict['username'] = self._username
        auth_credentials_dict['password'] = self._password
        if self._auth_type == clients.HttpClient.AUTH_CERT_BASIC:
            _, cert_file = tempfile.mkstemp()
            with open(cert_file, 'w', encoding='utf-8') as fp:
                fp.write(self._cert_file)
            _, key_file = tempfile.mkstemp()
            with open(key_file, 'w', encoding='utf-8') as fp:
                fp.write(self._key_file)
            auth_credentials_dict['cert_file'] = cert_file
            auth_credentials_dict['key_file'] = key_file
        return auth_credentials_dict

    def poll_11(self):
        auth_credentials_dict = self._get_auth_credentials_dict()
        self._client.set_auth_credentials(auth_credentials_dict)
        try:
            poll_parameters = tm11.PollParameters()
            poll_request = tm11.PollRequest(
                message_id=tm11.generate_message_id(),
                collection_name=self._collection_name,
                exclusive_begin_timestamp_label=self._start,
                inclusive_end_timestamp_label=self._end,
                poll_parameters=poll_parameters,
            )
            last_requested = datetime.datetime.now(pytz.utc)

            http_resp = self._client.call_taxii_service2(
                self._address,
                self._path,
                const.VID_TAXII_XML_11,
                poll_request.to_xml(),
                port=self._port)

            taxii_message = libtaxii.get_message_from_http_response(
                http_resp, poll_request.message_id)

            try:
                if taxii_message.content_blocks is None:
                    return 0
            except AttributeError:
                return 0

            count = 0
            for cb in taxii_message.content_blocks:
                _, stix_file_path = tempfile.mkstemp(suffix='.xml')
                with open(stix_file_path, 'wb+') as fp:
                    fp.write(cb.content)
                try:
                    from ctirs.core.stix.regist import regist
                    if self._community is not None:
                        regist(stix_file_path, self._community, self._via)
                    count += 1
                except BaseException:
                    traceback.print_exc()

            self._taxii.last_requested = last_requested
            self._taxii.save()

            return count
        finally:
            if 'cert_file' in auth_credentials_dict:
                try:
                    os.remove(auth_credentials_dict['cert_file'])
                except Exception:
                    pass
            if 'key_file' in auth_credentials_dict:
                try:
                    os.remove(auth_credentials_dict['key_file'])
                except Exception:
                    pass

    def get_taxii20_base_url(self):
        url = '%s:%d%s/' % (self._address, self._port, self._path)
        headers = {
            "Accept": "application/vnd.oasis.taxii+json; version=2.0",
        }
        try:
            resp = requests.get(url,
                                headers=headers,
                                auth=(self._username, self._password),
                                verify=False,
                                proxies=self._proxies)
        except Exception as e:
            traceback.print_exc()
            raise e

        j = resp.json()
        base_url = None
        if ('default' in j):
            base_url = j['default']
        else:
            base_url = url
        return base_url

    def get_taxii20_objects_url(self):
        base_url = self.get_taxii20_base_url()
        if base_url is None:
            return None
        if base_url[-1] != '/':
            base_url = base_url + '/'
        url = base_url + 'collections/' + self._collection_name + '/objects/'
        return url

    def poll_20(self):
        url = self.get_taxii20_objects_url()
        if url is None:
            return 0
        headers = {
            'Accept': 'application/vnd.oasis.stix+json; version=2.0',
        }

        resp = requests.get(url,
                            headers=headers,
                            auth=(self._username, self._password),
                            verify=False,
                            proxies=self._proxies)

        count = 0
        stixes = []
        js = resp.json()
        if isinstance(js, list):
            stixes = js
        else:
            stixes = [js]
        for j in stixes:
            _, stix_file_path = tempfile.mkstemp(suffix='.json')
            with open(stix_file_path, 'wb+') as fp:
                fp.write(json.dumps(j, indent=4))
            try:
                from ctirs.core.stix.regist import regist
                if self._community is not None:
                    regist(stix_file_path, self._community, self._via)
                count += 1
            except BaseException:
                traceback.print_exc()

        last_requested = datetime.datetime.now(pytz.utc)
        self._taxii.last_requested = last_requested
        self._taxii.save()
        return count

    def push(self, stix_file_doc):
        if self._protocol_version == '2.0':
            self.push_20(stix_file_doc)
        else:
            self.push_11(stix_file_doc)

    def push_11(self, stix_file_doc):
        if stix_file_doc.version.startswith('2.'):
            try:
                content = stix_file_doc.get_slide_12()
            except Exception as e:
                traceback.print_exc()
                raise e
        else:
            with open(stix_file_doc.origin_path, 'r', encoding='utf-8') as fp:
                content = fp.read()

        auth_credentials_dict = self._get_auth_credentials_dict()
        self._client.set_auth_credentials(auth_credentials_dict)

        try:
            subscription_information = tm11.SubscriptionInformation(
                collection_name=self._collection_name,
                subscription_id='subscripiton_id')
            cb = tm11.ContentBlock(const.CB_STIX_XML_11, content)
            im = tm11.InboxMessage(
                message_id=tm11.generate_message_id(),
                destination_collection_names=[self._collection_name],
                subscription_information=subscription_information,
                content_blocks=[cb])

            http_resp = self._client.call_taxii_service2(
                self._address,
                self._path,
                const.VID_TAXII_XML_11,
                im.to_xml(),
                port=self._port)
            taxii_message = libtaxii.get_message_from_http_response(
                http_resp, im.message_id)
            if taxii_message.status_type != 'SUCCESS':
                raise Exception('taxii_message.status_type is not SUCCESS')
        except Exception as e:
            traceback.print_exc()
            raise e
        finally:
            if 'cert_file' in auth_credentials_dict:
                try:
                    os.remove(auth_credentials_dict['cert_file'])
                except Exception:
                    pass
            if 'key_file' in auth_credentials_dict:
                try:
                    os.remove(auth_credentials_dict['key_file'])
                except Exception:
                    pass

    def push_20(self, stix_file_doc):
        if stix_file_doc.version != '2.0':
            content = stix_file_doc.get_elevate_20()
        else:
            with open(stix_file_doc.origin_path, 'r', encoding='utf-8') as fp:
                content = fp.read()

        url = self.get_taxii20_objects_url()
        headers = {
            "Accept": "application/vnd.oasis.taxii+json; version=2.0",
            "Content-Type": "application/vnd.oasis.stix+json; version=2.0"
        }
        resp = requests.post(url,
                             headers=headers,
                             auth=(self._username, self._password),
                             data=content,
                             verify=False,
                             proxies=self._proxies)

        if resp.status_code != 202:
            raise Exception('Invalid http response: %s' % (resp.status_code))

        try:
            j = resp.json()
            if j['status'] != 'complete':
                return
            if j['failure_count'] != 0:
                return
            if j['pending_count'] != 0:
                return
        except Exception as e:
            traceback.print_exc()
            raise e
Esempio n. 6
0
class MispAdapterDownloadControl(object):
    ns_url = 'http://s-tip.fujtisu.com'
    ns_name = 's-tip'
    default_identity_name = 's-tip'

    # MISP2STIX Converter
    mc = MISP2STIXConverter(identity_name=default_identity_name,
                            ns_url=ns_url,
                            ns_name=ns_name)

    # シングルトン
    __instance = None
    # Scheduler
    _regist_schedule_flag = False

    def __new__(cls, *args, **kwargs):
        if cls.__instance is None:
            cls.__instance = object.__new__(cls)
        return cls.__instance

    def __init__(self):
        # 二重登録チェック
        if MispAdapterDownloadControl._regist_schedule_flag:
            return
        MispAdapterDownloadControl._regist_schedule_flag = True
        self._schedule = CtirsScheduler()
        # schedulerを起動
        # 起動時に1回のみ動作
        # 各job設定ごとに
        misp = None
        try:
            misp = MispAdapter.objects.get()
            for job in misp.jobs:
                # add_jobするとstatusが変わるためその前に保持
                status = job.status
                self.add_job(job)
                if status == ScheduleJobs.STATUS_IN_OPERATION:
                    self.resume_job(job.id)
                else:
                    self.pause_job(job.id)
        except DoesNotExist:
            # OtxAdapterが存在しない場合はjob追加を行わない
            pass
        # interval の設定があった場合は追加する
        if misp is not None:
            self._interval_job = misp.interval_schedule_job
        else:
            self._interval_job = None
        if self._interval_job is not None:
            if misp.interval_schedule_job.status == ScheduleJobs.STATUS_IN_OPERATION:
                self.add_job(misp.interval_schedule_job)

    # misp から from_dt から to_dt までのデータを取得する
    def get_misp_stix(self,
                      from_dt=None,
                      to_dt=None,
                      identity=default_identity_name):
        # identity を更新
        self.mc.identity_name = identity
        # misp アダプタの設定を取得
        misp_conf = MispAdapter.get()
        url = misp_conf.url
        stix_id_prefix = misp_conf.stix_id_prefix
        apikey = misp_conf.apikey
        published_only = misp_conf.published_only
        # 登録情報を取得
        community = misp_conf.community
        uploader = misp_conf.uploader
        via = Vias.get_via_adapter_misp(uploader)

        # mispから取得
        try:
            if url[-1] != '/':
                url += '/'
            url = url + 'events/xml/download.json'
            md = MISPDownloader(url, apikey)
            text = md.get(from_dt=from_dt, to_dt=to_dt)
            if text is None:
                return 0
            stix_packages = self.mc.convert(text=text.encode(),
                                            published_only=published_only,
                                            stix_id_prefix=stix_id_prefix)
        except Exception as e:
            traceback.print_exc()
            raise e

        # last_requested更新
        misp_conf.modify_last_requested()

        count = 0
        # ひとつずつ取得する
        for stix_package in stix_packages:
            try:
                # stix一つごとに登録処理
                # 取得したSTIXを登録
                try:
                    StixFiles.objects.get(package_id=stix_package.id_)
                except DoesNotExist:
                    # 存在しない場合は登録する
                    _regist_stix(stix_package.to_xml(), community, via)
                    count += 1
            except Exception as e:
                # エラーが発生した場合はログを表示して処理は実行する
                traceback.print_exc()

        # 件数を返却
        return count

    # job起動用 misp から last_requested 以降のデータを取得する
    def _get_misp_stix_job(self):
        misp = MispAdapter.objects.get()
        start_time_dt = misp.last_requested
        print(start_time_dt)
        self.get_misp_stix(from_dt=start_time_dt)

    # add job
    def add_job(self, schedule_job):
        # スケジューラに job 追加
        self._schedule.add_job(schedule_job, self._get_misp_stix_job)
        # job start
        self._schedule.start_job(schedule_job)
        # mongo の設定に格納する
        misp = MispAdapter.objects.get()
        if schedule_job.job_type == ScheduleJobs.JOB_INTERVAL:
            # Interval指定の場合は_interval_jobを更新する
            misp.interval_schedule_job = schedule_job
            self._interval_job = schedule_job
        else:
            misp.jobs.append(schedule_job)

    # resume job
    def resume_job(self, job_id):
        schedule_job = ScheduleJobs.objects.get(id=job_id)
        if schedule_job in MispAdapter.get().jobs:
            if schedule_job.status == ScheduleJobs.STATUS_STOP:
                pass
            else:
                print('already working.')
                return
        else:
            raise Exception('invalid job_id')
        self._schedule.resume_job(schedule_job)

    # stop job
    def pause_job(self, job_id):
        schedule_job = ScheduleJobs.objects.get(id=job_id)
        if schedule_job in MispAdapter.get().jobs:
            if schedule_job.status == ScheduleJobs.STATUS_IN_OPERATION:
                pass
            else:
                print('not yet start.')
                return
        else:
            raise Exception('invalid job_id')

            return
        self._schedule.pause_job(schedule_job)

    # remove_job job
    def remove_job(self, job_id):
        # MispAdapter の jobs から job 削除
        misp = MispAdapter.get()
        misp.remove_job(job_id)
        # スケジューラから job 削除
        schedule_job = ScheduleJobs.objects.get(id=job_id)
        self._schedule.remove_job(schedule_job)
        # mongo の schedule_jobs から schedule_job 削除
        schedule_job.remove()

    def remove_interval_job(self):
        if self._interval_job is None:
            # まだ設定されていない
            return
        schedule_job = ScheduleJobs.objects.get(id=self._interval_job.id)
        # スケジューラからjob削除
        self._schedule.remove_job(schedule_job)
        self._interval_job = None
Esempio n. 7
0
class Client(object):
    def __init__(self, community=None, taxii_client=None, taxii2_client=None):
        if taxii2_client:
            taxii = taxii2_client
        elif taxii_client:
            taxii = taxii_client

        self._name = taxii.name
        self._protocol_version = taxii.protocol_version
        self._username = taxii.login_id
        self._password = taxii.login_password
        self._jobs = taxii.jobs
        self._interval_job = taxii.interval_schedule_job
        self._can_read = taxii.can_read
        self._can_write = taxii.can_write
        if taxii.is_use_cert:
            self._auth_type = clients.HttpClient.AUTH_CERT_BASIC
            self._key_file = taxii.key_file
            self._cert_file = taxii.cert_file
        else:
            self._auth_type = clients.HttpClient.AUTH_BASIC
        if taxii2_client:
            self._api_root = taxii.api_root
            self._collection = taxii.collection
            self._via = Vias.get_via_taxii_poll(taxii2_client=taxii,
                                                uploader=taxii.uploader)
        else:
            self._address = taxii.address
            self._port = taxii.port
            self._path = taxii.path
            self._collection_name = taxii.collection
            self._via = Vias.get_via_taxii_poll(taxii_client=taxii,
                                                uploader=taxii.uploader)
            self._client = clients.HttpClient()

            self._ssl = taxii.ssl
            self._client.set_use_https(self._ssl)
            self._client.set_auth_type(self._auth_type)

        self._proxies = System.get_request_proxies()
        if self._proxies is not None:
            p = urlparse(self._address)
            if p.scheme == 'https':
                self._client.set_proxy(self._proxies['https'])
            else:
                self._client.set_proxy(self._proxies['http'])

        try:
            self._community = taxii.community
        except BaseException:
            self._community = None
        self._taxii = taxii
        self._start = None
        self._end = None
        self._schedule = CtirsScheduler()

    def set_start_time(self, start):
        self._start = start

    def set_end_time(self, end):
        self._end = end

    def add_job(self, schedule_job):
        self._schedule.add_job(schedule_job, self.poll_job)
        self._schedule.start_job(schedule_job)

    def resume_job(self, job_id):
        schedule_job = ScheduleJobs.objects.get(id=job_id)
        index = -1
        if self._interval_job == schedule_job:
            if schedule_job.status == ScheduleJobs.STATUS_IN_OPERATION:
                return
        if schedule_job in self._jobs:
            index = self._jobs.index(schedule_job)
            if schedule_job.status == ScheduleJobs.STATUS_IN_OPERATION:
                return
        else:
            raise Exception('invalid job_id')
        self._schedule.resume_job(schedule_job)
        if index > -1:
            self._jobs[index] = schedule_job

    def pause_job(self, job_id):
        schedule_job = ScheduleJobs.objects.get(id=job_id)
        index = -1
        if self._interval_job == schedule_job:
            if schedule_job.status == ScheduleJobs.STATUS_STOP:
                return
        if schedule_job in self._jobs:
            index = self._jobs.index(schedule_job)
            if schedule_job.status == ScheduleJobs.STATUS_STOP:
                return
        else:
            raise Exception('invalid job_id')
        self._schedule.pause_job(schedule_job)
        if index > -1:
            self._jobs[index] = schedule_job

    def remove_job(self, job_id):
        schedule_job = ScheduleJobs.objects.get(id=job_id)
        self._schedule.remove_job(schedule_job)
        schedule_job.remove()

    def remove_interval_job(self):
        if self._interval_job is None:
            return
        schedule_job = ScheduleJobs.objects.get(id=self._interval_job.id)
        self._schedule.remove_job(schedule_job)
        schedule_job.remove()
        self._interval_job = None

    def poll_job(self):
        if self._taxii.last_requested is not None:
            self.set_start_time(
                self._taxii.last_requested.replace(tzinfo=pytz.utc))
        try:
            self.poll()
        except BaseException:
            traceback.print_exc()

    def poll(self):
        if not self._can_read:
            print('This collection is not for polling/consuming.: %s ' %
                  (self._name))
            return 0
        if self._protocol_version == '2.0':
            return poll_20(self, protocol_version='2.0')
        elif self._protocol_version == '2.1':
            return poll_20(self, protocol_version='2.1')
        else:
            return poll_11(self)

    def push(self, stix_file_doc):
        if not self._can_write:
            msg = 'This collection is not for inbox/publishing: %s ' % (
                self._name)
            print(msg)
            return msg
        if self._protocol_version == '2.0':
            return push_20(self, stix_file_doc, protocol_version='2.0')
        elif self._protocol_version == '2.1':
            return push_20(self, stix_file_doc, protocol_version='2.1')
        else:
            return push_11(self, stix_file_doc)