Beispiel #1
0
def runSearchGitlabSnippets():
  try:
    logger.info('--START GITLAB SNIPPETS SEARCH--')
    now = datetime.date.today()
    today = now.strftime('%Y-%m-%d')

    target = 'gitlab_snippet'
    keywords = ec.getEnableKeywords(target)

    if ec.isEnable(target) and keywords != None and keywords != []:
      safe_limit = 6
      error_safety = ec.getSafetyCount(target)
      for key in keywords:
        channel = key['Channel']
        limittime = datetime.datetime.strptime(key['Expire_date'], '%Y-%m-%d').date()
        if now > limittime:
          postdata = '`' + key['KEY'] + '` is expired in _gitlab_snippet_, and disabled.'
          master.postAnyData(postdata, channel)
          ec.enableKeywordSetting('gitlab_snippet', key['Index'], False)
          logger.info(postdata)
      keywords = ec.getEnableKeywords(target)
      if keywords != None and keywords != []:
        keylist = [d.get('KEY') for d in keywords]
        (results, statuscode) = search_api.searchGitlabSnippets(keylist)
        if statuscode != 200:
          error_safety += 1
          ec.setSafetyCount(target, error_safety)
          postdata = '_gitlab_snippet_ failed to search.\nStatus Code: ' + str(statuscode)
          master.postAnyData(postdata, channel)
          logger.info(postdata)
          if error_safety > safe_limit:
            postdata = 'Too Many Errors. _Gitlab Snippet_ Module is disabled for safety'
            ec.disable(target)
            master.postAnyData(postdata, channel)
            logger.info(postdata)
        else:
          ec.setSafetyCount(target, 0)
          for key in keywords:
            if key['KEY'] in results.keys():
              result = list(set(results[key['KEY']]) - set(key['Exclude_list']))
              if result != []:
                channel = key['Channel']
                postdata = 'New Code Found about `' + key['KEY'] + '` in _gitlab_snippet_'
                master.postAnyData(postdata, channel)
                logger.info(postdata)
                url = []
                for i in result:
                  url.append('https://gitlab.com' + i)
                  logger.info('https://gitlab.com' + i)
#                exclude = list(set(results[word]) & set(keywords[word][1]))
                exclude = results[key['KEY']]
                if channel in getSpecialChannel():
                  doSpecialAct(target, channel, key['KEY'], url)
                master.postAnyData('\n'.join(url), channel)
                ec.clearExcludeList(target, key['Index'])
                ec.addExcludeList(target, key['Index'], exclude)
  except:
    logger.error('--ERROR HAS OCCURED IN GITLAB SNIPPETS SEARCH--')
    logger.error(traceback.format_exc())
    master.postAnyData(traceback.format_exc(), slackbot_settings.channels[0])
Beispiel #2
0
def runSearchGoogleCustom():
  try:
    engine_id = slackbot_settings.google_custom_search_engine_id
    api_key = slackbot_settings.google_custom_api_key
    logger.info('--START GOOGLE CUSTOM SEARCH--')
    now = datetime.date.today()
    today = now.strftime('%Y-%m-%d')

    target = 'google_custom'
    keywords = ec.getEnableKeywords(target)

    if ec.isEnable(target) and keywords != None and keywords != []:
      safe_limit = 6
      error_safety = ec.getSafetyCount(target)
      for key in keywords:
        limittime = datetime.datetime.strptime(key['Expire_date'], '%Y-%m-%d').date()
        channel = key['Channel']
        if now < limittime:
          (result, statuscode) = search_api.googleCustomSearch(key['KEY'], engine_id, api_key)
          if statuscode != 200:
            error_safety += 1
            ec.setSafetyCount(target, error_safety)
            postdata = '`' + key['KEY'] + '` failed to search in _google_custom_.\nStatus Code: ' + str(statuscode)
            master.postAnyData(postdata, channel)
            logger.info(postdata)
            if error_safety > safe_limit:
              postdata = 'Too Many Errors. _Google Custom_ Module is disabled for safety'
              ec.disable(target)
              master.postAnyData(postdata, channel)
              logger.info(postdata)
          else:
            result_post = list(set(result.keys()) - set(key['Exclude_list']))
            ec.setSafetyCount(target, 0)
            if key['__INITIAL__'] == True:
              ec.haveSearched(target, key['Index'])
            if result_post != []:
              postdata = 'New Code Found about `' + key['KEY'] + '` in _google_custom_'
              master.postAnyData(postdata, channel)
              logger.info(postdata)
              if key['__INITIAL__'] == True:
                result_post = result_post[:1]
              for i in result_post:
                logger.info(i)
                post_code = result[i][0] + '\n' + i + '\n'
                if channel in getSpecialChannel():
                  doSpecialAct(target, channel, key['KEY'], post_code)
                master.postAnyData(post_code, channel)
              exclude = list(result.keys())
#                  ec.clearExcludeList('google_custom', conf['Index'])
              ec.addExcludeList(target, key['Index'], exclude)
          time.sleep(30)
        else:
          postdata = '`' + key['KEY'] + '` is expired in _google_custom_, and disabled.'
          master.postAnyData(postdata, channel)
          ec.enableKeywordSetting(target, key['Index'], False)
          logger.info(postdata)
  except:
    logger.error('--ERROR HAS OCCURED IN GOOGLE CUSTOM SEARCH--')
    logger.error(traceback.format_exc())
    master.postAnyData(traceback.format_exc(), slackbot_settings.channels[0])
Beispiel #3
0
def runSearchGithubCode():
  try:
    logger.info('--START GITHUB CODE SEARCH--')
    now = datetime.date.today()
    today = now.strftime('%Y-%m-%d')

    api_key = slackbot_settings.github_access_token

    target = 'github_code'
    keywords = ec.getEnableKeywords(target)

    if ec.isEnable(target) and keywords != None and keywords != []:
      safe_limit = 6
      error_safety = ec.getSafetyCount(target)
      for key in keywords:
        channel = key['Channel']
        limittime = datetime.datetime.strptime(key['Expire_date'], '%Y-%m-%d').date()
        if now < limittime:
          (results, statuscode) = search_api.searchGithubCode(key['KEY'], key['SearchLevel'], api_key)
          result = list(set(results) - set(key['Exclude_list']))
          if statuscode != 200:
            error_safety += 1
            ec.setSafetyCount('github_code', error_safety)
            postdata = '`' + key['KEY'] + '` failed to search in _github_code_.\nStatus Code: ' + str(statuscode)
            master.postAnyData(postdata, channel)
            logger.info(postdata)
            if error_safety > safe_limit:
              postdata = 'Too Many Errors. _Github Code_ Module is disabled for safety'
              ec.disable('github_code')
              master.postAnyData(postdata, channel)
              logger.info(postdata)
          else:
            ec.setSafetyCount('github_code', 0)
            if key['__INITIAL__'] == True:
              ec.haveSearched(target, key['Index'])
            if result != []:
              postdata = 'New Code Found about `' + key['KEY'] + '` in _github_code_'
              master.postAnyData(postdata, channel)
              if key['__INITIAL__'] == True:
                master.postAnyData(result[0], channel)
              else:
                if channel in getSpecialChannel():
                  doSpecialAct(target, channel, key['KEY'], result)
                master.postAnyData('\n'.join(result), channel)
              logger.info('keyword : ' + key['KEY'])
              logger.info('\n'.join(result))
              exclude = results
#              ec.clearExcludeList('github_code', conf['Index'])
              ec.addExcludeList('github_code', key['Index'], exclude)
          time.sleep(10)
        else:
          postdata = '`' + key['KEY'] + '` expired in _github_code_, and was disabled.'
          logger.info(postdata)
          master.postAnyData(postdata, channel)
          ec.enableKeywordSetting('github_code', key['Index'], False)
  except:
    logger.error('--ERROR HAS OCCURED IN GITHUB SEARCH--')
    logger.error(traceback.format_exc())
    master.postAnyData(traceback.format_exc(), slackbot_settings.channels[0])
Beispiel #4
0
def runTwitterSearch():
    try:
        logger.info('--START TWITTER SEARCH--')

        target = 'twitter'
        keywords = ec.getEnableKeywords(target)

        if ec.isEnable(target) and keywords != None and keywords != []:
            safe_limit = 6
            error_safety = ec.getSafetyCount(target)

            for key in keywords:
                channel = key['Channel']
                query = key['Query']
                users = key['Users']
                lastpost = key['Last_Post']
                initialstate = key['__INITIAL__']
                (result,
                 statuscode) = search_api.getTweets(users, query, lastpost)
                if statuscode != 200:
                    error_safety += 1
                    ec.setSafetyCount(target, error_safety)
                    postdata = '`' + key[
                        'Name'] + '` failed to get _Twitter_.\nStatus Code: ' + str(
                            statuscode)
                    master.postAnyData(postdata, channel)
                    logger.info(postdata)
                    if error_safety > safe_limit:
                        postdata = 'Too Many Errors. _Twitter_ Module is disabled for safety'
                        ec.disable(target)
                        master.postAnyData(postdata, channel)
                        logger.info(postdata)
                else:
                    if error_safety != 0:
                        ec.setSafetyCount(target, 0)
                    if len(result) > 0:
                        if initialstate:
                            result = result[:1]
                            ec.haveSearched(target, key['Index'])
                        lastpost = result[0]
                        ec.setTwitterLastPost(key['Index'], lastpost)
                        postdata = 'New Tweets in `' + key['KEY'] + '`'
                        master.postAnyData(postdata, channel)
                        logger.info(postdata)
                        postdata = ''
                        for tw in result:
                            postdata = 'https://twitter.com' + tw['link']
                            postdata += ' (FROM: ' + tw['user'] + ')\n'
                            postdata += '>>>' + tw['tweet'] + '\n'
                            logger.info(postdata)
                            if channel in getSpecialChannel():
                                doSpecialAct(target, channel, key['KEY'], tw)
                            master.postAnyData(postdata, channel)
                    time.sleep(30)
    except:
        logger.error('--ERROR HAS OCCURED IN SEARCHING TWITTER--')
        logger.error(traceback.format_exc())
        master.postAnyData(traceback.format_exc(),
                           slackbot_settings.channels[0])
Beispiel #5
0
def runSearchGithub():
    try:
        logger.info('--START GITHUB SEARCH--')
        now = datetime.date.today()
        today = now.strftime('%Y-%m-%d')

        target = 'github'
        keywords = ec.getEnableKeywords(target)

        if ec.isEnable(target) and keywords != None and keywords != []:
            safe_limit = 6
            error_safety = ec.getSafetyCount(target)
            for key in keywords:
                channel = key['Channel']
                limittime = datetime.datetime.strptime(key['Expire_date'],
                                                       '%Y-%m-%d').date()
                if now < limittime:
                    oldtime = now - datetime.timedelta(key['Time_Range'])
                    oldday = oldtime.strftime('%Y-%m-%d')
                    (results, statuscode) = search_api.searchGithub(
                        key['KEY'], oldday, key['SearchLevel'])
                    result = list(set(results) - set(key['Exclude_list']))
                    if statuscode != 200:
                        error_safety += 1
                        ec.setSafetyCount(target, error_safety)
                        postdata = '`' + key[
                            'KEY'] + '` failed to search in _github_.\nStatus Code: ' + str(
                                statuscode)
                        master.postAnyData(postdata, channel)
                        logger.info(postdata)
                        if error_safety > safe_limit:
                            postdata = 'Too Many Errors. _Github_ Module is disabled for safety'
                            ec.disable(target)
                            master.postAnyData(postdata, channel)
                            logger.info(postdata)
                    else:
                        ec.setSafetyCount(target, 0)
                        if result != []:
                            if channel in getSpecialChannel():
                                doSpecialAct(target, channel, key['KEY'],
                                             result)
                            master.postNewPoCFound(key['KEY'], result, channel)
                            logger.info('keyword : ' + key['KEY'])
                            logger.info('\n'.join(result))
                            exclude = results
                            ec.clearExcludeList(target, key['Index'])
                            ec.addExcludeList(target, key['Index'], exclude)
                    time.sleep(10)
                else:
                    postdata = '`' + key[
                        'Index'] + '` expired in _github_, and was disabled.'
                    logger.info(postdata)
                    master.postAnyData(postdata, channel)
                    ec.enableKeywordSetting(target, key['Index'], False)
    except:
        logger.error('--ERROR HAS OCCURED IN GITHUB SEARCH--')
        logger.error(traceback.format_exc())
        master.postAnyData(traceback.format_exc(),
                           slackbot_settings.channels[0])
Beispiel #6
0
def runRSSFeeds():
  try:
    logger.info('--GET NEW RSS FEEDS--')

    target = 'rss_feed'
    keywords = ec.getEnableKeywords(target)

    if ec.isEnable(target) and keywords != None and keywords != []:
      safe_limit = 6
      error_safety = ec.getSafetyCount(target)

      for key in keywords:
        channel = key['Channel']
        filter = key['Filters']
        url = key['URL']
        lastpost = key['Last_Post']
        initialstate = key['__INITIAL__']
        (result, statuscode) = search_api.getRSSFeeds(url, lastpost)
        if statuscode != 200:
            error_safety += 1
            ec.setSafetyCount(target, error_safety)
            postdata = '`' + key['Name'] + '` failed to get _RSS_Feeds_.\nStatus Code: ' + str(statuscode)
            master.postAnyData(postdata, channel)
            logger.info(postdata)
            if error_safety > safe_limit:
              postdata = 'Too Many Errors. _RSS_Feeds_ Module is disabled for safety'
              ec.disable(target)
              master.postAnyData(postdata, channel)
              logger.info(postdata)
        else:
          if error_safety != 0:
            ec.setSafetyCount(target, 0)
          if len(result) > 0:
            if initialstate:
              result = result[:1]
              ec.haveSearched(target, key['Name'])
            filteredfeeds = {}
            if filter != []:
              for f in filter:
                c = f['Channel']
                w = f['Words']
                ff = filterFeeds(result, w)
                if ff != []:
                  if c in filteredfeeds.keys():
                    filteredfeeds[c] += ff
                  else:
                    filteredfeeds[c] = ff
            else:
              if result != {}:
                filteredfeeds[channel] = result
            lastpost = {'title':result[0]['title'], 'link':result[0]['link'], 'timestamp':result[0]['timestamp']}
            ec.setRSSLastPost(key['Name'], lastpost)
            if filteredfeeds != {}:
              for c, feeds in filteredfeeds.items():
                if c in getSpecialChannel():
                  doSpecialAct(target, c, key['Name'], feeds)
                postdata = 'New Feed in `' + key['Name'] + '`'
                master.postAnyData(postdata, c)
                logger.info(postdata)
                postdata = ''
                for f in feeds:
                  postdata = f['title'] + '\n'
                  postdata += f['link']
                  logger.info(postdata)
                  master.postAnyData(postdata, c)
          time.sleep(30)
  except:
    logger.error('--ERROR HAS OCCURED IN GETTING RSS FEEDS--')
    logger.error(traceback.format_exc())
    master.postAnyData(traceback.format_exc(), slackbot_settings.channels[0])
Beispiel #7
0
def runSearchPastebin():
  logger.info('--START PASTEBIN SEARCH--')
  while True:
    try:
      now = datetime.date.today()
      today = now.strftime('%Y-%m-%d')
      target = 'pastebin'
      keywords = ec.getEnableKeywords(target)

      if ec.isEnable(target) and keywords != None and keywords != []:
        safe_limit = 10
        error_safety = ec.getSafetyCount(target)
        for key in keywords:
          channel = key['Channel']
          limittime = datetime.datetime.strptime(key['Expire_date'], '%Y-%m-%d').date()
          if now > limittime:
            postdata = '`' + key['KEY'] + '` is expired in _pastebin_, and disabled.'
            master.postAnyData(postdata, channel)
            ec.enableKeywordSetting(target, key['Index'], False)
            logger.info(postdata)
        keywords = ec.getEnableKeywords(target)
        if keywords != None and keywords != []:
          (pastelist, statuscode) = search_api.getPasteList(100)
          if statuscode != 200:
            error_safety += 1
            ec.setSafetyCount(target, error_safety)
            postdata = 'pastebin serach failed in _pastebin_.\nStatus Code: ' + str(statuscode)
            master.postAnyData(postdata, channel)
            logger.info(postdata)
            if error_safety == 5:
              postdata = 'Pause to access pastebin'
              master.postAnyData(postdata, channel)
              logger.info(postdata)
              time.sleep(300)
            if error_safety > safe_limit:
              postdata = 'Too Many Errors. _Pastebin_ Module is disabled for safety'
              ec.disable('pastebin')
              master.postAnyData(postdata, channel)
              logger.info(postdata)
          else:
            searchedpastes = ec.getSearchedPastes()
            searchlist = {}
            for paste, conf in pastelist.items():
              if not paste in searchedpastes:
                searchlist[paste] = conf
            if len(searchlist.keys()) > 30:
              ec.setSearchedPastes(pastelist.keys())
              logger.info('The number of scraping pastes is ' + str(len(searchlist.keys())))
              keylist = [d.get('KEY') for d in keywords]
              (results, statuscode) = search_api.scrapePastebin(keylist, searchlist)
              if statuscode != 200:
                error_safety += 1
                ec.setSafetyCount(target, error_safety)
                postdata = 'pastebin serach failed in _pastebin_.\nStatus Code: ' + str(statuscode)
                master.postAnyData(postdata, channel)
                logger.info(postdata)
                if error_safety > safe_limit:
                  postdata = 'Too Many Errors. _Pastebin_ Module is disabled for safety'
                  ec.disable(target)
                  master.postAnyData(postdata, channel)
                  logger.info(postdata)
              else:
                ec.setSafetyCount(target, 0)
                for key in keywords:
                  if key['KEY'] in results.keys():
                    if results[key['KEY']] != []:
                      channel = key['Channel']
                      postdata = 'New Code Found about `' + key['KEY'] + '` in _pastebin_'
                      if channel in getSpecialChannel():
                        doSpecialAct(target, channel, key['KEY'], results[key['KEY']])
                      master.postAnyData(postdata, channel)
                      logger.info(postdata)
                      exclude = results[key['KEY']]
                      master.postAnyData('\n'.join(results[key['KEY']]), channel)
      time.sleep(10)
    except:
      logger.error('--ERROR HAS OCCURED IN PASTEBIN SEARCH--')
      logger.error(traceback.format_exc())
      master.postAnyData(traceback.format_exc(), slackbot_settings.channels[0])
      time.sleep(10)
Beispiel #8
0
def runSearchGitlab():
  try:
    logger.info('--START GITLAB SEARCH--')
    now = datetime.date.today()
    today = now.strftime('%Y-%m-%d')

    target = 'gitlab'
    keywords = ec.getEnableKeywords(target)

    if ec.isEnable(target) and keywords != None and keywords != []:
      safe_limit = 6
      error_safety = ec.getSafetyCount(target)
      for key in keywords:
        channel = key['Channel']
        limittime = datetime.datetime.strptime(key['Expire_date'], '%Y-%m-%d').date()
        if now < limittime:
          (results, statuscode) = search_api.searchGitlab(key['KEY'])
          result = list(set(results) - set(key['Exclude_list']))
          if statuscode != 200:
            error_safety += 1
            ec.setSafetyCount(target, error_safety)
            postdata = '`' + key['KEY'] + '` failed to search in _gitlab_.\nStatus Code: ' + str(statuscode)
            master.postAnyData(postdata, channel)
            logger.info(postdata)
            if error_safety > safe_limit:
              postdata = 'Too Many Errors. _Gitlab_ Module is disabled for safety'
              ec.disable(target)
              master.postAnyData(postdata, channel)
              logger.info(postdata)
          else:
            if error_safety != 0:
              ec.setSafetyCount(target, 0)
            if key['__INITIAL__'] == True:
              ec.haveSearched(target, key['Index'])
            if result != []:
              postdata = 'New Code Found about `' + key['KEY'] + '` in _gitlab_'
              master.postAnyData(postdata, channel)
              url = []
              for i in result:
                url.append('https://gitlab.com' + i)
              if key['__INITIAL__'] == True:
                master.postAnyData(url[0], channel)
              else:
                if channel in getSpecialChannel():
                  doSpecialAct(target, channel, key['KEY'], url)
                master.postAnyData('\n'.join(url), channel)
              logger.info('keyword : ' + key['KEY'])
              logger.info('\n'.join(url))
              exclude = results
              ec.clearExcludeList(target, key['Index'])
              ec.addExcludeList(target, key['Index'], exclude)
          time.sleep(30)
        else:
          postdata = '`' + key['KEY'] + '` is expired in _gitlab_, and disabled.'
          master.postAnyData(postdata, channel)
          ec.enableKeywordSetting(target, key['Index'], False)
          logger.info(postdata)
  except:
    logger.error('--ERROR HAS OCCURED IN GITLAB SEARCH--')
    logger.error(traceback.format_exc())
    master.postAnyData(traceback.format_exc(), slackbot_settings.channels[0])