def runSearchGitlabSnippets(): try: logger.info('--START GITLAB SNIPPETS SEARCH--') now = datetime.date.today() today = now.strftime('%Y-%m-%d') target = 'gitlab_snippet' keywords = ec.getEnableKeywords(target) if ec.isEnable(target) and keywords != None and keywords != []: safe_limit = 6 error_safety = ec.getSafetyCount(target) for key in keywords: channel = key['Channel'] limittime = datetime.datetime.strptime(key['Expire_date'], '%Y-%m-%d').date() if now > limittime: postdata = '`' + key['KEY'] + '` is expired in _gitlab_snippet_, and disabled.' master.postAnyData(postdata, channel) ec.enableKeywordSetting('gitlab_snippet', key['Index'], False) logger.info(postdata) keywords = ec.getEnableKeywords(target) if keywords != None and keywords != []: keylist = [d.get('KEY') for d in keywords] (results, statuscode) = search_api.searchGitlabSnippets(keylist) if statuscode != 200: error_safety += 1 ec.setSafetyCount(target, error_safety) postdata = '_gitlab_snippet_ failed to search.\nStatus Code: ' + str(statuscode) master.postAnyData(postdata, channel) logger.info(postdata) if error_safety > safe_limit: postdata = 'Too Many Errors. _Gitlab Snippet_ Module is disabled for safety' ec.disable(target) master.postAnyData(postdata, channel) logger.info(postdata) else: ec.setSafetyCount(target, 0) for key in keywords: if key['KEY'] in results.keys(): result = list(set(results[key['KEY']]) - set(key['Exclude_list'])) if result != []: channel = key['Channel'] postdata = 'New Code Found about `' + key['KEY'] + '` in _gitlab_snippet_' master.postAnyData(postdata, channel) logger.info(postdata) url = [] for i in result: url.append('https://gitlab.com' + i) logger.info('https://gitlab.com' + i) # exclude = list(set(results[word]) & set(keywords[word][1])) exclude = results[key['KEY']] if channel in getSpecialChannel(): doSpecialAct(target, channel, key['KEY'], url) master.postAnyData('\n'.join(url), channel) ec.clearExcludeList(target, key['Index']) ec.addExcludeList(target, key['Index'], exclude) except: logger.error('--ERROR HAS OCCURED IN GITLAB SNIPPETS SEARCH--') logger.error(traceback.format_exc()) master.postAnyData(traceback.format_exc(), slackbot_settings.channels[0])
def runSearchGoogleCustom(): try: engine_id = slackbot_settings.google_custom_search_engine_id api_key = slackbot_settings.google_custom_api_key logger.info('--START GOOGLE CUSTOM SEARCH--') now = datetime.date.today() today = now.strftime('%Y-%m-%d') target = 'google_custom' keywords = ec.getEnableKeywords(target) if ec.isEnable(target) and keywords != None and keywords != []: safe_limit = 6 error_safety = ec.getSafetyCount(target) for key in keywords: limittime = datetime.datetime.strptime(key['Expire_date'], '%Y-%m-%d').date() channel = key['Channel'] if now < limittime: (result, statuscode) = search_api.googleCustomSearch(key['KEY'], engine_id, api_key) if statuscode != 200: error_safety += 1 ec.setSafetyCount(target, error_safety) postdata = '`' + key['KEY'] + '` failed to search in _google_custom_.\nStatus Code: ' + str(statuscode) master.postAnyData(postdata, channel) logger.info(postdata) if error_safety > safe_limit: postdata = 'Too Many Errors. _Google Custom_ Module is disabled for safety' ec.disable(target) master.postAnyData(postdata, channel) logger.info(postdata) else: result_post = list(set(result.keys()) - set(key['Exclude_list'])) ec.setSafetyCount(target, 0) if key['__INITIAL__'] == True: ec.haveSearched(target, key['Index']) if result_post != []: postdata = 'New Code Found about `' + key['KEY'] + '` in _google_custom_' master.postAnyData(postdata, channel) logger.info(postdata) if key['__INITIAL__'] == True: result_post = result_post[:1] for i in result_post: logger.info(i) post_code = result[i][0] + '\n' + i + '\n' if channel in getSpecialChannel(): doSpecialAct(target, channel, key['KEY'], post_code) master.postAnyData(post_code, channel) exclude = list(result.keys()) # ec.clearExcludeList('google_custom', conf['Index']) ec.addExcludeList(target, key['Index'], exclude) time.sleep(30) else: postdata = '`' + key['KEY'] + '` is expired in _google_custom_, and disabled.' master.postAnyData(postdata, channel) ec.enableKeywordSetting(target, key['Index'], False) logger.info(postdata) except: logger.error('--ERROR HAS OCCURED IN GOOGLE CUSTOM SEARCH--') logger.error(traceback.format_exc()) master.postAnyData(traceback.format_exc(), slackbot_settings.channels[0])
def runSearchGithubCode(): try: logger.info('--START GITHUB CODE SEARCH--') now = datetime.date.today() today = now.strftime('%Y-%m-%d') api_key = slackbot_settings.github_access_token target = 'github_code' keywords = ec.getEnableKeywords(target) if ec.isEnable(target) and keywords != None and keywords != []: safe_limit = 6 error_safety = ec.getSafetyCount(target) for key in keywords: channel = key['Channel'] limittime = datetime.datetime.strptime(key['Expire_date'], '%Y-%m-%d').date() if now < limittime: (results, statuscode) = search_api.searchGithubCode(key['KEY'], key['SearchLevel'], api_key) result = list(set(results) - set(key['Exclude_list'])) if statuscode != 200: error_safety += 1 ec.setSafetyCount('github_code', error_safety) postdata = '`' + key['KEY'] + '` failed to search in _github_code_.\nStatus Code: ' + str(statuscode) master.postAnyData(postdata, channel) logger.info(postdata) if error_safety > safe_limit: postdata = 'Too Many Errors. _Github Code_ Module is disabled for safety' ec.disable('github_code') master.postAnyData(postdata, channel) logger.info(postdata) else: ec.setSafetyCount('github_code', 0) if key['__INITIAL__'] == True: ec.haveSearched(target, key['Index']) if result != []: postdata = 'New Code Found about `' + key['KEY'] + '` in _github_code_' master.postAnyData(postdata, channel) if key['__INITIAL__'] == True: master.postAnyData(result[0], channel) else: if channel in getSpecialChannel(): doSpecialAct(target, channel, key['KEY'], result) master.postAnyData('\n'.join(result), channel) logger.info('keyword : ' + key['KEY']) logger.info('\n'.join(result)) exclude = results # ec.clearExcludeList('github_code', conf['Index']) ec.addExcludeList('github_code', key['Index'], exclude) time.sleep(10) else: postdata = '`' + key['KEY'] + '` expired in _github_code_, and was disabled.' logger.info(postdata) master.postAnyData(postdata, channel) ec.enableKeywordSetting('github_code', key['Index'], False) except: logger.error('--ERROR HAS OCCURED IN GITHUB SEARCH--') logger.error(traceback.format_exc()) master.postAnyData(traceback.format_exc(), slackbot_settings.channels[0])
def runTwitterSearch(): try: logger.info('--START TWITTER SEARCH--') target = 'twitter' keywords = ec.getEnableKeywords(target) if ec.isEnable(target) and keywords != None and keywords != []: safe_limit = 6 error_safety = ec.getSafetyCount(target) for key in keywords: channel = key['Channel'] query = key['Query'] users = key['Users'] lastpost = key['Last_Post'] initialstate = key['__INITIAL__'] (result, statuscode) = search_api.getTweets(users, query, lastpost) if statuscode != 200: error_safety += 1 ec.setSafetyCount(target, error_safety) postdata = '`' + key[ 'Name'] + '` failed to get _Twitter_.\nStatus Code: ' + str( statuscode) master.postAnyData(postdata, channel) logger.info(postdata) if error_safety > safe_limit: postdata = 'Too Many Errors. _Twitter_ Module is disabled for safety' ec.disable(target) master.postAnyData(postdata, channel) logger.info(postdata) else: if error_safety != 0: ec.setSafetyCount(target, 0) if len(result) > 0: if initialstate: result = result[:1] ec.haveSearched(target, key['Index']) lastpost = result[0] ec.setTwitterLastPost(key['Index'], lastpost) postdata = 'New Tweets in `' + key['KEY'] + '`' master.postAnyData(postdata, channel) logger.info(postdata) postdata = '' for tw in result: postdata = 'https://twitter.com' + tw['link'] postdata += ' (FROM: ' + tw['user'] + ')\n' postdata += '>>>' + tw['tweet'] + '\n' logger.info(postdata) if channel in getSpecialChannel(): doSpecialAct(target, channel, key['KEY'], tw) master.postAnyData(postdata, channel) time.sleep(30) except: logger.error('--ERROR HAS OCCURED IN SEARCHING TWITTER--') logger.error(traceback.format_exc()) master.postAnyData(traceback.format_exc(), slackbot_settings.channels[0])
def runSearchGithub(): try: logger.info('--START GITHUB SEARCH--') now = datetime.date.today() today = now.strftime('%Y-%m-%d') target = 'github' keywords = ec.getEnableKeywords(target) if ec.isEnable(target) and keywords != None and keywords != []: safe_limit = 6 error_safety = ec.getSafetyCount(target) for key in keywords: channel = key['Channel'] limittime = datetime.datetime.strptime(key['Expire_date'], '%Y-%m-%d').date() if now < limittime: oldtime = now - datetime.timedelta(key['Time_Range']) oldday = oldtime.strftime('%Y-%m-%d') (results, statuscode) = search_api.searchGithub( key['KEY'], oldday, key['SearchLevel']) result = list(set(results) - set(key['Exclude_list'])) if statuscode != 200: error_safety += 1 ec.setSafetyCount(target, error_safety) postdata = '`' + key[ 'KEY'] + '` failed to search in _github_.\nStatus Code: ' + str( statuscode) master.postAnyData(postdata, channel) logger.info(postdata) if error_safety > safe_limit: postdata = 'Too Many Errors. _Github_ Module is disabled for safety' ec.disable(target) master.postAnyData(postdata, channel) logger.info(postdata) else: ec.setSafetyCount(target, 0) if result != []: if channel in getSpecialChannel(): doSpecialAct(target, channel, key['KEY'], result) master.postNewPoCFound(key['KEY'], result, channel) logger.info('keyword : ' + key['KEY']) logger.info('\n'.join(result)) exclude = results ec.clearExcludeList(target, key['Index']) ec.addExcludeList(target, key['Index'], exclude) time.sleep(10) else: postdata = '`' + key[ 'Index'] + '` expired in _github_, and was disabled.' logger.info(postdata) master.postAnyData(postdata, channel) ec.enableKeywordSetting(target, key['Index'], False) except: logger.error('--ERROR HAS OCCURED IN GITHUB SEARCH--') logger.error(traceback.format_exc()) master.postAnyData(traceback.format_exc(), slackbot_settings.channels[0])
def runRSSFeeds(): try: logger.info('--GET NEW RSS FEEDS--') target = 'rss_feed' keywords = ec.getEnableKeywords(target) if ec.isEnable(target) and keywords != None and keywords != []: safe_limit = 6 error_safety = ec.getSafetyCount(target) for key in keywords: channel = key['Channel'] filter = key['Filters'] url = key['URL'] lastpost = key['Last_Post'] initialstate = key['__INITIAL__'] (result, statuscode) = search_api.getRSSFeeds(url, lastpost) if statuscode != 200: error_safety += 1 ec.setSafetyCount(target, error_safety) postdata = '`' + key['Name'] + '` failed to get _RSS_Feeds_.\nStatus Code: ' + str(statuscode) master.postAnyData(postdata, channel) logger.info(postdata) if error_safety > safe_limit: postdata = 'Too Many Errors. _RSS_Feeds_ Module is disabled for safety' ec.disable(target) master.postAnyData(postdata, channel) logger.info(postdata) else: if error_safety != 0: ec.setSafetyCount(target, 0) if len(result) > 0: if initialstate: result = result[:1] ec.haveSearched(target, key['Name']) filteredfeeds = {} if filter != []: for f in filter: c = f['Channel'] w = f['Words'] ff = filterFeeds(result, w) if ff != []: if c in filteredfeeds.keys(): filteredfeeds[c] += ff else: filteredfeeds[c] = ff else: if result != {}: filteredfeeds[channel] = result lastpost = {'title':result[0]['title'], 'link':result[0]['link'], 'timestamp':result[0]['timestamp']} ec.setRSSLastPost(key['Name'], lastpost) if filteredfeeds != {}: for c, feeds in filteredfeeds.items(): if c in getSpecialChannel(): doSpecialAct(target, c, key['Name'], feeds) postdata = 'New Feed in `' + key['Name'] + '`' master.postAnyData(postdata, c) logger.info(postdata) postdata = '' for f in feeds: postdata = f['title'] + '\n' postdata += f['link'] logger.info(postdata) master.postAnyData(postdata, c) time.sleep(30) except: logger.error('--ERROR HAS OCCURED IN GETTING RSS FEEDS--') logger.error(traceback.format_exc()) master.postAnyData(traceback.format_exc(), slackbot_settings.channels[0])
def runSearchPastebin(): logger.info('--START PASTEBIN SEARCH--') while True: try: now = datetime.date.today() today = now.strftime('%Y-%m-%d') target = 'pastebin' keywords = ec.getEnableKeywords(target) if ec.isEnable(target) and keywords != None and keywords != []: safe_limit = 10 error_safety = ec.getSafetyCount(target) for key in keywords: channel = key['Channel'] limittime = datetime.datetime.strptime(key['Expire_date'], '%Y-%m-%d').date() if now > limittime: postdata = '`' + key['KEY'] + '` is expired in _pastebin_, and disabled.' master.postAnyData(postdata, channel) ec.enableKeywordSetting(target, key['Index'], False) logger.info(postdata) keywords = ec.getEnableKeywords(target) if keywords != None and keywords != []: (pastelist, statuscode) = search_api.getPasteList(100) if statuscode != 200: error_safety += 1 ec.setSafetyCount(target, error_safety) postdata = 'pastebin serach failed in _pastebin_.\nStatus Code: ' + str(statuscode) master.postAnyData(postdata, channel) logger.info(postdata) if error_safety == 5: postdata = 'Pause to access pastebin' master.postAnyData(postdata, channel) logger.info(postdata) time.sleep(300) if error_safety > safe_limit: postdata = 'Too Many Errors. _Pastebin_ Module is disabled for safety' ec.disable('pastebin') master.postAnyData(postdata, channel) logger.info(postdata) else: searchedpastes = ec.getSearchedPastes() searchlist = {} for paste, conf in pastelist.items(): if not paste in searchedpastes: searchlist[paste] = conf if len(searchlist.keys()) > 30: ec.setSearchedPastes(pastelist.keys()) logger.info('The number of scraping pastes is ' + str(len(searchlist.keys()))) keylist = [d.get('KEY') for d in keywords] (results, statuscode) = search_api.scrapePastebin(keylist, searchlist) if statuscode != 200: error_safety += 1 ec.setSafetyCount(target, error_safety) postdata = 'pastebin serach failed in _pastebin_.\nStatus Code: ' + str(statuscode) master.postAnyData(postdata, channel) logger.info(postdata) if error_safety > safe_limit: postdata = 'Too Many Errors. _Pastebin_ Module is disabled for safety' ec.disable(target) master.postAnyData(postdata, channel) logger.info(postdata) else: ec.setSafetyCount(target, 0) for key in keywords: if key['KEY'] in results.keys(): if results[key['KEY']] != []: channel = key['Channel'] postdata = 'New Code Found about `' + key['KEY'] + '` in _pastebin_' if channel in getSpecialChannel(): doSpecialAct(target, channel, key['KEY'], results[key['KEY']]) master.postAnyData(postdata, channel) logger.info(postdata) exclude = results[key['KEY']] master.postAnyData('\n'.join(results[key['KEY']]), channel) time.sleep(10) except: logger.error('--ERROR HAS OCCURED IN PASTEBIN SEARCH--') logger.error(traceback.format_exc()) master.postAnyData(traceback.format_exc(), slackbot_settings.channels[0]) time.sleep(10)
def runSearchGitlab(): try: logger.info('--START GITLAB SEARCH--') now = datetime.date.today() today = now.strftime('%Y-%m-%d') target = 'gitlab' keywords = ec.getEnableKeywords(target) if ec.isEnable(target) and keywords != None and keywords != []: safe_limit = 6 error_safety = ec.getSafetyCount(target) for key in keywords: channel = key['Channel'] limittime = datetime.datetime.strptime(key['Expire_date'], '%Y-%m-%d').date() if now < limittime: (results, statuscode) = search_api.searchGitlab(key['KEY']) result = list(set(results) - set(key['Exclude_list'])) if statuscode != 200: error_safety += 1 ec.setSafetyCount(target, error_safety) postdata = '`' + key['KEY'] + '` failed to search in _gitlab_.\nStatus Code: ' + str(statuscode) master.postAnyData(postdata, channel) logger.info(postdata) if error_safety > safe_limit: postdata = 'Too Many Errors. _Gitlab_ Module is disabled for safety' ec.disable(target) master.postAnyData(postdata, channel) logger.info(postdata) else: if error_safety != 0: ec.setSafetyCount(target, 0) if key['__INITIAL__'] == True: ec.haveSearched(target, key['Index']) if result != []: postdata = 'New Code Found about `' + key['KEY'] + '` in _gitlab_' master.postAnyData(postdata, channel) url = [] for i in result: url.append('https://gitlab.com' + i) if key['__INITIAL__'] == True: master.postAnyData(url[0], channel) else: if channel in getSpecialChannel(): doSpecialAct(target, channel, key['KEY'], url) master.postAnyData('\n'.join(url), channel) logger.info('keyword : ' + key['KEY']) logger.info('\n'.join(url)) exclude = results ec.clearExcludeList(target, key['Index']) ec.addExcludeList(target, key['Index'], exclude) time.sleep(30) else: postdata = '`' + key['KEY'] + '` is expired in _gitlab_, and disabled.' master.postAnyData(postdata, channel) ec.enableKeywordSetting(target, key['Index'], False) logger.info(postdata) except: logger.error('--ERROR HAS OCCURED IN GITLAB SEARCH--') logger.error(traceback.format_exc()) master.postAnyData(traceback.format_exc(), slackbot_settings.channels[0])