def runSearchGoogleCustom():
try:
engine_id = slackbot_settings.google_custom_search_engine_id
api_key = slackbot_settings.google_custom_api_key
logger.info('--START GOOGLE CUSTOM SEARCH--')
now = datetime.date.today()
today = now.strftime('%Y-%m-%d')
target = 'google_custom'
keywords = ec.getEnableKeywords(target)
if ec.isEnable(target) and keywords != None and keywords != []:
safe_limit = 6
error_safety = ec.getSafetyCount(target)
for key in keywords:
limittime = datetime.datetime.strptime(key['Expire_date'], '%Y-%m-%d').date()
channel = key['Channel']
if now < limittime:
(result, statuscode) = search_api.googleCustomSearch(key['KEY'], engine_id, api_key)
if statuscode != 200:
error_safety += 1
ec.setSafetyCount(target, error_safety)
postdata = '`' + key['KEY'] + '` failed to search in _google_custom_.\nStatus Code: ' + str(statuscode)
master.postAnyData(postdata, channel)
logger.info(postdata)
if error_safety > safe_limit:
postdata = 'Too Many Errors. _Google Custom_ Module is disabled for safety'
ec.disable(target)
master.postAnyData(postdata, channel)
logger.info(postdata)
else:
result_post = list(set(result.keys()) - set(key['Exclude_list']))
ec.setSafetyCount(target, 0)
if key['__INITIAL__'] == True:
ec.haveSearched(target, key['Index'])
if result_post != []:
postdata = 'New Code Found about `' + key['KEY'] + '` in _google_custom_'
master.postAnyData(postdata, channel)
logger.info(postdata)
if key['__INITIAL__'] == True:
result_post = result_post[:1]
for i in result_post:
logger.info(i)
post_code = result[i][0] + '\n' + i + '\n'
if channel in getSpecialChannel():
doSpecialAct(target, channel, key['KEY'], post_code)
master.postAnyData(post_code, channel)
exclude = list(result.keys())
# ec.clearExcludeList('google_custom', conf['Index'])
ec.addExcludeList(target, key['Index'], exclude)
time.sleep(30)
else:
postdata = '`' + key['KEY'] + '` is expired in _google_custom_, and disabled.'
master.postAnyData(postdata, channel)
ec.enableKeywordSetting(target, key['Index'], False)
logger.info(postdata)
except:
logger.error('--ERROR HAS OCCURED IN GOOGLE CUSTOM SEARCH--')
logger.error(traceback.format_exc())
master.postAnyData(traceback.format_exc(), slackbot_settings.channels[0])
def runSearchGitlabSnippets():
try:
logger.info('--START GITLAB SNIPPETS SEARCH--')
now = datetime.date.today()
today = now.strftime('%Y-%m-%d')
target = 'gitlab_snippet'
keywords = ec.getEnableKeywords(target)
if ec.isEnable(target) and keywords != None and keywords != []:
safe_limit = 6
error_safety = ec.getSafetyCount(target)
for key in keywords:
channel = key['Channel']
limittime = datetime.datetime.strptime(key['Expire_date'], '%Y-%m-%d').date()
if now > limittime:
postdata = '`' + key['KEY'] + '` is expired in _gitlab_snippet_, and disabled.'
master.postAnyData(postdata, channel)
ec.enableKeywordSetting('gitlab_snippet', key['Index'], False)
logger.info(postdata)
keywords = ec.getEnableKeywords(target)
if keywords != None and keywords != []:
keylist = [d.get('KEY') for d in keywords]
(results, statuscode) = search_api.searchGitlabSnippets(keylist)
if statuscode != 200:
error_safety += 1
ec.setSafetyCount(target, error_safety)
postdata = '_gitlab_snippet_ failed to search.\nStatus Code: ' + str(statuscode)
master.postAnyData(postdata, channel)
logger.info(postdata)
if error_safety > safe_limit:
postdata = 'Too Many Errors. _Gitlab Snippet_ Module is disabled for safety'
ec.disable(target)
master.postAnyData(postdata, channel)
logger.info(postdata)
else:
ec.setSafetyCount(target, 0)
for key in keywords:
if key['KEY'] in results.keys():
result = list(set(results[key['KEY']]) - set(key['Exclude_list']))
if result != []:
channel = key['Channel']
postdata = 'New Code Found about `' + key['KEY'] + '` in _gitlab_snippet_'
master.postAnyData(postdata, channel)
logger.info(postdata)
url = []
for i in result:
url.append('https://gitlab.com' + i)
logger.info('https://gitlab.com' + i)
# exclude = list(set(results[word]) & set(keywords[word][1]))
exclude = results[key['KEY']]
if channel in getSpecialChannel():
doSpecialAct(target, channel, key['KEY'], url)
master.postAnyData('\n'.join(url), channel)
ec.clearExcludeList(target, key['Index'])
ec.addExcludeList(target, key['Index'], exclude)
except:
logger.error('--ERROR HAS OCCURED IN GITLAB SNIPPETS SEARCH--')
logger.error(traceback.format_exc())
master.postAnyData(traceback.format_exc(), slackbot_settings.channels[0])
def runSearchGithubCode():
try:
logger.info('--START GITHUB CODE SEARCH--')
now = datetime.date.today()
today = now.strftime('%Y-%m-%d')
api_key = slackbot_settings.github_access_token
target = 'github_code'
keywords = ec.getEnableKeywords(target)
if ec.isEnable(target) and keywords != None and keywords != []:
safe_limit = 6
error_safety = ec.getSafetyCount(target)
for key in keywords:
channel = key['Channel']
limittime = datetime.datetime.strptime(key['Expire_date'], '%Y-%m-%d').date()
if now < limittime:
(results, statuscode) = search_api.searchGithubCode(key['KEY'], key['SearchLevel'], api_key)
result = list(set(results) - set(key['Exclude_list']))
if statuscode != 200:
error_safety += 1
ec.setSafetyCount('github_code', error_safety)
postdata = '`' + key['KEY'] + '` failed to search in _github_code_.\nStatus Code: ' + str(statuscode)
master.postAnyData(postdata, channel)
logger.info(postdata)
if error_safety > safe_limit:
postdata = 'Too Many Errors. _Github Code_ Module is disabled for safety'
ec.disable('github_code')
master.postAnyData(postdata, channel)
logger.info(postdata)
else:
ec.setSafetyCount('github_code', 0)
if key['__INITIAL__'] == True:
ec.haveSearched(target, key['Index'])
if result != []:
postdata = 'New Code Found about `' + key['KEY'] + '` in _github_code_'
master.postAnyData(postdata, channel)
if key['__INITIAL__'] == True:
master.postAnyData(result[0], channel)
else:
if channel in getSpecialChannel():
doSpecialAct(target, channel, key['KEY'], result)
master.postAnyData('\n'.join(result), channel)
logger.info('keyword : ' + key['KEY'])
logger.info('\n'.join(result))
exclude = results
# ec.clearExcludeList('github_code', conf['Index'])
ec.addExcludeList('github_code', key['Index'], exclude)
time.sleep(10)
else:
postdata = '`' + key['KEY'] + '` expired in _github_code_, and was disabled.'
logger.info(postdata)
master.postAnyData(postdata, channel)
ec.enableKeywordSetting('github_code', key['Index'], False)
except:
logger.error('--ERROR HAS OCCURED IN GITHUB SEARCH--')
logger.error(traceback.format_exc())
master.postAnyData(traceback.format_exc(), slackbot_settings.channels[0])
def runSearchGithub():
try:
logger.info('--START GITHUB SEARCH--')
now = datetime.date.today()
today = now.strftime('%Y-%m-%d')
target = 'github'
keywords = ec.getEnableKeywords(target)
if ec.isEnable(target) and keywords != None and keywords != []:
safe_limit = 6
error_safety = ec.getSafetyCount(target)
for key in keywords:
channel = key['Channel']
limittime = datetime.datetime.strptime(key['Expire_date'],
'%Y-%m-%d').date()
if now < limittime:
oldtime = now - datetime.timedelta(key['Time_Range'])
oldday = oldtime.strftime('%Y-%m-%d')
(results, statuscode) = search_api.searchGithub(
key['KEY'], oldday, key['SearchLevel'])
result = list(set(results) - set(key['Exclude_list']))
if statuscode != 200:
error_safety += 1
ec.setSafetyCount(target, error_safety)
postdata = '`' + key[
'KEY'] + '` failed to search in _github_.\nStatus Code: ' + str(
statuscode)
master.postAnyData(postdata, channel)
logger.info(postdata)
if error_safety > safe_limit:
postdata = 'Too Many Errors. _Github_ Module is disabled for safety'
ec.disable(target)
master.postAnyData(postdata, channel)
logger.info(postdata)
else:
ec.setSafetyCount(target, 0)
if result != []:
if channel in getSpecialChannel():
doSpecialAct(target, channel, key['KEY'],
result)
master.postNewPoCFound(key['KEY'], result, channel)
logger.info('keyword : ' + key['KEY'])
logger.info('\n'.join(result))
exclude = results
ec.clearExcludeList(target, key['Index'])
ec.addExcludeList(target, key['Index'], exclude)
time.sleep(10)
else:
postdata = '`' + key[
'Index'] + '` expired in _github_, and was disabled.'
logger.info(postdata)
master.postAnyData(postdata, channel)
ec.enableKeywordSetting(target, key['Index'], False)
except:
logger.error('--ERROR HAS OCCURED IN GITHUB SEARCH--')
logger.error(traceback.format_exc())
master.postAnyData(traceback.format_exc(),
slackbot_settings.channels[0])
def runSearchGitlab():
try:
logger.info('--START GITLAB SEARCH--')
now = datetime.date.today()
today = now.strftime('%Y-%m-%d')
target = 'gitlab'
keywords = ec.getEnableKeywords(target)
if ec.isEnable(target) and keywords != None and keywords != []:
safe_limit = 6
error_safety = ec.getSafetyCount(target)
for key in keywords:
channel = key['Channel']
limittime = datetime.datetime.strptime(key['Expire_date'], '%Y-%m-%d').date()
if now < limittime:
(results, statuscode) = search_api.searchGitlab(key['KEY'])
result = list(set(results) - set(key['Exclude_list']))
if statuscode != 200:
error_safety += 1
ec.setSafetyCount(target, error_safety)
postdata = '`' + key['KEY'] + '` failed to search in _gitlab_.\nStatus Code: ' + str(statuscode)
master.postAnyData(postdata, channel)
logger.info(postdata)
if error_safety > safe_limit:
postdata = 'Too Many Errors. _Gitlab_ Module is disabled for safety'
ec.disable(target)
master.postAnyData(postdata, channel)
logger.info(postdata)
else:
if error_safety != 0:
ec.setSafetyCount(target, 0)
if key['__INITIAL__'] == True:
ec.haveSearched(target, key['Index'])
if result != []:
postdata = 'New Code Found about `' + key['KEY'] + '` in _gitlab_'
master.postAnyData(postdata, channel)
url = []
for i in result:
url.append('https://gitlab.com' + i)
if key['__INITIAL__'] == True:
master.postAnyData(url[0], channel)
else:
if channel in getSpecialChannel():
doSpecialAct(target, channel, key['KEY'], url)
master.postAnyData('\n'.join(url), channel)
logger.info('keyword : ' + key['KEY'])
logger.info('\n'.join(url))
exclude = results
ec.clearExcludeList(target, key['Index'])
ec.addExcludeList(target, key['Index'], exclude)
time.sleep(30)
else:
postdata = '`' + key['KEY'] + '` is expired in _gitlab_, and disabled.'
master.postAnyData(postdata, channel)
ec.enableKeywordSetting(target, key['Index'], False)
logger.info(postdata)
except:
logger.error('--ERROR HAS OCCURED IN GITLAB SEARCH--')
logger.error(traceback.format_exc())
master.postAnyData(traceback.format_exc(), slackbot_settings.channels[0])