def shutdown():
# Create necessary database connections to use
mysql.close()
mongo.close()
data['userId'] = user_id
data['activeDirectorySeq'] = category['activeDirectorySeq']
data['directoryNo'] = category['directoryNo']
data['category'] = category['name']
insert(category['index-name'], data)
def get_driver(timeout=30):
options = webdriver.ChromeOptions()
options.add_argument('headless')
options.add_argument("--no-sandbox")
options.add_argument("--disable-dev-shm-usage")
prefs = {"profile.managed_default_content_settings.images": 2}
options.add_experimental_option("prefs", prefs)
driver = webdriver.Chrome('./driver/chromedriver-' + OS_CONFIG['os'], chrome_options=options)
driver.set_page_load_timeout(timeout)
return driver
if __name__ == '__main__':
driver = get_driver()
for idx, category in enumerate(CATEGORIES):
print('start crawling new category...(' + str(idx + 1) + '/' + str(len(CATEGORIES)) + ')')
try:
crawl_category(category)
except Exception:
print('skipped this category: ' + category['index-name'])
print(traceback.format_exc())
print('successfully completed')
driver.quit()
close()
# 부산광역시 sites.busan_scan() # 한국인공지능협회 sites.koraia_scan() # 아이디어마루 sites.ideamaru_scan() # 부산테크노파크_사업공고 sites.btp_scan() # 서울기업지원센터 sites.sbsc_scan() # 중소벤처기업부 sites.mss_scan() sys.stdout.close() # 출력결과 저장 sys.stdout.close() # 디비 닫기 mongo.close() # file 내용 push message.telegram_push() sys.exit(0)