def CrawlingByInstagramCrawlBot(userURL, username, fb_pageID):
start_time_all = time.time()
with requests.Session() as s:
loginURL = 'https://www.instagram.com/accounts/login/'
headers = {
'User-Agent':
'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:20.0) Gecko/20100101 Firefox/20.0'
}
response = requests.get(loginURL, headers=headers)
print(response)
print('Auto login start.')
chrome_options = Options()
chrome_options.add_argument("--headless")
chrome_options.add_argument("--window-size=1920x1080")
prefs = {}
prefs['profile.default_content_setting_values.notifications'] = 2
chrome_options.add_experimental_option('prefs', prefs)
driver_chrome = r"C:\python_project\chromedriver.exe"
driver = webdriver.Chrome(chrome_options=chrome_options,
executable_path=driver_chrome)
driver.get(loginURL)
print('driver.current_url : ', driver.current_url)
basic_url = 'https://www.instagram.com/'
userInstaURL = basic_url + userURL + '/?hl=ko'
#Crawler 호출
instagram_returnResult = bringContentBasicData(driver, userInstaURL)
print('return Result : ', instagram_returnResult)
'''
'사용자ID': 'dasolmom_',
'게시글수': 244,
'팔로워수': 545000,
'팔로잉수': 65,
'게시글내단어수': '938',
'해쉬테그수': '239',
'''
instagram_T_Value = 0
instagram_C_Value = 0
instagram_M_Value = 0
try:
if instagram_returnResult['사용자ID'] is not None:
if instagram_returnResult['사용자ID'] == username:
instagram_T_Value += 50
else:
instagram_T_Value += 35
else:
print('불가능한 경우: 사용자ID 값이 없는 경우')
except Exception as e:
print()
instagram_returnResult['insta_TSCORE'] = instagram_T_Value
try:
if instagram_returnResult['게시글수'] >= 500:
instagram_C_Value += 50
elif instagram_returnResult[
'게시글수'] < 500 and instagram_returnResult['게시글수'] >= 300:
instagram_C_Value += 35
else:
instagram_C_Value += 15
except Exception as e:
print('게시글이 존재하지 않는 경우')
try:
if instagram_returnResult['팔로워수'] >= 100000:
instagram_C_Value += 50
elif instagram_returnResult[
'팔로워수'] < 100000 and instagram_returnResult[
'팔로워수'] >= 50000:
instagram_C_Value += 40
elif instagram_returnResult[
'팔로워수'] < 50000 and instagram_returnResult['팔로워수'] >= 10000:
instagram_C_Value += 30
elif instagram_returnResult[
'팔로워수'] < 10000 and instagram_returnResult['팔로워수'] >= 5000:
instagram_C_Value += 20
else:
instagram_C_Value += 10
except Exception as e:
print('팔로워가 존재하지 않는 경우')
try:
if instagram_returnResult['팔로잉수'] >= 10000:
instagram_C_Value += 50
elif instagram_returnResult[
'팔로잉수'] < 10000 and instagram_returnResult['팔로잉수'] >= 5000:
instagram_C_Value += 40
elif instagram_returnResult[
'팔로잉수'] < 5000 and instagram_returnResult['팔로잉수'] >= 1000:
instagram_C_Value += 30
elif instagram_returnResult[
'팔로잉수'] < 1000 and instagram_returnResult['팔로잉수'] >= 500:
instagram_C_Value += 20
elif instagram_returnResult[
'팔로잉수'] < 500 and instagram_returnResult['팔로잉수'] >= 100:
instagram_C_Value += 10
else:
instagram_C_Value += 5
except Exception as e:
print('팔로잉이 존재하지 않는 경우')
instagram_returnResult['insta_CSCORE'] = instagram_C_Value
try:
if instagram_returnResult['게시글내단어수'] >= 1000:
instagram_M_Value += 70
elif instagram_returnResult[
'게시글내단어수'] < 1000 and instagram_returnResult[
'게시글내단어수'] >= 500:
instagram_M_Value += 60
elif instagram_returnResult[
'게시글내단어수'] < 500 and instagram_returnResult[
'게시글내단어수'] >= 100:
instagram_M_Value += 50
elif instagram_returnResult[
'게시글내단어수'] < 100 and instagram_returnResult[
'게시글내단어수'] >= 10:
instagram_M_Value += 40
else:
instagram_M_Value += 30
except Exception as e:
print('게시글내단어수가 존재하지 않는 경우')
try:
if instagram_returnResult['해쉬테그수'] >= 500:
instagram_M_Value += 70
elif instagram_returnResult[
'해쉬테그수'] < 500 and instagram_returnResult['해쉬테그수'] >= 300:
instagram_M_Value += 60
elif instagram_returnResult[
'해쉬테그수'] < 300 and instagram_returnResult['해쉬테그수'] >= 100:
instagram_M_Value += 50
elif instagram_returnResult[
'해쉬테그수'] < 100 and instagram_returnResult['해쉬테그수'] >= 50:
instagram_M_Value += 40
elif instagram_returnResult[
'해쉬테그수'] < 50 and instagram_returnResult['해쉬테그수'] >= 10:
instagram_M_Value += 30
else:
instagram_M_Value += 20
except Exception as e:
print('해쉬테그수가 존재하지 않는 경우')
instagram_returnResult['insta_MSCORE'] = instagram_C_Value
returnValue_instagram = True
# DB Connection=======================================
currDate = str(time.localtime().tm_year) + '-' + str(time.localtime().tm_mon) + '-' + str(time.localtime().tm_mday) + '-' \
+ str(time.localtime().tm_hour) + '-' + str(time.localtime().tm_min) + '-' + str(time.localtime().tm_sec)
print(currDate)
end_time = time.time() - start_time_all
print('데이터 기반 크롤링 총 구동 시간 :', end_time)
try:
# Server Connection to MySQL
databaseConnection = mysqlConnection.DatabaseConnection_origin()
databaseConnection.update_instagramRecord(
str(instagram_returnResult['사용자ID'].replace(" ", "")),
str(instagram_returnResult['게시글수']),
str(instagram_returnResult['팔로워수']),
str(instagram_returnResult['팔로잉수']),
str(instagram_returnResult['게시글내단어수']),
str(instagram_returnResult['해쉬테그수']),
str(instagram_returnResult['insta_TSCORE']),
str(instagram_returnResult['insta_CSCORE']),
str(instagram_returnResult['insta_MSCORE']), fb_pageID)
except Exception as e_maria:
logger.error(
'[ Error ] MariaDB About information Insertion => {}'.format(
e_maria))
instagram_returnResult['trueOrFalse'] = True
driver.close()
return instagram_returnResult
def get_TotalScore(tot_TSCORE, tot_CSCORE, tot_MSCORE, fromDate, toDate,
userName):
#tot_TSCORE = 50
#tot_CSCORE = 50
#tot_MSCORE = 50
userSNSRank = 'D-'
userSNS_T_Rank_score = 1
userSNS_C_Rank_score = 1
userSNS_M_Rank_score = 1
# 사용자 등급 : userSnsRank
userSNS_T_Rank = ''
if (tot_TSCORE <= 600 and tot_TSCORE > 500):
userSNS_T_Rank = 'T_A+'
userSNS_T_Rank_score = 8
elif (tot_TSCORE <= 500 and tot_TSCORE > 400):
userSNS_T_Rank = 'T_A-'
userSNS_T_Rank_score = 7
elif (tot_TSCORE <= 400 and tot_TSCORE > 350):
userSNS_T_Rank = 'T_B+'
userSNS_T_Rank_score = 6
elif (tot_TSCORE <= 350 and tot_TSCORE > 300):
userSNS_T_Rank = 'T_B-'
userSNS_T_Rank_score = 5
elif (tot_TSCORE <= 300 and tot_TSCORE > 250):
userSNS_T_Rank = 'T_C+'
userSNS_T_Rank_score = 4
elif (tot_TSCORE <= 250 and tot_TSCORE > 200):
userSNS_T_Rank = 'T_C-'
userSNS_T_Rank_score = 3
elif (tot_TSCORE <= 200 and tot_TSCORE > 100):
userSNS_T_Rank = 'T_D+'
userSNS_T_Rank_score = 2
elif (tot_TSCORE <= 100 and tot_TSCORE > 1):
userSNS_T_Rank = 'T_D-'
userSNS_T_Rank_score = 1
print('사용자의 T 등급 :', userSNS_T_Rank)
userSNS_C_Rank = ''
if (tot_CSCORE <= 1000 and tot_CSCORE > 850):
userSNS_C_Rank = 'C_A+'
userSNS_C_Rank_score = 8
elif (tot_CSCORE <= 850 and tot_CSCORE > 700):
userSNS_C_Rank = 'C_A-'
userSNS_C_Rank_score = 7
elif (tot_CSCORE <= 700 and tot_CSCORE > 550):
userSNS_C_Rank = 'C_B+'
userSNS_C_Rank_score = 6
elif (tot_CSCORE <= 550 and tot_CSCORE > 400):
userSNS_C_Rank = 'C_B-'
userSNS_C_Rank_score = 5
elif (tot_CSCORE <= 400 and tot_CSCORE > 300):
userSNS_C_Rank = 'C_C+'
userSNS_C_Rank_score = 4
elif (tot_CSCORE <= 300 and tot_CSCORE > 200):
userSNS_C_Rank = 'C_C-'
userSNS_C_Rank_score = 3
elif (tot_CSCORE <= 200 and tot_CSCORE > 100):
userSNS_C_Rank = 'C_D+'
userSNS_C_Rank_score = 2
elif (tot_CSCORE <= 100 and tot_CSCORE > 1):
userSNS_C_Rank = 'C_D-'
userSNS_C_Rank_score = 1
print('사용자의 C 등급 :', userSNS_C_Rank)
userSNS_M_Rank = ''
if (tot_MSCORE <= 400 and tot_MSCORE > 350):
userSNS_M_Rank = 'M_A+'
userSNS_M_Rank_score = 8
elif (tot_MSCORE <= 350 and tot_MSCORE > 300):
userSNS_M_Rank = 'M_A-'
userSNS_M_Rank_score = 7
elif (tot_MSCORE <= 300 and tot_MSCORE > 250):
userSNS_M_Rank = 'M_B+'
userSNS_M_Rank_score = 6
elif (tot_MSCORE <= 250 and tot_MSCORE > 200):
userSNS_M_Rank = 'M_B-'
userSNS_M_Rank_score = 5
elif (tot_MSCORE <= 200 and tot_MSCORE > 150):
userSNS_M_Rank = 'M_C+'
userSNS_M_Rank_score = 4
elif (tot_MSCORE <= 150 and tot_MSCORE > 100):
userSNS_M_Rank = 'M_C-'
userSNS_M_Rank_score = 3
elif (tot_MSCORE <= 100 and tot_MSCORE > 50):
userSNS_M_Rank = 'M_D+'
userSNS_M_Rank_score = 2
elif (tot_MSCORE <= 50 and tot_MSCORE > 1):
userSNS_M_Rank = 'M_D-'
userSNS_M_Rank_score = 1
print('사용자의 M 등급 :', userSNS_M_Rank)
avg_RankgScore = math.ceil(
(userSNS_T_Rank_score + userSNS_C_Rank_score + userSNS_M_Rank_score) /
3)
print('사용자의 평균 SCORE : ', avg_RankgScore)
if avg_RankgScore <= 8 and avg_RankgScore > 7:
userSNSRank = 'A+'
elif avg_RankgScore <= 7 and avg_RankgScore > 6:
userSNSRank = 'A-'
elif avg_RankgScore <= 6 and avg_RankgScore > 5:
userSNSRank = 'B+'
elif avg_RankgScore <= 5 and avg_RankgScore > 4:
userSNSRank = 'B-'
elif avg_RankgScore <= 4 and avg_RankgScore > 3:
userSNSRank = 'C+'
elif avg_RankgScore <= 3 and avg_RankgScore > 2:
userSNSRank = 'C-'
elif avg_RankgScore <= 2 and avg_RankgScore > 1:
userSNSRank = 'D+'
elif avg_RankgScore <= 1 and avg_RankgScore > 0:
userSNSRank = 'D-'
print('userSNS_Rank:', userSNSRank)
# DB insert
try:
# Server Connection to MySQL
databaseConnection = mysqlConnection.DatabaseConnection_origin()
databaseConnection.update_totalTCM_Record(str(tot_TSCORE),
str(tot_CSCORE),
str(tot_MSCORE), userSNSRank,
fromDate, toDate,
userCellPhNum, userName)
except Exception as e_maria:
print('[ Error ] MariaDB About information Insertion :', e_maria)
print('분석 종료')
print()
print()
def CrawlingByNaverBlogCrawlBot(userNaverBlogURL, userName, userCellPhNum):
returnedValue_naverBlog = False
start_time_all = time.time()
'''
currDate = str(time.localtime().tm_year) + '-' + str(time.localtime().tm_mon) + '-' + str(
time.localtime().tm_mday) + '-' \
+ str(time.localtime().tm_hour) + '-' + str(time.localtime().tm_min) + '-' + str(time.localtime().tm_sec)
'''
currDate = currTime
#print(currDate)
logger.debug('current date =? {}'.format(currDate))
Results_DictionatuType = {}
# Session 생성, with 구문 안에서 유지
with requests.Session() as s:
total_num = 0
error_list = []
ResultNBDict = {}
blog_id = userNaverBlogURL
start_p = 1
end_p = 100
naverBlog_Tscore = 0
naverBlog_Cscore = 0
naverBlog_Mscore = 0
#naverBlog_returnResult = []
print("\nCreating File Naver_Blog_Crawling_Result.txt...\n")
# 파일 열기
#file = codecs.open("Log_Text_Folder/Naver_Blog_Crawling_Result.txt", 'w', encoding="utf-8")
ResultNBDict['사용자ID'] = userNaverBlogURL
ResultNBDict['게시글수'] = 0
ResultNBDict['금일방문자수'] = 0
ResultNBDict['전체방문자수'] = 0
ResultNBDict['게시글내단어수'] = 0
ResultNBDict['nb_TSCORE'] = 0
ResultNBDict['nb_CSCORE'] = 0
ResultNBDict['nb_MSCORE'] = 0
# 페이지 단위
for page in range(1, 30 + 1):
#구분자 표시
print("=" * 50)
#file.write("=" * 50 + "\n")
#doc = collections.OrderedDict()
doc = {}
url = "http://blog.naver.com/PostList.nhn?blogId=" + blog_id + "¤tPage=" + str(
page)
r = requests.get(url)
if (not r.ok):
print("Page_" + str(page) + "연결 실패, Skip")
print('본사용자의 네이버 블로그는 크롤링이 불가합니다. ')
returnedValue_naverBlog = False
try:
# Server Connection to MySQL
databaseConnection = mysqlConnection.DatabaseConnection_origin(
)
databaseConnection.update_naverblogRecord(
str(ResultNBDict['사용자ID'].replace(" ", "")),
str(ResultNBDict['게시글수']), str(ResultNBDict['금일방문자수']),
str(ResultNBDict['전체방문자수']),
str(ResultNBDict['게시글내단어수']),
str(ResultNBDict['nb_TSCORE']),
str(ResultNBDict['nb_CSCORE']),
str(ResultNBDict['nb_MSCORE']), userCellPhNum)
except Exception as e_maria:
logger.error(
'[ Error ] MariaDB About information Insertion => {}'.
format(e_maria))
ResultNBDict['trueOrFalse'] = False
return ResultNBDict
break
#continue
# html 파싱
soup = BeautifulSoup(r.text.encode("utf-8"), "html.parser")
naverCrawledData_dictionary = {}
#네이버 블로그가 존재하면 기본 TSCORE 15점 부여
naverCrawledData_dictionary['nb_TSCORE'] = 15
#전체보기 글 전체 개수 : MSCORE
post_tot_cnt = soup.find_all('h4', class_='pcol2')
try:
#게시글이 존재하는 지 여부 따지기
print(
'post_tot_cnt :',
post_tot_cnt[0].text.split(" ")[1].replace("개의",
"").replace(
",", ""))
postCnt = int(post_tot_cnt[0].text.split(" ")[1].replace(
"개의", "").replace(",", ""))
print('포스트 개수 :', postCnt)
ResultNBDict['게시글수'] = postCnt
ResultNBDict['금일방문자수'] = 0
ResultNBDict['전체방문자수'] = 0
if postCnt >= 5000:
naverBlog_Mscore += 20
elif postCnt < 5000 and postCnt >= 3000:
naverBlog_Mscore += 15
elif postCnt < 3000 and postCnt >= 1000:
naverBlog_Mscore += 10
elif postCnt < 1000 and postCnt >= 500:
naverBlog_Mscore += 5
else:
print('게시글 개수가 부족합니다.')
naverCrawledData_dictionary['nb_MSCORE'] = naverBlog_Mscore
cycleLength = postCnt / 10
# 페이지 당 포스트 수 (printPost_# 형식의 id를 가진 태그 수)
post_count = len(
soup.find_all("table", {"id": re.compile("printPost.")}))
doc['page_' + str(page)] = page
#naverCrawledData_dictionary[str(doc['page_' + str(page)])] = page
#print('@#@#:', naverCrawledData_dictionary[str(doc['page_' + str(page)])])
# 포스트 단위
for pidx in range(1, post_count + 1):
print('-' * 50)
#file.write('-' * 50 + "\n")
doc['post_number_' + str(pidx)] = pidx
post = soup.find("table", {"id": "printPost" + str(pidx)})
naverCrawledData_dictionary[str(doc['post_number_' +
str(pidx)])] = pidx
# 제목 찾기---------------------------
title = post.find("h3", {"class": "se_textarea"})
# 스마트에디터3 타이틀 제거 임시 적용 (클래스가 다름)
if (title == None):
title = post.find(
"span", {"class": "pcol1 itemSubjectBoldfont"})
if (title != None):
doc["title_" + str(pidx)] = title.text.strip()
naverCrawledData_dictionary[str(
doc["title_" + str(pidx)])] = title.text.strip()
#naverCrawledPostData_list.append(list(str(doc['title']).replace(",", "").replace(" ", "")))
#print('$$_dictionary', naverCrawledData_dictionary)
else:
doc["title"] = "TITLE ERROR"
# 날짜 찾기---------------------------
date = post.find("span",
{"class": "se_publishDate pcol2 fil5"})
# 스마트에디터3 타이틀 제거 임시 적용 (클래스가 다름)
if (date == None):
date = post.find(
"p", {"class": "date fil5 pcol2 _postAddDate"})
if (date != None):
doc['date_' + currDate] = date.text.replace(". ", "_")
naverCrawledData_dictionary[str(
doc['date_' + currDate])] = date.text.replace(
". ", "_")
else:
doc['date_' + currDate] = "DATE ERROR"
naverCrawledData_dictionary[str(
doc['date_' + currDate])] = "DATE ERROR"
logger.error('ERROR_DATE => {}'.format(currDate))
# 내용 찾기---------------------------
content = post.find("div", {
"class":
"se_component_wrap sect_dsc __se_component_area"
})
# 스마트에디터3 타이틀 제거 임시 적용 (클래스가 다름)
if (content == None):
content = post.find("div", {"id": "postViewArea"})
if (title != None):
# Enter 5줄은 하나로
doc["content_postNo_" +
str(pidx)] = "\n" + content.text.strip().replace(
"\n" * 5, "\n").replace(".", "__")
naverCrawledData_dictionary[str(doc[
"content_postNo_" +
str(pidx)])] = "\n" + content.text.strip().replace(
"\n" * 5, "\n")
logger.debug('DEBUG_CONTENT_postNo_ => {}'.format(
str(pidx)))
else:
doc["content_postNo_" + str(pidx)] = "CONTENT ERROR"
naverCrawledData_dictionary[str(
doc["content_postNo_" +
str(pidx)])] = "CONTENT ERROR"
logger.error('ERROR_CONTENT_postNo_ => {}'.format(
str(pidx)))
# doc 출력 (UnicodeError - 커맨드 창에서 실행 시 발생)
print('text data', list(doc))
tot_wordCnt = 0
for i in doc:
str_doc = str(i) + ": " + str(doc[i])
try:
print(str_doc)
wordCnt = len(str_doc.split(" "))
tot_wordCnt += wordCnt
except UnicodeError:
print(str_doc.encode("utf-8"))
wordCnt = 0
#file.write(str_doc + "\n")
# 파일 쓰기
if ("ERROR" in str(doc[i])):
insert_error(doc[i], doc, error_list)
ResultNBDict['게시글내단어수'] += tot_wordCnt
# 전체 수 증가
total_num += 1
continue
print('END')
except Exception as e:
print('게시글이 존재하지 않습니다. ')
break
# 결과 출력 (전체 글 수, 에러 수)
print("=" * 50)
#file.write("=" * 50 + "\n")
print("Total : " + str(total_num))
logger.debug('DEBUG_TOTAL_POST_COUNT => {}'.format(str(total_num)))
#DB insert
try:
# Server Connection to MySQL
databaseConnection = mysqlConnection.DatabaseConnection_origin()
databaseConnection.update_naverblogRecord(
str(ResultNBDict['사용자ID'].replace(" ", "")),
str(ResultNBDict['게시글수']), str(ResultNBDict['금일방문자수']),
str(ResultNBDict['전체방문자수']), str(ResultNBDict['게시글내단어수']),
str(ResultNBDict['nb_TSCORE']), str(ResultNBDict['nb_CSCORE']),
str(ResultNBDict['nb_MSCORE']), userCellPhNum)
returnedValue_naverBlog = True
ResultNBDict['trueOrFalse'] = True
except Exception as e_maria:
logger.error(
'[ Error ] MariaDB About information Insertion => {}'.format(
e_maria))
#ERROR detection
error_count = len(error_list)
print("Error : " + str(error_count))
logger.error('ERROR_COUNT => {}'.format(str(error_count)))
# 에러가 있을 경우 출력
if (error_count != 0):
print("Error Post : ")
for i in error_list:
print(i)
logger.error('ERROR_POST_LIST => {}'.format(i))
#returnedValue_naverBlog = True
#list(doc) --> [ "page", "post_number", "title", "date", "content" ]
#DB Connection=======================================
end_time = time.time() - start_time_all
print('데이터 기반 크롤링 총 구동 시간 :', end_time)
ResultNBDict['trueOrFalse'] = True
return ResultNBDict
print('분석 종료')
print()
print()
# 실행 함수 선언
if __name__ == '__main__':
# start_crawling(userCellPhNum, kakaoStoryURL, naverblogURL, facebookURL, reqClient)
global roll
roll = 0
while roll < 10000:
databaseConnection = mysqlConnection.DatabaseConnection_origin()
search_idx_queue = databaseConnection.pre_Insert_sci_record()
print('search_idx_queue :', search_idx_queue)
search_idx_tbl = databaseConnection.select_lastIndex_sci_record(
str(search_idx_queue[0]).replace(" ", ""))
print('search log index로 가져온 search_log_index값 :',
str(search_idx_tbl[0]).replace(" ", ""), ', ',
str(search_idx_tbl[1]).replace(" ", ""))
userRealName = str(search_idx_tbl[1]).replace(" ", "")
databaseConnection = mysqlConnection.DatabaseConnection_origin()
# usr_origin_data = databaseConnection.select_sci_record(str(search_idx_queue[0]).replace(" ","") )
usr_origin_data = databaseConnection.select_sci_record(
def parsed_POST_data(self, post_data):
#getParam = urllib.parse.unquote(post_data.decode('utf-8'))
getParam = urllib.parse.unquote(post_data.decode())
print('입력한 데이터: ', getParam)
#http://172.30.1.33:8000/kakao_story=https%3A%2F%2Fstory.kakao.com%2F_9W0DZ3&naver_blog=https%3A%2F%2Fblog.naver.com%2Ftramper2&facebook=https%3A%2F%2Fwww.facebook.com%2Fdanny.woo.33&instagram=XXXX&username=%EA%B9%80%ED%83%9C%ED%98%B8
# kakao_story=https%3A%2F%2Fstory.kakao.com%2F_9W0DZ3&
# naver_blog=https%3A%2F%2Fblog.naver.com%2Ftramper2&
# facebook=https%3A%2F%2Fwww.facebook.com%2Fdanny.woo.33&
# instagram=&
# username=%EA%B9%80%ED%83%9C%ED%98%B8
userData = {}
if getParam is not None:
snsURL_ADDR_fromWeb = getParam.split('&')
#print(snsURL_ADDR_fromWeb[0], snsURL_ADDR_fromWeb[1], snsURL_ADDR_fromWeb[2], snsURL_ADDR_fromWeb[3], snsURL_ADDR_fromWeb[4])
#kakao_story=XXXX&naver_blog=XXXX&facebook=XXXX&instagram=XXXX&username=XXXX
print('&로 나눠진 데이터 리스트:', snsURL_ADDR_fromWeb)
if snsURL_ADDR_fromWeb[0]:
kakaoStory_USERURL_tot = snsURL_ADDR_fromWeb[0].split(
"=")[1].replace(" ", "")
try:
print('kakaoStory SNS 주소가 존재합니다.')
kakaoStory_USERURL_pre = kakaoStory_USERURL_tot.split(
'//')[1]
kakaoStory_USERURL = kakaoStory_USERURL_pre.split('/')[1]
userData['kakao_story'] = kakaoStory_USERURL
except Exception as e:
print('kakaoStory SNS 주소가 존재하지 않습니다.', e)
kakaoStory_USERURL = None
else:
print('kakaoStory SNS 주소가 존재하지 않습니다.')
kakaoStory_USERURL = None
'''
if snsURL_ADDR_fromWeb[1]:
naverBlog_USERURL_tot= snsURL_ADDR_fromWeb[1].split("=")[1].replace(" ", "")
try:
print('naverBlog SNS 주소가 존재합니다.')
naverBlog_USERURL_pre = naverBlog_USERURL_tot.split('//')[1]
naverBlog_USERURL = naverBlog_USERURL_pre.split('/')[1]
userData['naver_blog'] = naverBlog_USERURL
except Exception as e:
print('naverBlog SNS 주소가 존재하지 않습니다.', e)
naverBlog_USERURL = None
else:
print('naverBlog SNS 주소가 존재하지 않습니다.')
naverBlog_USERURL = None
'''
naverBlog_USERURL = None
if snsURL_ADDR_fromWeb[2]:
facebook_USERURL_tot = snsURL_ADDR_fromWeb[2].split(
"=")[1].replace(" ", "")
try:
print('facebook SNS 주소가 존재합니다.')
facebook_USERURL_pre = facebook_USERURL_tot.split('//')[1]
facebook_USERURL = facebook_USERURL_pre.split('/')[1]
userData['facebook'] = facebook_USERURL
except Exception as e:
print('facebook SNS 주소가 존재하지 않습니다.', e)
facebook_USERURL = None
else:
print('facebook SNS 주소가 존재하지 않습니다.')
facebook_USERURL = None
if snsURL_ADDR_fromWeb[3]:
instagram_USERURL_tot = snsURL_ADDR_fromWeb[3].split(
"=")[1].replace(" ", "")
try:
instagram_USERURL_pre = instagram_USERURL_tot.split(
'//')[1]
instagram_USERURL = instagram_USERURL_pre.split('/')[1]
print('instagram SNS 주소가 존재합니다.')
userData['instagram_USERURL'] = instagram_USERURL
except Exception as e:
print('instagram SNS 주소가 존재하지 않습니다.', e)
instagram_USERURL = None
else:
print('instagram SNS 주소가 존재하지 않습니다.')
instagram_USERURL = None
if snsURL_ADDR_fromWeb[4]:
print('사용자 이름이 존재합니다.', snsURL_ADDR_fromWeb[4])
try:
userName = snsURL_ADDR_fromWeb[4].split("=")[1].replace(
" ", "")
userData['username'] = userName
except Exception as e:
print('사용자 이름이 존재하지 않습니다.', e)
userName = None
else:
print('사용자 이름이 존재하지 않습니다.')
userName = None
#print('전달 받은 계정정보 값 :', kakaoStory_USERURL, naverBlog_USERURL, facebook_USERURL, instagram_USERURL, userName)
print('전달 받은 계정정보 값 :', kakaoStory_USERURL, facebook_USERURL,
instagram_USERURL, userName)
print('user Data : ', userData)
tot_TSCORE = 0
tot_CSCORE = 0
tot_MSCORE = 0
userSNSRank = ''
#FACEBOOK
if facebook_USERURL is not None:
try:
#CrawlingByFacebookCrawlBot
ResultDict1 = snsScrapBot.login_facebook(
self, 1, facebook_USERURL, userName)
if ResultDict1['trueOrFalse'] == True:
print('페이스북 크롤링 성공, Score : ',
ResultDict1['tcmScore']['T_SCORE'],
ResultDict1['tcmScore']['C_SCORE'],
ResultDict1['tcmScore']['M_SCORE'])
tot_TSCORE += ResultDict1['tcmScore']['T_SCORE']
tot_CSCORE += ResultDict1['tcmScore']['C_SCORE']
tot_MSCORE += ResultDict1['tcmScore']['M_SCORE']
print('facebook : ', tot_TSCORE, tot_CSCORE,
tot_MSCORE)
jsonData = {
'TSCORE': ResultDict1['tcmScore']['T_SCORE'],
'CSCORE': ResultDict1['tcmScore']['C_SCORE'],
'MSCORE': ResultDict1['tcmScore']['M_SCORE']
}
data = json.dumps(jsonData).encode("utf-8")
print('data 값:', data)
self.wfile.write(data)
print('data 전달 끝')
return
else:
print('페이스북 크롤링이 내부 요인에 의해 결과 값이 제대로 전달되지 않았습니다.')
except Exception as ex_facebook:
print('페이스북 크롤링이 내부 요인에 의해 중지 되었습니다. -> ', ex_facebook)
try:
ResultDict2 = snsScrapBot.login_facebook(
self, 1, facebook_USERURL, userName)
if ResultDict2 == True:
print('2차 페이스북 크롤링 성공, Score : ',
ResultDict2['tcmScore'])
tot_TSCORE += ResultDict2['tcmScore']['T_SCORE']
tot_CSCORE += ResultDict2['tcmScore']['C_SCORE']
tot_MSCORE += ResultDict2['tcmScore']['M_SCORE']
data = json.dumps(ResultDict2['tcmScore'])
print('data 값:', data)
self.wfile.write(data).encode()
print('data 전달 끝')
return
else:
print(
'2차 페이스북 크롤링이 내부 요인에 의해 결과 값이 제대로 전달되지 않았습니다.')
except Exception as ex_facebook2:
print('2차 페이스북 크롤링이 내부 요인에 의해 중지 되었습니다. -> ',
ex_facebook2)
#KAKAOSTORY
if kakaoStory_USERURL and facebook_USERURL is not None:
if kakaoStory_USERURL:
returnValue_kakaostory = kakaostoryScrapBot.crawling_singleData_KakaoStoryCrawlerBot(
kakaoStory_USERURL, userName, facebook_USERURL)
print('returnValue_kakaostory : ', returnValue_kakaostory)
if returnValue_kakaostory[0] == True:
print('카카오스토리 크롤링 성공')
tot_TSCORE += int(
returnValue_kakaostory[1]['kk_TSCORE'])
tot_CSCORE += int(
returnValue_kakaostory[1]['kk_CSCORE'])
tot_MSCORE += int(
returnValue_kakaostory[1]['kk_MSCORE'])
print('kakao: ', tot_TSCORE, tot_CSCORE, tot_MSCORE)
else:
print('카카오스토리 크롤링봇의 내부 요인에 의해 결과 값이 제대로 전달되지 않았습니다.')
elif kakaoStory_USERURL is not None:
facebook_USERURL = ''
if kakaoStory_USERURL:
returnValue_kakaostory = kakaostoryScrapBot.crawling_singleData_KakaoStoryCrawlerBot(
kakaoStory_USERURL, userName, facebook_USERURL)
print('returnValue_kakaostory : ',
returnValue_kakaostory)
if returnValue_kakaostory == True:
print('카카오스토리 크롤링 성공')
tot_TSCORE += returnValue_kakaostory[1][
'kk_TSCORE']
tot_CSCORE += returnValue_kakaostory[1][
'kk_CSCORE']
tot_MSCORE += returnValue_kakaostory[1][
'kk_MSCORE']
print('kakao: ', tot_TSCORE, tot_CSCORE,
tot_MSCORE)
else:
print(
'카카오스토리 크롤링봇의 내부 요인에 의해 결과 값이 제대로 전달되지 않았습니다.')
else:
# kakaoStory_USERURL is None:
returnValue_kakaostory = kakaostoryScrapBot.crawling_CSVdata_KakaoStoryCrawlerBot(
facebook_USERURL)
print('returnValue_kakaostory : ', returnValue_kakaostory)
if returnValue_kakaostory[0] == True:
print('카카오스토리 크롤링 성공')
tot_TSCORE += returnValue_kakaostory[1]['kk_TSCORE']
tot_CSCORE += returnValue_kakaostory[1]['kk_CSCORE']
tot_MSCORE += returnValue_kakaostory[1]['kk_MSCORE']
print('kakao: ', tot_TSCORE, tot_CSCORE, tot_MSCORE)
else:
print('카카오스토리 크롤링봇의 내부 요인에 의해 결과 값이 제대로 전달되지 않았습니다.')
else:
returnValue_kakaostory = kakaostoryScrapBot.crawling_CSVdata_KakaoStoryCrawlerBot(
)
print('returnValue_kakaostory : ', returnValue_kakaostory[0])
print('currDate : ', returnValue_kakaostory[1])
if returnValue_kakaostory[0] == True:
print('카카오스토리 크롤링 성공')
tot_TSCORE += returnValue_kakaostory[1]['kk_TSCORE']
tot_CSCORE += returnValue_kakaostory[1]['kk_CSCORE']
tot_MSCORE += returnValue_kakaostory[1]['kk_MSCORE']
print('kakao: ', tot_TSCORE, tot_CSCORE, tot_MSCORE)
else:
print('카카오스토리 크롤링 봇의 내부 요인에 의해 결과 값이 제대로 전달되지 않았습니다.')
#INSTAGRAM
if instagram_USERURL is not None:
print('인스타그램 크롤링')
returnValue_instagram = instagramScrapBot.CrawlingByInstagramCrawlBot(
instagram_USERURL, userName, facebook_USERURL)
if returnValue_instagram[0] == True:
print('인스타그램 크롤링 성공')
tot_TSCORE += returnValue_instagram[1]['insta_TSCORE']
tot_CSCORE += returnValue_instagram[1]['insta_CSCORE']
tot_MSCORE += returnValue_instagram[1]['insta_MSCORE']
print('instagram : ', tot_TSCORE, tot_CSCORE, tot_MSCORE)
else:
print('인스타그램 크롤링봇의 내부 요인에 의해 결과 값이 제대로 전달되지 않았습니다.')
else:
print('인스타그램 크롤링을 위한 계정정보가 존재하지 않습니다.')
# 사용자 등급 : userSnsRank
if (tot_TSCORE <= 600 and tot_TSCORE > 500) and (
tot_CSCORE <= 1000
and tot_CSCORE > 850) and (tot_MSCORE <= 400
and tot_MSCORE > 350):
userSNSRank = 'A+'
elif (tot_TSCORE <= 500 and tot_TSCORE > 400) and (
tot_CSCORE <= 850
and tot_CSCORE > 700) and (tot_MSCORE <= 350
and tot_MSCORE > 300):
userSNSRank = 'A-'
elif (tot_TSCORE <= 400 and tot_TSCORE > 350) and (
tot_CSCORE <= 700
and tot_CSCORE > 550) and (tot_MSCORE <= 300
and tot_MSCORE > 250):
userSNSRank = 'B+'
elif (tot_TSCORE <= 350 and tot_TSCORE > 300) and (
tot_CSCORE <= 550
and tot_CSCORE > 400) and (tot_MSCORE <= 250
and tot_MSCORE > 200):
userSNSRank = 'B-'
elif (tot_TSCORE <= 300 and tot_TSCORE > 250) and (
tot_CSCORE <= 400
and tot_CSCORE > 300) and (tot_MSCORE <= 200
and tot_MSCORE > 150):
userSNSRank = 'C+'
elif (tot_TSCORE <= 250 and tot_TSCORE > 200) and (
tot_CSCORE <= 300
and tot_CSCORE > 200) and (tot_MSCORE <= 150
and tot_MSCORE > 100):
userSNSRank = 'C-'
elif (tot_TSCORE <= 200 and tot_TSCORE > 100) and (
tot_CSCORE <= 200
and tot_CSCORE > 100) and (tot_MSCORE <= 100
and tot_MSCORE > 50):
userSNSRank = 'D+'
elif (tot_TSCORE <= 100 and tot_TSCORE > 0) and (
tot_CSCORE <= 100 and
tot_CSCORE > 0) and (tot_MSCORE <= 50 and tot_MSCORE > 1):
userSNSRank = 'D-'
print('userSNS_Rank :', userSNSRank)
# DB insert
try:
# Server Connection to MySQL
databaseConnection = mysqlConnection.DatabaseConnection_origin(
)
databaseConnection.update_totalTCM_Record(
str(tot_TSCORE), str(tot_CSCORE), str(tot_MSCORE),
userSNSRank, facebook_USERURL)
except Exception as e_maria:
print('[ Error ] MariaDB About information Insertion :',
e_maria)
else:
print("전달받은 데이터가 없습니다. ")