continentList = codeList[2] siteList = codeList[3] # 2014. 8. 3. site 추가 if len(cityList) == 0 and len(nationList) == 0 and len(continentList) == 0: codeList = codes.getCityCode(cityClass.city) cityList = codeList[0] nationList = codeList[1] continentList = codeList[2] siteList = codeList[3] # 2014. 8. 3. site 추가 query = tourQuery.getMasterMergeQuery(tourAgency, productList.productCode, productList.productname, level2.tourkind, 'A', productList.comment, '') # A : 해외(Abroad) #print query cursor = con.cursor() cursor.execute(query) con.commit() codes.insertRegionData(tourAgency, productList.productCode, cityList, nationList, continentList, siteList) detailProductHtml = urllib2.urlopen(detailProductUrl).read() detailProductHtml = detailProductHtml[detailProductHtml.find('<tbody id'):detailProductHtml.find('<p class="seeMore"')] detailProductHtml = detailProductHtml.replace('<td class=', '\r\n<td class=') detailProductHtml = savefilegethtml.htmlToList(detailProductHtml, 'detailProductHtml.txt') try: detailProductCls = clsDetailProduct() waitSeat = False for detailProduct in detailProductHtml: #print 'Detail Product : ' + detailProduct if detailProduct.find('startDate">') > -1: detailProductCls = clsDetailProduct()
if len(cityList) == 0 and len(nationList) == 0 and len(continentList) == 0 and len(siteList) == 0: codeList = codes.getCityCode(detail.name) cityList = codeList[0] nationList = codeList[1] continentList = codeList[2] siteList = codeList[3] # 2014. 8. 3. site 추가 # Master 상품 입력 query = tourQuery.getMasterMergeQuery(tourAgency, productCls.productCode, productCls.productName.decode('utf-8'), mainMenu.tourType, mainMenu.dmst_div, productCls.description.decode('utf-8'), '') #print query cursor = con.cursor() cursor.execute(query) con.commit() # Region Data 삭제 codes.insertRegionData(tourAgency, productCls.productCode, cityList, nationList, continentList, siteList) # 2014. 8. 3. site 추가 detailProductHtml = savefilegethtml.getHtml(productCls.detailUrl, 'class="list"', '', 'onlinetourDetailProductHtml.txt') detailProductCls = clsDetailProduct() chkPrice = False for detailInfo in detailProductHtml: try: if detailInfo.find('airline_cd') > -1: detailProductCls = clsDetailProduct() if detailInfo.find('비') > -1 or detailInfo.find('티') > -1: detailProductCls.airCode = '' else: detailProductCls.airCode = tourUtil.getTagAttr(detailInfo, 'tr', 'airline_cd') #print 'airCode : ', detailProductCls.airCode elif detailInfo.find('class="start"') > -1:
continentList = codeList[2] siteList = codeList[ 3] # 2014. 8. 3. site 추가 # Master 상품 입력 query = tourQuery.getMasterMergeQuery( tourAgency, productCode, productName, tourtype, 'A', productComment, '') #print query cursor = con.cursor() cursor.execute(query) con.commit() # Region Data 삭제 codes.insertRegionData( tourAgency, productCode, cityList, nationList, continentList, siteList) if not tree['ModeSangPum'].has_key( 'SangList'): continue for t in tree['ModeSangPum'][ 'SangList']: reg_div = anCode prd_nm = t['SName'][ '#text'].replace("'", "") air_cd = t['SAirCode'][:2] st_dt = t['SPriceDay']['#text'] st_time = t[ 'SstartTime'].replace(
if len(cityList) == 0 and len(nationList) == 0 and len(continentList) == 0: codeList = codes.getCityCode(sub2package.menuName) cityList = codeList[0] nationList = codeList[1] continentList = codeList[2] siteList = codeList[3] # 2014. 8. 3. site 추가 query = tourQuery.getMasterMergeQuery(tourAgency, detailProduct, productNameList[codeIdx], packageMap[package.menuCode], dmst_div, productCommentList[codeIdx], '') print query codeIdx += 1 cursor = con.cursor() cursor.execute(query) con.commit() codes.insertRegionData(tourAgency, detailProduct, cityList, nationList, continentList, siteList) flag = False clsProduct = classProduct() for parcer in detailProductList: print parcer try: if parcer.strip()[:len('<td><input type="checkbox"')] == '<td><input type="checkbox"': flag = True clsProduct = classProduct() if flag: if parcer.find('<td><span class="blue">') > -1: spliter = parcer.strip().split(' ') clsProduct.departDay = spliter[1].split('>')[1].replace('/', '').strip() clsProduct.departTime = spliter[3].split('<')[0].replace(':', '').strip()
query = tourQuery.getMasterMergeQuery( tourAgency, detailProduct, productNameList[codeIdx], packageMap[package.menuCode], dmst_div, productCommentList[codeIdx], '') print query codeIdx += 1 cursor = con.cursor() cursor.execute(query) con.commit() codes.insertRegionData( tourAgency, detailProduct, cityList, nationList, continentList, siteList) flag = False clsProduct = classProduct() for parcer in detailProductList: print parcer try: if parcer.strip()[:len( '<td><input type="checkbox"' )] == '<td><input type="checkbox"': flag = True clsProduct = classProduct() if flag: if parcer.find(
#if len(nationList) == 0 and len(cityList) == 0: #print >> exceptFile, packageClass.pub_city + ' : ', valueParcing(product, 'mst_name":"', '","t_content').replace("'", "").decode('utf-8') #print >> exceptFile, 'detailProductUrl : ' + detailProductUrl dmst_div = 'A' if mode == 'D': dmst_div = 'D' # Master 상품 입력 query = tourQuery.getMasterMergeQuery(tourAgency, productClass.pkg_mst_code, productClass.mst_name, mode, dmst_div, productClass.content, '') #print query cursor = con.cursor() cursor.execute(query) con.commit() # Region Data 삭제 codes.insertRegionData(tourAgency, productClass.pkg_mst_code, cityList, nationList, continentList, siteList) detailProducthtml = urllib2.urlopen(detailProductUrl).read() #cont":[{"pcode":"PPP411140612KE5","sdate":"06/12 (목) 20:50","adate":"06/16 (월) 08:05","acode":"KE","aline":"대한항공","tday":"5","grade":"12","gname":"하나팩클래식","pname":"팔라우 5일[Luxury]팔라우퍼시픽리조트[용궁+유네스코+젤리피쉬]","amt":"1999000","lminute":"2"}, if detailProducthtml.find('[{"pcode"') < 0: continue #detailProductList = savefilegethtml.getHtmlList(detailProducthtml, '{"pcode"', '] })', 'detailProductFile.txt', '{', '\r\n{') temp = detailProducthtml[detailProducthtml.find('[{"pcode"') + 1:detailProducthtml.find('] })')].replace('{', '\r\n{') #detailProductList = savefilegethtml.htmlToList(temp, 'detailProductFile.txt') detailProductFilename = savefilegethtml.chkExistFile('detailProductFile.txt') detailProductFile = open(detailProductFilename, 'w') print >> detailProductFile, temp detailProductFile.close()
def insertData(productCls, detailUrl, regionUrl, tourAgency, kind, dmst_div): print 'Product Url : ', productCls.url print >> exceptFile, 'Product Url : ', productCls.url # 2014. 7. 23. 카테고리의 국가는 넣지 않기로 함... #codeList = codes.getCityCode(productCls.name.decode('utf-8'), detailUrl.name.decode('utf-8'), regionUrl.name.decode('utf-8')) codeList = codes.getCityCode(productCls.name.decode('utf-8')) cityList = codeList[0] nationList = codeList[1] continentList = codeList[2] siteList = codeList[3] # 2014. 8. 3. site 추가 if len(cityList) == 0 and len(nationList) == 0 and len(continentList) == 0: codeList = codes.getCityCode(detailUrl.name.decode('utf-8')) cityList = codeList[0] nationList = codeList[1] continentList = codeList[2] siteList = codeList[3] # 2014. 8. 3. site 추가 # Master 상품 입력 query = tourQuery.getMasterMergeQuery(tourAgency, productCls.code, productCls.name.decode('utf-8'), menu.kind, dmst_div, '', '') #print query cursor = con.cursor() cursor.execute(query) con.commit() # Region Data 삭제 codes.insertRegionData(tourAgency, productCls.code, cityList, nationList, continentList, siteList) detailProductHtml = savefilegethtml.getHtml(productCls.url, '', '', 'tour2000DetailHtml'+targetMonth+'.txt') pl10Idx = 0 for detailProduct in detailProductHtml: try: if detailProduct.find('<span class="text_pink">') > -1 and detailProduct.find('<a href=') < 0: detailCls = clsProductDetail() numArray = tourUtil.getNumArray(detailProduct) if len(numArray) > 7: detailCls.dDay = targetYear + numArray[0] + numArray[1] detailCls.dTime = numArray[2] + numArray[3] detailCls.aDay = targetYear + numArray[4] + numArray[5] detailCls.aTime = numArray[6] + numArray[7] elif len(numArray) == 4: detailCls.dDay = targetYear + numArray[0] + numArray[1] detailCls.dTime = '' detailCls.aDay = targetYear + numArray[2] + numArray[3] detailCls.aTime = '' elif detailProduct.find('onError') > -1: detailCls.airCode = detailProduct[detailProduct.find('.gif') - 4:detailProduct.find('.gif') - 2] elif detailProduct.find('text_redB') > -1: numArray = tourUtil.getNumArray(tourUtil.getRemovedHtmlTag(detailProduct)) for num in numArray: detailCls.price += num elif detailProduct.find('</a></td>') > -1: if detailProduct.find('text_pink') > -1: detailCls.status = codes.getStatus('tour2000', '예약가능') elif detailProduct.find('text_blau') > -1: detailCls.status = codes.getStatus('tour2000', '출발가능') elif detailProduct.find('text_green') > -1: detailCls.status = codes.getStatus('tour2000', '대기예약') elif detailProduct.find('text_grayLightSmall') > -1: detailCls.status = codes.getStatus('tour2000', '예약마감') detailCls.remainSeat = tourUtil.getRemovedHtmlTag(detailProduct).replace("'", "").strip() elif detailProduct.find('<p class="pl10">') > -1: if pl10Idx == 0: pl10Idx = 1 detailCls.productName = tourUtil.getRemovedHtmlTag(detailProduct).replace("'", "").strip() detailCls.url = mainUrl + tourUtil.getTagAttr(detailProduct, 'a', 'href') detailCls.productSeq = detailProduct.split('ev_ym=')[1].split('&')[0] + detailProduct.split('ev_seq=')[1].split('&')[0] else: pl10Idx = 0 if detailCls.productName.find('부산출발') > -1: departCity = 'PUS' else: departCity = 'ICN' query = tourQuery.getDetailMergeQuery(tourAgency, productCls.code, detailCls.productSeq, detailCls.productName.decode('utf-8'), detailCls.dDay+detailCls.dTime, detailCls.aDay+detailCls.aTime, productCls.period, departCity, '', detailCls.airCode, detailCls.status, detailCls.url, detailCls.price, '0', '0', '0', '', productCls.night) #print >> exceptFile, query #print query cursor = con.cursor() cursor.execute(query) con.commit() #break except: print >> exceptFile, 'detail parcing Error : ', sys.exc_info()[0] pass
def insertData(productCls, detailUrl, regionUrl, tourAgency, kind, dmst_div): print 'Product Url : ', productCls.url print >> exceptFile, 'Product Url : ', productCls.url # 2014. 7. 23. 카테고리의 국가는 넣지 않기로 함... #codeList = codes.getCityCode(productCls.name.decode('utf-8'), detailUrl.name.decode('utf-8'), regionUrl.name.decode('utf-8')) codeList = codes.getCityCode(productCls.name.decode('utf-8')) cityList = codeList[0] nationList = codeList[1] continentList = codeList[2] siteList = codeList[3] # 2014. 8. 3. site 추가 if len(cityList) == 0 and len(nationList) == 0 and len(continentList) == 0: codeList = codes.getCityCode(detailUrl.name.decode('utf-8')) cityList = codeList[0] nationList = codeList[1] continentList = codeList[2] siteList = codeList[3] # 2014. 8. 3. site 추가 # Master 상품 입력 query = tourQuery.getMasterMergeQuery(tourAgency, productCls.code, productCls.name.decode('utf-8'), menu.kind, dmst_div, '', '') #print query cursor = con.cursor() cursor.execute(query) con.commit() # Region Data 삭제 codes.insertRegionData(tourAgency, productCls.code, cityList, nationList, continentList, siteList) detailProductHtml = savefilegethtml.getHtml( productCls.url, '', '', 'tour2000DetailHtml' + targetMonth + '.txt') pl10Idx = 0 for detailProduct in detailProductHtml: try: if detailProduct.find( '<span class="text_pink">') > -1 and detailProduct.find( '<a href=') < 0: detailCls = clsProductDetail() numArray = tourUtil.getNumArray(detailProduct) if len(numArray) > 7: detailCls.dDay = targetYear + numArray[0] + numArray[1] detailCls.dTime = numArray[2] + numArray[3] detailCls.aDay = targetYear + numArray[4] + numArray[5] detailCls.aTime = numArray[6] + numArray[7] elif len(numArray) == 4: detailCls.dDay = targetYear + numArray[0] + numArray[1] detailCls.dTime = '' detailCls.aDay = targetYear + numArray[2] + numArray[3] detailCls.aTime = '' elif detailProduct.find('onError') > -1: detailCls.airCode = detailProduct[detailProduct.find('.gif') - 4:detailProduct.find('.gif' ) - 2] elif detailProduct.find('text_redB') > -1: numArray = tourUtil.getNumArray( tourUtil.getRemovedHtmlTag(detailProduct)) for num in numArray: detailCls.price += num elif detailProduct.find('</a></td>') > -1: if detailProduct.find('text_pink') > -1: detailCls.status = codes.getStatus('tour2000', '예약가능') elif detailProduct.find('text_blau') > -1: detailCls.status = codes.getStatus('tour2000', '출발가능') elif detailProduct.find('text_green') > -1: detailCls.status = codes.getStatus('tour2000', '대기예약') elif detailProduct.find('text_grayLightSmall') > -1: detailCls.status = codes.getStatus('tour2000', '예약마감') detailCls.remainSeat = tourUtil.getRemovedHtmlTag( detailProduct).replace("'", "").strip() elif detailProduct.find('<p class="pl10">') > -1: if pl10Idx == 0: pl10Idx = 1 detailCls.productName = tourUtil.getRemovedHtmlTag( detailProduct).replace("'", "").strip() detailCls.url = mainUrl + tourUtil.getTagAttr( detailProduct, 'a', 'href') detailCls.productSeq = detailProduct.split( 'ev_ym=')[1].split('&')[0] + detailProduct.split( 'ev_seq=')[1].split('&')[0] else: pl10Idx = 0 if detailCls.productName.find('부산출발') > -1: departCity = 'PUS' else: departCity = 'ICN' query = tourQuery.getDetailMergeQuery( tourAgency, productCls.code, detailCls.productSeq, detailCls.productName.decode('utf-8'), detailCls.dDay + detailCls.dTime, detailCls.aDay + detailCls.aTime, productCls.period, departCity, '', detailCls.airCode, detailCls.status, detailCls.url, detailCls.price, '0', '0', '0', '', productCls.night) #print >> exceptFile, query #print query cursor = con.cursor() cursor.execute(query) con.commit() #break except: print >> exceptFile, 'detail parcing Error : ', sys.exc_info()[0] pass
nationList = codeList[1] continentList = codeList[2] siteList = codeList[ 3] # 2014. 8. 3. site 추가 query = tourQuery.getMasterMergeQuery( tourAgency, mastercode, productName, tourType, region, productComment, '') # A : 해외(Abroad) #query = savefilegethtml.getMasterMergeQuery('vgtour', mastercode, '', '', productGroupCls.name, productName, tourType, region, productComment, '') # A : 해외(Abroad) #print query cursor = con.cursor() cursor.execute(query) con.commit() codes.insertRegionData(tourAgency, mastercode, cityList, nationList, continentList, siteList) #최종 상품들 잡아넣자.. try: productCls = clsProduct() #productListHtml = open('productListHtml.txt') departConfirm = False for product in productListHtml: #print 'product : ' + product if product.find('pro_date') > -1: productCls = clsProduct() departConfirm = False #productCls.sDay = targetYear + product.split('pro_date">')[1].split('(')[0].strip().replace('/', '') #productCls.sTime = product.split('<br/>')[0].split(')')[1].strip().replace(':', '') #productCls.aDay = targetYear + product.split('<span>')[1].split('(')[0].strip().replace('/', '')
siteList = codeList[3] # 2014. 8. 3. site 추가 if len(cityList) == 0 and len(nationList) == 0 and len(continentList) == 0: codeList = codes.getCityCode(productGroupCls.name) cityList = codeList[0] nationList = codeList[1] continentList = codeList[2] siteList = codeList[3] # 2014. 8. 3. site 추가 query = tourQuery.getMasterMergeQuery(tourAgency, mastercode, productName, tourType, region, productComment, '') # A : 해외(Abroad) #query = savefilegethtml.getMasterMergeQuery('vgtour', mastercode, '', '', productGroupCls.name, productName, tourType, region, productComment, '') # A : 해외(Abroad) #print query cursor = con.cursor() cursor.execute(query) con.commit() codes.insertRegionData(tourAgency, mastercode, cityList, nationList, continentList, siteList) #최종 상품들 잡아넣자.. try: productCls = clsProduct() #productListHtml = open('productListHtml.txt') departConfirm = False for product in productListHtml: #print 'product : ' + product if product.find('pro_date') > -1: productCls = clsProduct() departConfirm = False #productCls.sDay = targetYear + product.split('pro_date">')[1].split('(')[0].strip().replace('/', '') #productCls.sTime = product.split('<br/>')[0].split(')')[1].strip().replace(':', '') #productCls.aDay = targetYear + product.split('<span>')[1].split('(')[0].strip().replace('/', '')