def cate_link_read_main():
global lib_host_http
global lib_conn_http
rows = db_lib.db_get_g(db_sql.sql_lib_lang_cate_link_read_get, ())
for row in rows:
lang_href = row[0]
cate_path = row[1]
cate_param = row[2]
link_href = row[3]
if lib_host_http != lang_href.replace('http://', '').replace('/', '').strip():
lib_host_http = lang_href.replace('http://', '').replace('/', '').strip()
lib_conn_http = http.get_conn_http(lib_host_http)
try:
cate_link_read(link_href, lang_href, cate_path, cate_param)
util.sleep_i(1)
except Exception as e:
err.except_p(e)
def home_read_main():
global lib_host_http
global lib_conn_http
rows = db_lib.db_get_g(db_sql.sql_lib_lang_get, ())
for row in rows:
lang_href = row[0]
lang_title = row[1]
lib_host_http = lang_href.replace('http://', '').replace('/', '').strip()
lib_conn_http = http.get_conn_http(lib_host_http)
''' # if working on horizon pc, uncommit this if and commit above two lines.
if lib_host_http != lang_href.replace('http://', '').replace('/', '').strip():
lib_host_http = lang_href.replace('http://', '').replace('/', '').strip()
lib_conn_http = http.get_conn_http(lib_host_http)
'''
try:
home_read(lang_href)
db_lib.db_execute_g(db_sql.sql_lib_lang_update, (lang_href,))
util.sleep_i(1)
except Exception as e:
err.except_p(e)
def cate_read_main():
global lib_host_http
global lib_conn_http
rows = db_lib.db_get_g(db_sql.sql_lib_lang_cate_read_get, ())
i_t = len(rows)
i = 0
for row in rows:
i = i + 1
print '%d of %d'%(i, i_t),
lang_href = row[0]
cate_path = row[1]
cate_param = row[2]
if lib_host_http != lang_href.replace('http://', '').replace('/', '').strip():
lib_host_http = lang_href.replace('http://', '').replace('/', '').strip()
lib_conn_http = http.get_conn_http(lib_host_http)
try:
cate_read(lang_href, cate_path, cate_param)
util.sleep_i(1)
except Exception as e:
err.except_p(e)
def home_read_max_main():
global lib_host_http
global lib_conn_http
rows = db_lib.db_get_g(db_sql.sql_lib_lang_cate_get, ())
for row in rows:
lang_href = row[0]
cate_path = row[1]
cate_title = row[2]
cate_param_max = row[3]
lib_host_http = lang_href.replace('http://', '').replace('/', '').strip()
lib_conn_http = http.get_conn_http(lib_host_http)
''' # if working on horizon pc, uncommit this if, otherwise, commit above two lines.
if lib_host_http != lang_href.replace('http://', '').replace('/', '').strip():
lib_host_http = lang_href.replace('http://', '').replace('/', '').strip()
lib_conn_http = http.get_conn_http(lib_host_http)
'''
try:
home_read_max(lang_href, cate_path, cate_title)
except Exception as e:
err.except_p(e)
print '======='
def youtube_http_post(url, url_body):
global youtube_conn_http
if youtube_conn_http == None:
youtube_conn_http = http.get_conn_http(youtube_host_http)
status, body, youtube_conn_http = http.use_httplib_http(url, 'POST', url_body, youtube_conn_http, youtube_host_http, youtube_headers_http)
return status, body
def youtube_http_get(url):
global youtube_conn_http
if youtube_conn_http == None:
youtube_conn_http = http.get_conn_http(youtube_host_http)
status, body, youtube_conn_http = http.use_httplib_http(url, 'GET', '', youtube_conn_http, youtube_host_http, youtube_headers_http)
return status, body
import db_sql
import db_app
import http
import urllib
import err
from bs4 import BeautifulSoup
from datetime import datetime
import util
youtube_host_http = 'www.youtube.com'
youtube_conn_http = http.get_conn_http(youtube_host_http)
youtube_headers_http = {"Content-type": "application/x-www-form-urlencoded", "Accept": "text/plain", "Accept-Language": "en-UK"}
youtube_root = 'watch'
######### http without proxy
def youtube_http_get(url):
global youtube_conn_http
if youtube_conn_http == None:
youtube_conn_http = http.get_conn_http(youtube_host_http)
status, body, youtube_conn_http = http.use_httplib_http(url, 'GET', '', youtube_conn_http, youtube_host_http, youtube_headers_http)
return status, body
def youtube_http_post(url, url_body):
global youtube_conn_http
if youtube_conn_http == None:
youtube_conn_http = http.get_conn_http(youtube_host_http)
status, body, youtube_conn_http = http.use_httplib_http(url, 'POST', url_body, youtube_conn_http, youtube_host_http, youtube_headers_http)
return status, body
'''
# http proxy here: if not in proxy, please comment thsese lines
youtube_conn_http = http.get_conn_http_proxy(youtube_host_http)
def host_http_proxy ():