def get_file(url, js=None):
if js and 'js' in js[0]:
file = jspagedeal.getpage(url)
else:
req = requests.get(url, headers={'User-Agent' : "fake-clients"})
#get real url
file = req.content
return file
def get_file_bak(url, js=None):
if js and 'js' in js[0]:
file = jspagedeal.getpage(url)
else:
req = urllib2.Request(url, headers={'User-Agent' : "Mozilla/5.0"})
response = urllib2.urlopen(req, timeout = 15)
#get real url
file = response.read()
return file