def download(obj): logger.debug('download url: '+obj['url']) try: headers = { 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.107 Safari/537.36', 'Proxy-Connection': 'keep-alive', 'Accept-Language': 'en-US,en;q=0.8,zh-CN;q=0.6', 'Accept-Encoding': 'gzip,deflate,sdch', 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8', 'Cache-Control': 'max-age=0', 'Cookie': 'UTMPKEY=14367518; UTMPNUM=13792; UTMPUSERID=guest; LOGINTIME=1396536377' } proxies = None if gConfig["downloadUsingProxy"]["enabled"]: proxies = gConfig["downloadUsingProxy"]["proxies"] r = requests.get(obj['url'], headers=headers, proxies=proxies, timeout=gConfig['downloadTimeout']) r.raise_for_status() # 如果响应状态码不是 200,就主动抛出异常 except requests.Timeout as e: logger.warning('download file timeout! file url: '+obj['url']) except Exception as e: logger.exception(e.message) # namespace.emit('downloadError', obj) return None else: # save file if(obj.get('saveat') == 'upyun'): return UpyunUtil.uploadStream(r.content, obj) else: saveurl = writeFile(obj['savename'], obj.get('savedir'), r.content, 'wb') obj['saveurl'] = saveurl return obj
def uploadStream(content, obj): if not UpyunUtil.up: UpyunUtil.up = upyun.UpYun(UpyunUtil._spaceName, UpyunUtil._userName, UpyunUtil._userPasswd) savedir = os.path.join(UpyunUtil._rootDir, obj.get('savedir')) if obj.get('savedir') else UpyunUtil._rootDir savename = os.path.join(savedir, obj['savename']) try: res = UpyunUtil.up.put(savename, content) except Exception as e: logger.exception(e.message) # namespace.emit('downloadError', obj) return None else: logger.info(simplejson.dumps(res, indent=4, ensure_ascii=False)) obj['saveurl'] = UpyunUtil._rootUrl+savename # namespace.emit('uploadUpyun', obj) return obj