def download(url):
url = tran2UTF8(url)
lasterr = ''
for i in range(HTTP_GET_RETRY_NUM):
try:
c = getclient()
if c is not None:
buf = StringIO.StringIO()
c.setopt(pycurl.WRITEFUNCTION, buf.write)
# pycurl url is in str method otherwise unicode
c.setopt(pycurl.URL, url.encode('utf-8'))
c.perform()
status_code = c.getinfo(pycurl.HTTP_CODE)
if status_code == 200:
ret = buf.getvalue()
# debug reserve code
# logging.debug('down time total: %sms, connect: %sms, pretran: %sms' % (
# round(c.getinfo(pycurl.TOTAL_TIME)*1000, 2),
# round(c.getinfo(pycurl.CONNECT_TIME)*1000, 2),
# round(c.getinfo(pycurl.PRETRANSFER_TIME)*1000, 2),
# ))
# round(c.getinfo(pycurl.REDIRECT_TIME)*1000, 2)
return ret
else:
lasterr = 'status code %s' % c.getinfo(pycurl.HTTP_CODE)
c.close()
buf.close()
else:
lasterr = 'pycurl client get false'
except Exception:
lasterr = traceback.format_exc()
time.sleep(HTTP_RETRY_SLEEP_TIME)
logging.error('exception occurs when get url[%s]. [%s]' % (url, lasterr))
return None
def download(url):
# request url must in utf-8 code, in unicode or str does not import
start_url = url[:3]
if start_url == 'htt':
return get(tran2UTF8(url))
elif start_url == 'ftp':
return ftpdown(url)
else:
logging.error('unknown url format[%s].' % (url))
return None
def data_mapping(row):
return {
'cmd': 'addinfo',
'magic': 'hfrz',
'id': row[0],
'deviceId': row[1],
'snapshotTime': time_mapping(row[2]),
'ImageURL': url_mapping(row[3]) if row[3] and row[3] != '0' else None,
'plateNumber': tran2UTF8(row[4]) if row[4] and row[4] != '0' else '',
}
def download(url):
req = urllib2.Request(tran2UTF8(url))
lasterr = ''
for i in range(HTTP_GET_RETRY_NUM):
try:
resp = urllib2.urlopen(req, timeout=HTTP_GET_TIMEOUT)
if resp.getcode() == 200:
ret = resp.read()
resp.close()
return ret
else:
lasterr = 'status code %s' % resp.status_code
except Exception:
lasterr = traceback.format_exc()
time.sleep(HTTP_RETRY_SLEEP_TIME)
logging.error('exception occurs when get url[%s]. [%s]' % (url, lasterr))
return None
def post(url, param):
url = tran2UTF8(url)
param_json = param and json.dumps(param)
header = ['Content-Type:application/json']
lasterr = ''
for i in range(HTTP_POST_RETRY_NUM):
try:
c = getclient(method='post')
if c is not None:
buf = StringIO.StringIO()
c.setopt(pycurl.HTTPHEADER, header)
c.setopt(pycurl.POSTFIELDS, param_json)
c.setopt(pycurl.WRITEFUNCTION, buf.write)
c.setopt(pycurl.URL, url)
c.perform()
status_code = c.getinfo(pycurl.HTTP_CODE)
if status_code == 200:
ret = buf.getvalue()
return ret
else:
lasterr = 'status code %s' % c.getinfo(pycurl.HTTP_CODE)
c.close()
buf.close()
else:
lasterr = 'pycurl client get false'
except Exception:
lasterr = traceback.format_exc()
time.sleep(HTTP_POST_RETRY_NUM)
if param and param.has_key('imageData'):
del param['imageData']
logging.error('exception occurs when post url[%s], param[%s]. [%s]' %
(url, json.dumps(param), lasterr))
else:
logging.error('exception occurs when post url[%s], param[%s]. [%s]' %
(url, param_json, lasterr))
return None
def url_mapping(url):
# all to str and in utf-8
if url[:3] == 'ftp':
return url
else:
return tran2UTF8(url)