def parse(self, response):
symbol = (get_url_param(response.url, 'page_index'), get_url_host(response.url), response.url)
self.logger.info('Parsing No.%s Page %s Overdue Info From <%s>.' % symbol)
try:
content = json.loads(response.body_as_unicode())
if int(content.get('result_code', 0)) != 1:
raise ValueError
except Exception:
self.logger.warning('Response Error In No.%s Page %s Overdue Info From <%s>.' % symbol)
return None
item_list = []
for dy in content.get('data', []):
item = YuqiItem()
item['plat_id'] = self.plat_id
item['plat_name'] = self.plat_name
item['user_id'] = dy.get('user_id')
item['username'] = dy.get('username')
item['idcard'] = dy.get('idcard')
item['overdue_count'] = dy.get('overdue_count')
item['overdue_total'] = dy.get('overdue_total')
item['overdue_principal'] = dy.get('overdue_principal')
item['payment_total'] = dy.get('payment_total')
item['payment_count'] = dy.get('payment_count')
item['payment_period'] = dy.get('payment_period')
item['repay_amount'] = dy.get('repay_amount')
item['wait_amount'] = dy.get('wait_amount')
log_empty_fields(item, self.logger)
item_list.append(item)
return item_list
def parse(self, response):
symbol = (get_url_param(response.url, 'page_index'), get_url_host(response.url), \
get_url_param(response.url, 'id'), response.url)
self.logger.info('Parsing No.%s Page %s Invest Info About %s BidId From <%s>.' % symbol)
try:
content = json.loads(response.body_as_unicode())
if int(content.get('result_code', 0)) != 1:
raise ValueError
except Exception:
self.logger.info('Response Error In No.%s Page %s Invest Info About %s BidId From <%s>.' % symbol)
return None
item_list = []
for dt in content.get('data', []):
item = ToubiaoItem()
item['invest_id'] = dt.get('invest_id')
item['bid_id'] = dt.get('id')
item['plat_id'] = self.plat_id
item['plat_name'] = self.plat_name
item['user_id'] = dt.get('user_id')
item['username'] = dt.get('username')
item['amount'] = dt.get('amount')
item['valid_amount'] = dt.get('valid_amount')
item['add_date'] = dt.get('add_date')
item['status'] = dt.get('status')
item['type'] = dt.get('type')
item['url'] = dt.get('url')
log_empty_fields(item, self.logger)
item_list.append(item)
return item_list
def parse(self, response):
symbol = (self.plat_id, get_url_host(response.url), response.url)
self.logger.info('Parsing No.%s [%s] Plat Page Count From <%s>.' % symbol)
item = ExporterItem()
try:
content = json.loads(response.body_as_unicode())
#content = {'result': '1', 'data': {'token': 'yamiedie'}}
if int(content.get('result_code', 0)) != 1:
raise ValueError
except Exception as e:
self.logger.warning('Fail To Receive No.%s [%s] Plat Page Count From <%s>.' % symbol)
return None
item.set_record(content.get('page_count', 0))
return item
def parse(self, response):
symbol = (self.plat_id, get_url_host(response.url), response.url)
self.logger.info('Parsing No.%s [%s] Plat Login Info From <%s>.' % symbol)
try:
content = json.loads(response.body_as_unicode())
#content = {'result': '1', 'data': {'token': 'yamiedie'}}
if int(content.get('result', 0)) != 1:
raise ValueError
except Exception:
self.logger.warning('Fail To Receive No.%s [%s] Plat Login Info From <%s>.' % symbol)
return None
item = ExporterItem()
item.set_record(content.get('data', {}).get('token'))
item.set_record(json.dumps(response.headers.getlist('Set-Cookie')))
return item
def parse(self, response):
symbol = (get_url_param(response.url, 'page_index'),
get_url_host(response.url), response.url)
self.logger.info('Parsing No.%s Page %s Loan Info From <%s>.' % symbol)
try:
content = json.loads(response.body_as_unicode())
if int(content.get('result_code', 0)) != 1:
raise ValueError
except Exception:
self.logger.warning(
'Response Error In No.%s Page %s Loan Info From <%s>.' %
symbol)
return None
item_list = []
for dj in content.get('data', []):
item = JiekuanItem()
item['bid_id'] = dj.get('id')
item['plat_id'] = self.plat_id
item['plat_name'] = self.plat_name
item['status'] = get_url_param(response.url, 'status')
item['title'] = dj.get('title')
item['amount'] = dj.get('amount')
item['process'] = dj.get('process')
item['interest_rate'] = dj.get('interest_rate')
item['borrow_period'] = dj.get('borrow_period')
item['borrow_unit'] = dj.get('borrow_unit')
item['reward'] = dj.get('reward')
item['type'] = dj.get('type')
item['repay_type'] = dj.get('repay_type')
item['username'] = dj.get('username')
item['user_id'] = dj.get('user_id')
item['user_avatar_url'] = dj.get('user_avatar_url')
item['province'] = dj.get('province')
item['city'] = dj.get('city')
item['borrow_detail'] = dj.get('borrow_detail')
item['url'] = dj.get('url')
item['success_time'] = dj.get('success_time')
item['publish_time'] = dj.get('publish_time')
item['invest_count'] = dj.get('invest_count')
log_empty_fields(item, self.logger)
item_list.append(item)
return item_list
def parse(self, response):
symbol = (self.plat_id, get_url_host(response.url), response.url)
self.logger.info('Parsing No.%s [%s] Plat Page Count From <%s>.' %
symbol)
item = ExporterItem()
try:
content = json.loads(response.body_as_unicode())
#content = {'result': '1', 'data': {'token': 'yamiedie'}}
if int(content.get('result_code', 0)) != 1:
raise ValueError
except Exception as e:
self.logger.warning(
'Fail To Receive No.%s [%s] Plat Page Count From <%s>.' %
symbol)
return None
item.set_record(content.get('page_count', 0))
return item
def parse(self, response):
symbol = (get_url_param(response.url, 'page_index'), get_url_host(response.url), response.url)
self.logger.info('Parsing No.%s Page %s Loan Info From <%s>.' % symbol)
try:
content = json.loads(response.body_as_unicode())
if int(content.get('result_code', 0)) != 1:
raise ValueError
except Exception:
self.logger.warning('Response Error In No.%s Page %s Loan Info From <%s>.' % symbol)
return None
item_list = []
for dj in content.get('data', []):
item = JiekuanItem()
item['bid_id'] = dj.get('id')
item['plat_id'] = self.plat_id
item['plat_name'] = self.plat_name
item['status'] = get_url_param(response.url, 'status')
item['title'] = dj.get('title')
item['amount'] = dj.get('amount')
item['process'] = dj.get('process')
item['interest_rate'] = dj.get('interest_rate')
item['borrow_period'] = dj.get('borrow_period')
item['borrow_unit'] = dj.get('borrow_unit')
item['reward'] = dj.get('reward')
item['type'] = dj.get('type')
item['repay_type'] = dj.get('repay_type')
item['username'] = dj.get('username')
item['user_id'] = dj.get('user_id')
item['user_avatar_url'] = dj.get('user_avatar_url')
item['province'] = dj.get('province')
item['city'] = dj.get('city')
item['borrow_detail'] = dj.get('borrow_detail')
item['url'] = dj.get('url')
item['success_time'] = dj.get('success_time')
item['publish_time'] = dj.get('publish_time')
item['invest_count'] = dj.get('invest_count')
log_empty_fields(item, self.logger)
item_list.append(item)
return item_list
def parse(self, response):
symbol = (self.plat_id, get_url_host(response.url), response.url)
self.logger.info('Parsing No.%s [%s] Plat Login Info From <%s>.' %
symbol)
try:
content = json.loads(response.body_as_unicode())
#content = {'result': '1', 'data': {'token': 'yamiedie'}}
if int(content.get('result', 0)) != 1:
raise ValueError
except Exception:
self.logger.warning(
'Fail To Receive No.%s [%s] Plat Login Info From <%s>.' %
symbol)
return None
item = ExporterItem()
item.set_record(content.get('data', {}).get('token'))
item.set_record(json.dumps(response.headers.getlist('Set-Cookie')))
return item
def parse(self, response):
symbol = (self.plat_id, get_url_host(response.url), response.url)
self.logger.info('Parsing No.%s [%s] Plat Login Info From <%s>.' %
symbol)
try:
content = json.loads(response.body_as_unicode())
self.logger.info(content)
if int(content.get('result', 0)) != 1:
raise ValueError
except Exception:
self.logger.warning(
'Fail To Receive No.%s [%s] Plat Login Info From <%s>.' %
symbol)
return None
item = ExporterItem()
item.set_record(content.get('data', {}).get('token'))
print content.get('data', {}).get('token')
return item
def parse(self, response):
symbol = (get_url_param(response.url, 'page_index'),
get_url_host(response.url), response.url)
self.logger.info('Parsing No.%s Page %s Overdue Info From <%s>.' %
symbol)
try:
content = json.loads(response.body_as_unicode())
if int(content.get('result_code', 0)) != 1:
raise ValueError
except Exception:
self.logger.warning(
'Response Error In No.%s Page %s Overdue Info From <%s>.' %
symbol)
return None
item_list = []
for dy in content.get('data', []):
item = YuqiItem()
item['plat_id'] = self.plat_id
item['plat_name'] = self.plat_name
item['user_id'] = dy.get('user_id')
item['username'] = dy.get('username')
item['idcard'] = dy.get('idcard')
item['overdue_count'] = dy.get('overdue_count')
item['overdue_total'] = dy.get('overdue_total')
item['overdue_principal'] = dy.get('overdue_principal')
item['payment_total'] = dy.get('payment_total')
item['payment_count'] = dy.get('payment_count')
item['payment_period'] = dy.get('payment_period')
item['repay_amount'] = dy.get('repay_amount')
item['wait_amount'] = dy.get('wait_amount')
log_empty_fields(item, self.logger)
item_list.append(item)
return item_list
def parse(self, response):
symbol = (get_url_param(response.url, 'page_index'), get_url_host(response.url), \
get_url_param(response.url, 'id'), response.url)
self.logger.info(
'Parsing No.%s Page %s Invest Info About %s BidId From <%s>.' %
symbol)
try:
content = json.loads(response.body_as_unicode())
if int(content.get('result_code', 0)) != 1:
raise ValueError
except Exception:
self.logger.info(
'Response Error In No.%s Page %s Invest Info About %s BidId From <%s>.'
% symbol)
return None
item_list = []
for dt in content.get('data', []):
item = ToubiaoItem()
item['invest_id'] = dt.get('invest_id')
item['bid_id'] = dt.get('id')
item['plat_id'] = self.plat_id
item['plat_name'] = self.plat_name
item['user_id'] = dt.get('user_id')
item['username'] = dt.get('username')
item['amount'] = dt.get('amount')
item['valid_amount'] = dt.get('valid_amount')
item['add_date'] = dt.get('add_date')
item['status'] = dt.get('status')
item['type'] = dt.get('type')
item['url'] = dt.get('url')
log_empty_fields(item, self.logger)
item_list.append(item)
return item_list