def parse(self, response): symbol = (get_url_param(response.url, 'page_index'), get_url_host(response.url), response.url) self.logger.info('Parsing No.%s Page %s Overdue Info From <%s>.' % symbol) try: content = json.loads(response.body_as_unicode()) if int(content.get('result_code', 0)) != 1: raise ValueError except Exception: self.logger.warning('Response Error In No.%s Page %s Overdue Info From <%s>.' % symbol) return None item_list = [] for dy in content.get('data', []): item = YuqiItem() item['plat_id'] = self.plat_id item['plat_name'] = self.plat_name item['user_id'] = dy.get('user_id') item['username'] = dy.get('username') item['idcard'] = dy.get('idcard') item['overdue_count'] = dy.get('overdue_count') item['overdue_total'] = dy.get('overdue_total') item['overdue_principal'] = dy.get('overdue_principal') item['payment_total'] = dy.get('payment_total') item['payment_count'] = dy.get('payment_count') item['payment_period'] = dy.get('payment_period') item['repay_amount'] = dy.get('repay_amount') item['wait_amount'] = dy.get('wait_amount') log_empty_fields(item, self.logger) item_list.append(item) return item_list
def parse(self, response): symbol = (get_url_param(response.url, 'page_index'), get_url_host(response.url), \ get_url_param(response.url, 'id'), response.url) self.logger.info('Parsing No.%s Page %s Invest Info About %s BidId From <%s>.' % symbol) try: content = json.loads(response.body_as_unicode()) if int(content.get('result_code', 0)) != 1: raise ValueError except Exception: self.logger.info('Response Error In No.%s Page %s Invest Info About %s BidId From <%s>.' % symbol) return None item_list = [] for dt in content.get('data', []): item = ToubiaoItem() item['invest_id'] = dt.get('invest_id') item['bid_id'] = dt.get('id') item['plat_id'] = self.plat_id item['plat_name'] = self.plat_name item['user_id'] = dt.get('user_id') item['username'] = dt.get('username') item['amount'] = dt.get('amount') item['valid_amount'] = dt.get('valid_amount') item['add_date'] = dt.get('add_date') item['status'] = dt.get('status') item['type'] = dt.get('type') item['url'] = dt.get('url') log_empty_fields(item, self.logger) item_list.append(item) return item_list
def parse(self, response): symbol = (self.plat_id, get_url_host(response.url), response.url) self.logger.info('Parsing No.%s [%s] Plat Page Count From <%s>.' % symbol) item = ExporterItem() try: content = json.loads(response.body_as_unicode()) #content = {'result': '1', 'data': {'token': 'yamiedie'}} if int(content.get('result_code', 0)) != 1: raise ValueError except Exception as e: self.logger.warning('Fail To Receive No.%s [%s] Plat Page Count From <%s>.' % symbol) return None item.set_record(content.get('page_count', 0)) return item
def parse(self, response): symbol = (self.plat_id, get_url_host(response.url), response.url) self.logger.info('Parsing No.%s [%s] Plat Login Info From <%s>.' % symbol) try: content = json.loads(response.body_as_unicode()) #content = {'result': '1', 'data': {'token': 'yamiedie'}} if int(content.get('result', 0)) != 1: raise ValueError except Exception: self.logger.warning('Fail To Receive No.%s [%s] Plat Login Info From <%s>.' % symbol) return None item = ExporterItem() item.set_record(content.get('data', {}).get('token')) item.set_record(json.dumps(response.headers.getlist('Set-Cookie'))) return item
def parse(self, response): symbol = (get_url_param(response.url, 'page_index'), get_url_host(response.url), response.url) self.logger.info('Parsing No.%s Page %s Loan Info From <%s>.' % symbol) try: content = json.loads(response.body_as_unicode()) if int(content.get('result_code', 0)) != 1: raise ValueError except Exception: self.logger.warning( 'Response Error In No.%s Page %s Loan Info From <%s>.' % symbol) return None item_list = [] for dj in content.get('data', []): item = JiekuanItem() item['bid_id'] = dj.get('id') item['plat_id'] = self.plat_id item['plat_name'] = self.plat_name item['status'] = get_url_param(response.url, 'status') item['title'] = dj.get('title') item['amount'] = dj.get('amount') item['process'] = dj.get('process') item['interest_rate'] = dj.get('interest_rate') item['borrow_period'] = dj.get('borrow_period') item['borrow_unit'] = dj.get('borrow_unit') item['reward'] = dj.get('reward') item['type'] = dj.get('type') item['repay_type'] = dj.get('repay_type') item['username'] = dj.get('username') item['user_id'] = dj.get('user_id') item['user_avatar_url'] = dj.get('user_avatar_url') item['province'] = dj.get('province') item['city'] = dj.get('city') item['borrow_detail'] = dj.get('borrow_detail') item['url'] = dj.get('url') item['success_time'] = dj.get('success_time') item['publish_time'] = dj.get('publish_time') item['invest_count'] = dj.get('invest_count') log_empty_fields(item, self.logger) item_list.append(item) return item_list
def parse(self, response): symbol = (self.plat_id, get_url_host(response.url), response.url) self.logger.info('Parsing No.%s [%s] Plat Page Count From <%s>.' % symbol) item = ExporterItem() try: content = json.loads(response.body_as_unicode()) #content = {'result': '1', 'data': {'token': 'yamiedie'}} if int(content.get('result_code', 0)) != 1: raise ValueError except Exception as e: self.logger.warning( 'Fail To Receive No.%s [%s] Plat Page Count From <%s>.' % symbol) return None item.set_record(content.get('page_count', 0)) return item
def parse(self, response): symbol = (get_url_param(response.url, 'page_index'), get_url_host(response.url), response.url) self.logger.info('Parsing No.%s Page %s Loan Info From <%s>.' % symbol) try: content = json.loads(response.body_as_unicode()) if int(content.get('result_code', 0)) != 1: raise ValueError except Exception: self.logger.warning('Response Error In No.%s Page %s Loan Info From <%s>.' % symbol) return None item_list = [] for dj in content.get('data', []): item = JiekuanItem() item['bid_id'] = dj.get('id') item['plat_id'] = self.plat_id item['plat_name'] = self.plat_name item['status'] = get_url_param(response.url, 'status') item['title'] = dj.get('title') item['amount'] = dj.get('amount') item['process'] = dj.get('process') item['interest_rate'] = dj.get('interest_rate') item['borrow_period'] = dj.get('borrow_period') item['borrow_unit'] = dj.get('borrow_unit') item['reward'] = dj.get('reward') item['type'] = dj.get('type') item['repay_type'] = dj.get('repay_type') item['username'] = dj.get('username') item['user_id'] = dj.get('user_id') item['user_avatar_url'] = dj.get('user_avatar_url') item['province'] = dj.get('province') item['city'] = dj.get('city') item['borrow_detail'] = dj.get('borrow_detail') item['url'] = dj.get('url') item['success_time'] = dj.get('success_time') item['publish_time'] = dj.get('publish_time') item['invest_count'] = dj.get('invest_count') log_empty_fields(item, self.logger) item_list.append(item) return item_list
def parse(self, response): symbol = (self.plat_id, get_url_host(response.url), response.url) self.logger.info('Parsing No.%s [%s] Plat Login Info From <%s>.' % symbol) try: content = json.loads(response.body_as_unicode()) #content = {'result': '1', 'data': {'token': 'yamiedie'}} if int(content.get('result', 0)) != 1: raise ValueError except Exception: self.logger.warning( 'Fail To Receive No.%s [%s] Plat Login Info From <%s>.' % symbol) return None item = ExporterItem() item.set_record(content.get('data', {}).get('token')) item.set_record(json.dumps(response.headers.getlist('Set-Cookie'))) return item
def parse(self, response): symbol = (self.plat_id, get_url_host(response.url), response.url) self.logger.info('Parsing No.%s [%s] Plat Login Info From <%s>.' % symbol) try: content = json.loads(response.body_as_unicode()) self.logger.info(content) if int(content.get('result', 0)) != 1: raise ValueError except Exception: self.logger.warning( 'Fail To Receive No.%s [%s] Plat Login Info From <%s>.' % symbol) return None item = ExporterItem() item.set_record(content.get('data', {}).get('token')) print content.get('data', {}).get('token') return item
def parse(self, response): symbol = (get_url_param(response.url, 'page_index'), get_url_host(response.url), response.url) self.logger.info('Parsing No.%s Page %s Overdue Info From <%s>.' % symbol) try: content = json.loads(response.body_as_unicode()) if int(content.get('result_code', 0)) != 1: raise ValueError except Exception: self.logger.warning( 'Response Error In No.%s Page %s Overdue Info From <%s>.' % symbol) return None item_list = [] for dy in content.get('data', []): item = YuqiItem() item['plat_id'] = self.plat_id item['plat_name'] = self.plat_name item['user_id'] = dy.get('user_id') item['username'] = dy.get('username') item['idcard'] = dy.get('idcard') item['overdue_count'] = dy.get('overdue_count') item['overdue_total'] = dy.get('overdue_total') item['overdue_principal'] = dy.get('overdue_principal') item['payment_total'] = dy.get('payment_total') item['payment_count'] = dy.get('payment_count') item['payment_period'] = dy.get('payment_period') item['repay_amount'] = dy.get('repay_amount') item['wait_amount'] = dy.get('wait_amount') log_empty_fields(item, self.logger) item_list.append(item) return item_list
def parse(self, response): symbol = (get_url_param(response.url, 'page_index'), get_url_host(response.url), \ get_url_param(response.url, 'id'), response.url) self.logger.info( 'Parsing No.%s Page %s Invest Info About %s BidId From <%s>.' % symbol) try: content = json.loads(response.body_as_unicode()) if int(content.get('result_code', 0)) != 1: raise ValueError except Exception: self.logger.info( 'Response Error In No.%s Page %s Invest Info About %s BidId From <%s>.' % symbol) return None item_list = [] for dt in content.get('data', []): item = ToubiaoItem() item['invest_id'] = dt.get('invest_id') item['bid_id'] = dt.get('id') item['plat_id'] = self.plat_id item['plat_name'] = self.plat_name item['user_id'] = dt.get('user_id') item['username'] = dt.get('username') item['amount'] = dt.get('amount') item['valid_amount'] = dt.get('valid_amount') item['add_date'] = dt.get('add_date') item['status'] = dt.get('status') item['type'] = dt.get('type') item['url'] = dt.get('url') log_empty_fields(item, self.logger) item_list.append(item) return item_list