示例#1
0
 def update_price_code(cls):
     merge_codes = DataService.select_merge_codes()
     for item in merge_codes:
         train_code = item[0]
         merge_code = item[1]
         log.info('update price code ,%s,%s'% (train_code,merge_code))
         DataService.update_price_code(merge_code,train_code)
示例#2
0
 def parse_content(self, content, link_job):
     content = content[content.index('(') + 1:content.rindex(')')]
     json_obj = json.loads(content)
     trains = json_obj['data']['trains']
     for train in trains:
         try:
             train_code = train['trainNum']
             start_station = train['fromCity']
             end_station = train['toCity']
             origin = train['beginPlace']
             terminal = train['endPlace']
             depart_time = train['fromTime']
             arrive_time = train['toTime']
             duration = int(train['usedTimeInt']) * 60
             note = train['note']
             A1 = self.get_price('hardseat', train['ticketState'])
             A2 = self.get_price('softseat', train['ticketState'])
             A3 = self.get_price('hardsleepermid', train['ticketState'])
             A4 = self.get_price('softsleeperdown', train['ticketState'])
             A6 = self.get_price('advancedsoftsleeper', train['ticketState'])
             A9 = self.get_price('businessseat', train['ticketState'])
             O = self.get_price('secondseat', train['ticketState'])
             M = self.get_price('firstseat', train['ticketState'])
             P = self.get_price('specialseat', train['ticketState'])
             sequence = 0
             days = 0
             stayTime = 0
             grade = ''
             state = 0
             train_no = ''
             exist = DataService.check_traincode_exist(train_code)
             # is_correct 0:错误信息 1:正确
             is_correct = 1
             if len(train['ticketState']) == 0:
                 is_correct = 0
             # state 0:正常   1:未收录此车次    2:收录此车次但是此站点已经取消
             if not exist:
                 state = 1
             else:
                 station_s = DataService.find_station(train_code, start_station)
                 station_e = DataService.find_station(train_code, end_station)
                 if len(station_e) != 0 and len(station_s) != 0:
                     days_s = int(station_s[0][6])
                     if self.compareSS(station_s[0][4], station_s[0][8]) < 0:
                         days_s += 1
                     sequence = station_e[0][3]
                     days = station_e[0][6] - days_s
                     stayTime = station_e[0][5]
                     grade = station_e[0][9]
                     train_no = station_e[0][10]
                 else:
                     state = 2
             price = Price(train_code, end_station, start_station, depart_time, arrive_time, duration, A1, A2, A3,
                           A4, O, M, A6, A9, grade, days, P, origin, terminal, sequence, train_no, stayTime,
                           is_correct, state, note)
             DataService.save_train_price(price)
         except:
             t, v, tb = sys.exc_info()
             log.error("%s,%s,%s" % (t, v, traceback.format_tb(tb)))
 def update_price_code(cls):
     merge_codes = DataService.select_merge_codes()
     for item in merge_codes:
         train_code = item[0]
         merge_code = item[1]
         log.info('update price code ,%s,%s' % (train_code, merge_code))
         try:
             DataService.update_price_code(merge_code, train_code)
         except:
             t, v, tb = sys.exc_info()
             log.error("%s,%s,%s" % (t, v, traceback.format_tb(tb)))
示例#4
0
 def parse_content(self, content, link_job):
     json_obj = json.loads(content)
     list = json_obj['list']
     for shop in list:
         try:
             category_id = shop['categoryId']
             city_id = shop['cityId']
             shop_id = shop['id']
             url = self.shop_url_pattern % shop_id
             DataService.save_dp_shop(url, shop_id, city_id, category_id)
         except:
             t, v, tb = sys.exc_info()
             log.error("%s,%s,%s" % (t, v, traceback.format_tb(tb)))
示例#5
0
 def add_merge_train_code(cls):
     merge_codes = DataService.select_merge_codes()
     for item in merge_codes:
         train_code = item[0]
         merge_code = item[1]
         # exist = DataService.check_traincode_exist(merge_code)
         # if exist:
         #     continue
         stops = DataService.get_stops(train_code)
         log.info("insert merge_train_stop:%s" % merge_code)
         for stop in stops:
             train_stop = TrainStop(merge_code,stop[2],stop[3],stop[4],stop[5],stop[6],stop[7],stop[8],stop[9],stop[10])
             DataService.save_train_stop(train_stop)
 def start_merge(cls):
     code_dict = {}
     rmv_zero = r'^(0+)'
     stop_no = DataService.get_stop_no()
     for stop in stop_no:
         train_code = stop[0]
         train_no = stop[1]
         substring_str = re.sub(rmv_zero, "", train_no[2:len(train_no) - 2])
         DataService.insert_merge_stop(train_code, train_no, substring_str)
         log.info("insert:%s,%s,%s" % (train_code, train_no, substring_str))
         if substring_str in code_dict:
             temp_code = code_dict[substring_str]
             temp_code = "%s/%s" % (temp_code, train_code)
             code_dict[substring_str] = temp_code
         else:
             code_dict[substring_str] = train_code
     for sub_str in code_dict:
         DataService.update_merge_stop(code_dict[sub_str], sub_str)
示例#7
0
 def __check_proxy_queue(self):
     proxy_jobs = DataService.select_proxy()
     DataService.update_proxys()
     for proxy in proxy_jobs:
         proxy_job = ProxyJob(proxy[1], proxy[2])
         self.crawler.proxy_queue.put_checked_proxy(proxy_job)