Ejemplo n.º 1
0
    def do_job(self, job_data):
        try:
            args = json.loads(job_data)

            with open(args["log_file"], "at") as lf:
                lf.write("===new task===\n[%s]start import site:%s's data...\n" % (NowTime(), args["site"]))
                lf.close()

            # import
            start_date, end_date = import_csv_card_all(
                args["site"],
                args["all_file"],
                args["card_file"],
                args["location"],
                args["location_desc"],
                args["userid"],
                args["username"],
                args["station_id"],
            )

            with open(args["log_file"], "at") as lf:
                lf.write("[%s]end import site:%s's data...\n" % (NowTime(), args["site"]))
                lf.close()

            # need update station info?
            if args["need_update_station_info"]:
                with open(args["log_file"], "at") as lf:
                    lf.write("[%s]start update site:%s info...\n" % (NowTime(), args["site"]))
                    lf.close()

                nb_guns = get_nb_guns_of_station(args["site"])
                guns_id = get_guns_id_by_site(args["site"])
                update_station_info(
                    args["site"],
                    args["site_desc"],
                    args["locid"],
                    nb_guns,
                    None,  # phone
                    None,  # address
                    None,  # brand
                    None,  # distance
                    guns_id,
                )

                with open(args["log_file"], "at") as lf:
                    lf.write("[%s]end update site:%s info...\n" % (NowTime(), args["site"]))
                    lf.close()

            # 计算相关性
            with open(args["log_file"], "at") as lf:
                lf.write("[%s]start compute assoc on site:%s...\n" % (NowTime(), args["site"]))
                lf.close()

            call_command("compute_item_assoc", interactive=False, period=0, site=args["site"])
            call_command("compute_item_assoc", interactive=False, period=1, site=args["site"])
            call_command("compute_item_assoc", interactive=False, period=2, site=args["site"])
            call_command("compute_item_assoc", interactive=False, period=3, site=args["site"])

            with open(args["log_file"], "at") as lf:
                lf.write("[%s]%s\n" % (NowTime(), args["import_flag_string"]))
                lf.write("[%s]end compute assoc on site:%s...\n===finish task===\n" % (NowTime(), args["site"]))
                lf.close()

            # 离线计算油品每天销售
            args["start_date"] = start_date.strftime("%Y-%m-%d")
            args["end_date"] = end_date.strftime("%Y-%m-%d")
            call_command("gearman_submit_job", "compute_fuel_daybatch", json.dumps(args), foreground=False)
            call_command("gearman_submit_job", "compute_station_daybatch", json.dumps(args), foreground=False)
        except Exception, e:
            with open(args["log_file"], "at") as lf:
                lf.write("[%s]%s\n" % (NowTime(), args["import_flag_string"]))
                lf.write("[%s]handle task error:%s\n===finish task===\n" % (NowTime(), str(e)))
                lf.close()
            gearman_logger.error("gearman error: " + str(e))
            gearman_logger.error("stack:" + exception_stuck())
    def do_job(self, job_data):
        try:
            args=json.loads(job_data)

            with open(args['log_file'],'at') as lf:
                lf.write('===new task===\n[%s]start import site:%s\'s data...\n'%(
                    NowTime(),args['site']
                ))
                lf.close()
            #import
            start_date,end_date=import_ycshell_data(args['site'],
                                args['file'],
                                args['location'],
                                args['location_desc'],
                                args['userid'],
                                args['username'],
                                args['station_id'])

            with open(args['log_file'],'at') as lf:
                lf.write('[%s]end import site:%s\'s data...\n'%(
                    NowTime(),args['site']
                ))
                lf.close()

            #need update station info?
            if args['need_update_station_info']:
                with open(args['log_file'],'at') as lf:
                    lf.write('[%s]start update site:%s info...\n'%(
                        NowTime(),args['site']
                    ))
                    lf.close()

                nb_guns=get_nb_guns_of_station(args['site'])
                guns_id=get_guns_id_by_site(args['site'])
                update_station_info(
                    args['site'],
                    args['site_desc'],
                    args['locid'],
                    nb_guns,
                    None,#phone
                    None,#address
                    None,#brand
                    None,#distance
                    guns_id
                    )

                with open(args['log_file'],'at') as lf:
                    lf.write('[%s]end update site:%s info...\n'%(
                        NowTime(),args['site']
                    ))
                    lf.close()

            #计算相关性
            with open(args['log_file'],'at') as lf:
                lf.write('[%s]start compute assoc on site:%s...\n'%(
                    NowTime(),args['site']
                ))
                lf.close()

            call_command('compute_item_assoc',interactive=False,period=0,site=args['site'])
            call_command('compute_item_assoc',interactive=False,period=1,site=args['site'])
            call_command('compute_item_assoc',interactive=False,period=2,site=args['site'])
            call_command('compute_item_assoc',interactive=False,period=3,site=args['site'])

            with open(args['log_file'],'at') as lf:
                lf.write('[%s]%s\n' %(NowTime(),args['import_flag_string']))
                lf.write('[%s]end compute assoc on site:%s...\n===finish task===\n'%(
                    NowTime(),args['site']
                ))
                lf.close()

            #离线计算油品每天销售
            args['start_date']=start_date.strftime('%Y-%m-%d')
            args['end_date']=end_date.strftime('%Y-%m-%d')
            call_command('gearman_submit_job', 'compute_fuel_daybatch',
                json.dumps(args), foreground=False)
            call_command('gearman_submit_job', 'compute_station_daybatch',
                json.dumps(args), foreground=False)
        except Exception,e:
            with open(args['log_file'],'at') as lf:
                lf.write('[%s]%s\n' %(NowTime(),args['import_flag_string']))
                lf.write('[%s]handle task error:%s\n===finish task===\n'%(
                    NowTime(),str(e)
                ))
                lf.close()
            gearman_logger.error('gearman error: '+str(e))
            gearman_logger.error('stack:'+exception_stuck())
Ejemplo n.º 3
0
    def handle(self,  *args, **options):
        self.form=options

        #取得location的所有site列表
        self.sites=get_all_sites_by_location_id(int(self.form['location']))

        type=options['type']

        #check type
        if type not in all_type_args.keys():
            print 'ERROR:type not right'
            return

        #check args
        need_args=all_type_args[type]
        for need_arg in need_args:
            if options[need_arg] is None:
                print 'ERROR:'+type+' need arg '+need_arg
                return

        s_list=[]
        already_shard_ids=[]
        for site in self.sites:
            shard_id,sm=get_dash_session_maker(site,with_shard_id=True)
            if shard_id in already_shard_ids:
                continue
            already_shard_ids.append(shard_id)
            s_list.append(sm())

        if type=='pump_daily_avg_report':
            conditions = self.build_conditions()
            conditions.append(self.cube.d.trans_type == TransType.FUEL)

            results=[]
            for s in s_list:
                results += self.cube.aggregate(measures=['quantity'],
                                          drilldown=['barcode', 'month', 'day'],
                                          conditions=conditions,
                                          session=s)

            stats = {}
            categories = set()
            for result in results:
                barcode = result['barcode']
                stats.setdefault(barcode, {})
                cat = '%02d-%02d' % (result['month'], result['day'])
                categories.add(cat)
                stat = stats[barcode]
                stat[cat] = round(result['quantity'] / self.site_count, 2)

            categories = list(categories)
            categories.sort()
            data = {
                "categories": categories,
                "dataset": [],
            }
            for series, stat in stats.items():
                opt = {"data": [], "name": get_fuel_type_name(series), "type": "area", "stacking": "normal"}
                for cat in data['categories']:
                    opt["data"].append(stat.get(cat, 0))
                data["dataset"].append(opt)

            cache_key='%s_%s'%(type,'-'.join(all_type_args[type]))
            for arg in all_type_args[type]:
                cache_key+='_'+str(options[arg])

            cache_value=json.dumps(data)
            cache.set(cache_key,cache_value,settings.MEMCACHED_TIMEOUT)

        elif type=='gun_pump_avg_report':
            conditions = self.build_conditions()
            conditions.append(self.cube.d.trans_type == TransType.FUEL)
            #统计出油量,多少天数
            #group by 每个站,每小时
            results=[]
            for s in s_list:
                results += self.cube.aggregate(measures=['quantity','hours'],
                                          drilldown=['hour', 'site'],
                                          conditions=conditions,
                                          session=s)
            stats = {}
            counter_dic={}
            for result in results:
                hour = result['hour']
                site = result['site']
                cat = '%02d:00 - %02d:00' % (hour, hour + 1)
                stats.setdefault(cat, 0)
                counter_dic.setdefault(cat,0)
                #不同的站
                #加上枪效率,出油量是在多少枪,多少天完成的,需要平均下
                stats[cat]+=result['quantity'] / 40 / get_nb_guns_of_station(site) / result['hours']
                counter_dic[cat]+=1

            #平均到每个站
            for cat in stats:
                stats[cat] = round(stats[cat]/counter_dic[cat], 2)

            data = {
                "categories": ['%02d:00 - %02d:00' % (i, i + 1) for i in range(0, 24)],
                "dataset": [],
            }
            opt = {"data": [], "name": "油枪平均每小时工作时间", "type": "column"}
            for cat in data['categories']:
                opt["data"].append(stats.get(cat, 0))
            data["dataset"].append(opt)
            cache_key='%s_%s'%(type,'-'.join(all_type_args[type]))
            for arg in all_type_args[type]:
                cache_key+='_'+str(options[arg])

            cache_value=json.dumps(data)
            cache.set(cache_key,cache_value,settings.MEMCACHED_TIMEOUT)

        elif type=='non_fuel_sales_avg_report':
            conditions = self.build_conditions()
            conditions.append(self.cube.d.trans_type == TransType.NON_FUEL)
            conditions.append(self.cube.d.quantity > 0)

            results=[]
            for s in s_list:
                results += self.cube.aggregate(measures=['pay', 'hours'],
                                          drilldown=['hour'],
                                          conditions=conditions,
                                          session=s)
            stats = {}
            for result in results:
                hour = result['hour']
                cat = '%02d:00 - %02d:00' % (hour, hour + 1)
                stats[cat] = round(result['pay'] / (self.site_count * result['hours']), 2)

            data = {
                "categories": ['%02d:00 - %02d:00' % (i, i + 1) for i in range(0, 24)],
                "dataset": [],
            }
            opt = {"data": [], "name": "非油品平均每小时销售额", "type": "column"}
            for cat in data['categories']:
                opt["data"].append(stats.get(cat, 0))
            data["dataset"].append(opt)

            cache_key='%s_%s'%(type,'-'.join(all_type_args[type]))
            for arg in all_type_args[type]:
                cache_key+='_'+str(options[arg])

            cache_value=json.dumps(data)
            cache.set(cache_key,cache_value,settings.MEMCACHED_TIMEOUT)

        elif type=='non_fuel_sales_scale_report':
            conditions = self.build_conditions()
            conditions2 = conditions[:]
            conditions2.append(self.cube.d.trans_type == TransType.NON_FUEL)
            conditions2.append(self.cube.d.quantity > 0)

            results=[]
            for s in s_list:
                results += self.cube.aggregate(measures=['trans_count', 'hours'],
                                          drilldown=['hour'],
                                          conditions=conditions2,
                                          session=s)
            stats2 = {}
            for result in results:
                hour = result['hour']
                cat = '%02d:00 - %02d:00' % (hour, hour + 1)
                stats2[cat] = float(result['trans_count']) / result['hours']

            conditions.append(self.cube.d.trans_type == TransType.FUEL)

            results=[]
            for s in s_list:
                results += self.cube.aggregate(measures=['trans_count', 'hours'],
                                          drilldown=['hour'],
                                          conditions=conditions,
                                          session=s)
            stats = {}
            for result in results:
                hour = result['hour']
                cat = '%02d:00 - %02d:00' % (hour, hour + 1)
                stats[cat] = float(result['trans_count']) / result['hours']


            data = {
                "categories": ['%02d:00 - %02d:00' % (i, i + 1) for i in range(0, 24)],
                "dataset": [],
            }

            for cat in data['categories']:
                total = stats.get(cat, 0) + stats2.get(cat, 0)
                if total > 0:
                    stats[cat] = round(stats2.get(cat, 0) * 100 / total, 2)
                else:
                    stats[cat] = 0

            opt = {"data": [], "name": "非油品平均每小时交易比例", "type": "column"}
            for cat in data['categories']:
                opt["data"].append(stats.get(cat, 0))
            data["dataset"].append(opt)

            cache_key='%s_%s'%(type,'-'.join(all_type_args[type]))
            for arg in all_type_args[type]:
                cache_key+='_'+str(options[arg])

            cache_value=json.dumps(data)
            cache.set(cache_key,cache_value,settings.MEMCACHED_TIMEOUT)

        elif type=='top10_non_fuel_sales_avg_report':
            self.build_conditions()
            location = int(options['location'])

            conditions = [self.cube.d.quantity > 0,
                          self.cube.d.trans_type == TransType.NON_FUEL]
            if location > 0:
                conditions.append(self.cube.d.location == location)

            results=[]
            for s in s_list:
                results += self.cube.aggregate(measures=['pay'],
                                          drilldown=['barcode'],
                                          details=['desc'],
                                          conditions=conditions,
                                          order='pay desc',
                                          limit=10,
                                          session=s)
            barcodes = []
            for result in results:
                barcodes.append(result['barcode'])

            conditions = self.build_conditions()
            conditions.append(self.cube.d.trans_type == TransType.NON_FUEL)
            conditions.append(self.cube.d.barcode.in_(barcodes))
            conditions.append(self.cube.d.quantity > 0)

            results=[]
            for s in s_list:
                results += self.cube.aggregate(measures=['pay'],
                                          drilldown=['barcode', 'hour'],
                                          details=['desc'],
                                          conditions=conditions,
                                          session=s)
            stats = {}
            descs = {}
            for result in results:
                barcode = result['barcode']
                descs[barcode] = result['desc']
                stats.setdefault(barcode, {})
                stat = stats[barcode]
                hour = result['hour']
                cat = '%02d:00 - %02d:00' % (hour, hour + 1)
                stat[cat] = round(result['pay'] / (373 * 24 * self.site_count), 2)

            data = {
                "categories": ['%02d:00 - %02d:00' % (i, i + 1) for i in range(0, 24)],
                "dataset": [],
            }
            for series, stat in stats.items():
                opt = {"data": [], "name": descs[series], "type": "line"}
                for cat in data['categories']:
                    opt["data"].append(stat.get(cat, 0))
                data["dataset"].append(opt)

            cache_key='%s_%s'%(type,'-'.join(all_type_args[type]))
            for arg in all_type_args[type]:
                cache_key+='_'+str(options[arg])

            cache_value=json.dumps(data)
            cache.set(cache_key,cache_value,settings.MEMCACHED_TIMEOUT)


        elif type=='pump_hourly_avg_report':
            conditions = self.build_conditions()
            conditions.append(self.cube.d.trans_type == TransType.FUEL)

            results=[]
            for s in s_list:
                results += self.cube.aggregate(measures=['quantity','hours'],
                                          drilldown=['barcode', 'hour'],
                                          conditions=conditions,
                                          session=s)
            stats = {}
            categories = set()
            for result in results:
                barcode = result['barcode']
                stats.setdefault(barcode, {})
                hour = result['hour']
                cat = '%02d:00 - %02d:00' % (hour, hour + 1)
                categories.add(cat)
                stat = stats[barcode]
                stat[cat] = round(result['quantity'] / (result['hours'] * self.site_count), 2)

            data = {
                "categories": ['%02d:00 - %02d:00' % (i, i + 1) for i in range(0, 24)],
                "dataset": [],
            }
            for series, stat in stats.items():
                opt = {"data": [], "name": get_fuel_type_name(series), "type": "area", "stacking": "normal"}
                for cat in data['categories']:
                    opt["data"].append(stat.get(cat, 0))
                data["dataset"].append(opt)

            cache_key='%s_%s'%(type,'-'.join(all_type_args[type]))
            for arg in all_type_args[type]:
                cache_key+='_'+str(options[arg])

            cache_value=json.dumps(data)
            cache.set(cache_key,cache_value,settings.MEMCACHED_TIMEOUT)

        elif type=="pump_monthly_avg_report":
            conditions = self.build_conditions()
            conditions.append(self.cube.d.trans_type == TransType.FUEL)

            results=[]
            for s in s_list:
                s=get_dash_session_maker(site)()
                results += self.cube.aggregate(measures=['quantity'],
                                          drilldown=['barcode', 'month'],
                                          conditions=conditions,
                                          session=s)

            stats = {}
            for result in results:
                barcode = result['barcode']
                stats.setdefault(barcode, {})
                stat = stats[barcode]
                stat[result['month']] = round(result['quantity'] / self.site_count, 2)

            data = {
                "categories": [i for i in range(1, 13)],
                "dataset": [],
            }
            for series, stat in stats.items():
                opt = {"data": [], "name": get_fuel_type_name(series), "type": "area", "stacking": "normal"}
                for cat in data['categories']:
                    opt["data"].append(stat.get(cat, 0))
                data["dataset"].append(opt)

            cache_key='%s_%s'%(type,'-'.join(all_type_args[type]))
            for arg in all_type_args[type]:
                cache_key+='_'+str(options[arg])

            cache_value=json.dumps(data)
            cache.set(cache_key,cache_value,settings.MEMCACHED_TIMEOUT)

        elif type=='trans_count_avg_report':
            conditions = self.build_conditions()
            conditions.append(self.cube.d.trans_type == TransType.FUEL)

            results=[]
            for s in s_list:
                s=get_dash_session_maker(site)()
                results += self.cube.aggregate(measures=['trans_count','hours'],
                                          drilldown=['hour'],
                                          conditions=conditions,
                                          session=s)

            stats = {}
            for result in results:
                hour = result['hour']
                cat = '%02d:00 - %02d:00' % (hour, hour + 1)
                stats[cat] = int(result['trans_count'] / (self.site_count * result['hours']))

            data = {
                "categories": ['%02d:00 - %02d:00' % (i, i + 1) for i in range(0, 24)],
                "dataset": [],
            }
            opt = {"data": [], "name": "进站车辆", "type": "column"}
            for cat in data['categories']:
                opt["data"].append(stats.get(cat, 0))
            data["dataset"].append(opt)

            cache_key='%s_%s'%(type,'-'.join(all_type_args[type]))
            for arg in all_type_args[type]:
                cache_key+='_'+str(options[arg])

            cache_value=json.dumps(data)
            cache.set(cache_key,cache_value,settings.MEMCACHED_TIMEOUT)

        #close session
        for s in s_list:
            s.close()