コード例 #1
0
ファイル: DailyDecode.py プロジェクト: SolarHalo/gcp
 def __init__(self,conf,filename,source):
     self.conf = conf
     self.filename = filename
     self.source = source
     
     configs = Config.getConfig()
     self.batchSize = configs['sys']['db.batch.size']
コード例 #2
0
ファイル: UrlParserHandler.py プロジェクト: SolarHalo/gcp
 def execute(self):
     configs = Config.getConfig()
     self.logger.info("Ready to parser url !")
     
     urlTemples = []
     
     for key in configs['game.alawar'].keys():
         if key.startswith('url'):
             attr = key.split(".")
             table = attr[1]
             locale = attr[2]
             urlTemple = configs['game.alawar'][key]
             urlTemples.append([urlTemple,table,locale])
     
     for urlTemple in urlTemples:
         if self.table is not None and urlTemple[1] != self.table:
             continue
         
         cf = configs['game.alawar']
         urls = StaticUtil.convertUrl(urlTemple, cf)
         self.logger.info("Parser xml convert count[%d]!" % len(urls));
         for conf in urls:
             filepath = os.path.join(StaticUtil.mkdir(configs['game.alawar']['data_path'], conf),
                                     StaticUtil.getTimeStrForFold() + "alawar.xml")
             #filepath = Config.configs['game.alawar']['data_path'] +StaticUtil.getPara(conf)+ StaticUtil.getTimeStrForFold() + "alawar.xml"
             download = DownloadHandler(conf, filepath, GcpConstant.Source.Alawar)
             TaskRun.getInstance().submit(download)
             
             self.logger.info("Parser xml[%s] name , url[%s] end !"%(filepath,conf));
コード例 #3
0
ファイル: BigfishDecode.py プロジェクト: SolarHalo/gcp
 def __init__(self,conf,filename,source):
     self.conf = conf
     self.filename = filename
     self.source = source
     
     configs = Config.getConfig()
     self.batchSize = configs['sys']['db.batch.size']
     self.sitePrix = configs['game.bigfish']['dtd'][1]
     self.assets =  configs['game.bigfish']['dtd'][2]
コード例 #4
0
ファイル: Dbutil.py プロジェクト: SolarHalo/gcp
 def __init__(self):  
     self.cursor = None
     configs = Config.getConfig()
     self.host = configs['db']['db_host']
     self.user = configs['db']['db_user']
     self.pwd = configs['db']['db_pass']
     self.db = configs['db']['db_db']
     self.conn = None
     try:
         self.conn = MySQLdb.Connect(host=self.host, user=self.user, passwd=self.pwd, db=self.db, charset="utf8", use_unicode="True") 
     except MySQLdb.Error, e:
         print e
         print "can't connect database error num %d the desc is %s" % (e[0], e[1])
コード例 #5
0
ファイル: DownloadHandler.py プロジェクト: SolarHalo/gcp
    def execute(self):
        
        StaticUtil.remove(self.filename)
        
        configs = Config.getConfig()
        dowLoadNum = configs['sys']['download.num']
        
        DownloadHandler.Threads_lock.acquire()
        try:
            #self.logger.info("Download Num is %d , MaxNum is %d ."%(DownloadHandler.dowNum,dowLoadNum));
                
            if DownloadHandler.dowNum >= dowLoadNum:
                TaskRun.getInstance().submit(self)
                #self.logger.info("Now file[%s] download go back ."%self.filename);
                return
            DownloadHandler.dowNum += 1
        finally:
            DownloadHandler.Threads_lock.release()

        try:
            
            self.logger.info("Download file[%s] start , %s !"%(self.filename,self.source))
            download = Download(self.url, self.filename)
            download.executeDown();
            
            if self.source == GcpConstant.Source.Bigfish:
                parse = lib.gcp.handler.bigfish.FeatureParserHandler.FeatureParserHandler(self.filename, self.source,self.conf);
                TaskRun.getInstance().submit(parse)
            elif  self.source == GcpConstant.Source.Alawar:
                parse = lib.gcp.handler.alawar.FeatureParserHandler.FeatureParserHandler(self.filename, self.source,self.conf);
                TaskRun.getInstance().submit(parse)
            
        except Exception , e:
            self.logger.error("Download file[%s] error !"%self.filename)
            
            #re load
            self.logger.info("Reload filename[%s] cfg[%s]",(self.filename,self.conf))
            download = DownloadHandler(self.conf, self.filename, self.source)
            TaskRun.getInstance().submit(download)
            
            self.logger.exception(e)
コード例 #6
0
ファイル: Startup.py プロジェクト: SolarHalo/gcp
from conf import Config
from lib.gcp.util.Logger import LoggerFactory
from lib.core.TaskRun import TaskRun
import lib.gcp.handler.bigfish.UrlParserHandler
import lib.gcp.handler.alawar.UrlParserHandler
from lib.core.TimeScheduler import TimeScheduler
from apscheduler.threadpool import ThreadPool
from lib.gcp.handler.TagHandler import TagHandler
from lib.gcp.handler.GenreEnnameHandler import GenreEnnameHandler

if __name__ == '__main__':
 
    logger = LoggerFactory.getLogger()
    
    #thread pool
    configs = Config.getConfig()
    poolmin = configs['sys']['threadpool.min']
    poolmax = configs['sys']['threadpool.max']
    keep = configs['sys']['threadpool.keep']
    pool = ThreadPool(poolmin,poolmax,keep);
    
    logger.info("Ready to start Task !");
    TaskRun.getInstance().start(treadpool= pool);
    logger.info("Start Task end!");
    
    logger.info("Ready to start Scheduler !");
    TimeScheduler.getInstance().init(pool);
    TimeScheduler.getInstance().registerInterval(lib.gcp.handler.alawar.UrlParserHandler.UrlParserHandler(),
                                                 days = 1,start_date ='2014-11-02 00:00:00')
    
    TimeScheduler.getInstance().registerInterval(lib.gcp.handler.bigfish.UrlParserHandler.UrlParserHandler('game'),
コード例 #7
0
ファイル: BigfishDecode.py プロジェクト: SolarHalo/gcp
 def resolveEntity(self, publicId, systemId):
     configs = Config.getConfig()
     return configs['game.bigfish']['dtd'][0] + systemId
コード例 #8
0
ファイル: AlawarDecode.py プロジェクト: SolarHalo/gcp
 def resolveEntity(self, publicId, systemId):
     configs = Config.getConfig()
     return configs['game.alawar']['dtd']+systemId