Ejemplo n.º 1
0
 def __init__(self,conf,filename,source):
     self.conf = conf
     self.filename = filename
     self.source = source
     
     configs = Config.getConfig()
     self.batchSize = configs['sys']['db.batch.size']
Ejemplo n.º 2
0
 def execute(self):
     configs = Config.getConfig()
     self.logger.info("Ready to parser url !")
     
     urlTemples = []
     
     for key in configs['game.alawar'].keys():
         if key.startswith('url'):
             attr = key.split(".")
             table = attr[1]
             locale = attr[2]
             urlTemple = configs['game.alawar'][key]
             urlTemples.append([urlTemple,table,locale])
     
     for urlTemple in urlTemples:
         if self.table is not None and urlTemple[1] != self.table:
             continue
         
         cf = configs['game.alawar']
         urls = StaticUtil.convertUrl(urlTemple, cf)
         self.logger.info("Parser xml convert count[%d]!" % len(urls));
         for conf in urls:
             filepath = os.path.join(StaticUtil.mkdir(configs['game.alawar']['data_path'], conf),
                                     StaticUtil.getTimeStrForFold() + "alawar.xml")
             #filepath = Config.configs['game.alawar']['data_path'] +StaticUtil.getPara(conf)+ StaticUtil.getTimeStrForFold() + "alawar.xml"
             download = DownloadHandler(conf, filepath, GcpConstant.Source.Alawar)
             TaskRun.getInstance().submit(download)
             
             self.logger.info("Parser xml[%s] name , url[%s] end !"%(filepath,conf));
Ejemplo n.º 3
0
 def __init__(self,conf,filename,source):
     self.conf = conf
     self.filename = filename
     self.source = source
     
     configs = Config.getConfig()
     self.batchSize = configs['sys']['db.batch.size']
     self.sitePrix = configs['game.bigfish']['dtd'][1]
     self.assets =  configs['game.bigfish']['dtd'][2]
Ejemplo n.º 4
0
 def __init__(self):  
     self.cursor = None
     configs = Config.getConfig()
     self.host = configs['db']['db_host']
     self.user = configs['db']['db_user']
     self.pwd = configs['db']['db_pass']
     self.db = configs['db']['db_db']
     self.conn = None
     try:
         self.conn = MySQLdb.Connect(host=self.host, user=self.user, passwd=self.pwd, db=self.db, charset="utf8", use_unicode="True") 
     except MySQLdb.Error, e:
         print e
         print "can't connect database error num %d the desc is %s" % (e[0], e[1])
Ejemplo n.º 5
0
    def execute(self):
        
        StaticUtil.remove(self.filename)
        
        configs = Config.getConfig()
        dowLoadNum = configs['sys']['download.num']
        
        DownloadHandler.Threads_lock.acquire()
        try:
            #self.logger.info("Download Num is %d , MaxNum is %d ."%(DownloadHandler.dowNum,dowLoadNum));
                
            if DownloadHandler.dowNum >= dowLoadNum:
                TaskRun.getInstance().submit(self)
                #self.logger.info("Now file[%s] download go back ."%self.filename);
                return
            DownloadHandler.dowNum += 1
        finally:
            DownloadHandler.Threads_lock.release()

        try:
            
            self.logger.info("Download file[%s] start , %s !"%(self.filename,self.source))
            download = Download(self.url, self.filename)
            download.executeDown();
            
            if self.source == GcpConstant.Source.Bigfish:
                parse = lib.gcp.handler.bigfish.FeatureParserHandler.FeatureParserHandler(self.filename, self.source,self.conf);
                TaskRun.getInstance().submit(parse)
            elif  self.source == GcpConstant.Source.Alawar:
                parse = lib.gcp.handler.alawar.FeatureParserHandler.FeatureParserHandler(self.filename, self.source,self.conf);
                TaskRun.getInstance().submit(parse)
            
        except Exception , e:
            self.logger.error("Download file[%s] error !"%self.filename)
            
            #re load
            self.logger.info("Reload filename[%s] cfg[%s]",(self.filename,self.conf))
            download = DownloadHandler(self.conf, self.filename, self.source)
            TaskRun.getInstance().submit(download)
            
            self.logger.exception(e)
Ejemplo n.º 6
0
from conf import Config
from lib.gcp.util.Logger import LoggerFactory
from lib.core.TaskRun import TaskRun
import lib.gcp.handler.bigfish.UrlParserHandler
import lib.gcp.handler.alawar.UrlParserHandler
from lib.core.TimeScheduler import TimeScheduler
from apscheduler.threadpool import ThreadPool
from lib.gcp.handler.TagHandler import TagHandler
from lib.gcp.handler.GenreEnnameHandler import GenreEnnameHandler

if __name__ == '__main__':
 
    logger = LoggerFactory.getLogger()
    
    #thread pool
    configs = Config.getConfig()
    poolmin = configs['sys']['threadpool.min']
    poolmax = configs['sys']['threadpool.max']
    keep = configs['sys']['threadpool.keep']
    pool = ThreadPool(poolmin,poolmax,keep);
    
    logger.info("Ready to start Task !");
    TaskRun.getInstance().start(treadpool= pool);
    logger.info("Start Task end!");
    
    logger.info("Ready to start Scheduler !");
    TimeScheduler.getInstance().init(pool);
    TimeScheduler.getInstance().registerInterval(lib.gcp.handler.alawar.UrlParserHandler.UrlParserHandler(),
                                                 days = 1,start_date ='2014-11-02 00:00:00')
    
    TimeScheduler.getInstance().registerInterval(lib.gcp.handler.bigfish.UrlParserHandler.UrlParserHandler('game'),
Ejemplo n.º 7
0
 def resolveEntity(self, publicId, systemId):
     configs = Config.getConfig()
     return configs['game.bigfish']['dtd'][0] + systemId
Ejemplo n.º 8
0
 def resolveEntity(self, publicId, systemId):
     configs = Config.getConfig()
     return configs['game.alawar']['dtd']+systemId