def __init__(self,conf,filename,source):
self.conf = conf
self.filename = filename
self.source = source
configs = Config.getConfig()
self.batchSize = configs['sys']['db.batch.size']
def execute(self):
configs = Config.getConfig()
self.logger.info("Ready to parser url !")
urlTemples = []
for key in configs['game.alawar'].keys():
if key.startswith('url'):
attr = key.split(".")
table = attr[1]
locale = attr[2]
urlTemple = configs['game.alawar'][key]
urlTemples.append([urlTemple,table,locale])
for urlTemple in urlTemples:
if self.table is not None and urlTemple[1] != self.table:
continue
cf = configs['game.alawar']
urls = StaticUtil.convertUrl(urlTemple, cf)
self.logger.info("Parser xml convert count[%d]!" % len(urls));
for conf in urls:
filepath = os.path.join(StaticUtil.mkdir(configs['game.alawar']['data_path'], conf),
StaticUtil.getTimeStrForFold() + "alawar.xml")
#filepath = Config.configs['game.alawar']['data_path'] +StaticUtil.getPara(conf)+ StaticUtil.getTimeStrForFold() + "alawar.xml"
download = DownloadHandler(conf, filepath, GcpConstant.Source.Alawar)
TaskRun.getInstance().submit(download)
self.logger.info("Parser xml[%s] name , url[%s] end !"%(filepath,conf));
def __init__(self,conf,filename,source):
self.conf = conf
self.filename = filename
self.source = source
configs = Config.getConfig()
self.batchSize = configs['sys']['db.batch.size']
self.sitePrix = configs['game.bigfish']['dtd'][1]
self.assets = configs['game.bigfish']['dtd'][2]
def __init__(self):
self.cursor = None
configs = Config.getConfig()
self.host = configs['db']['db_host']
self.user = configs['db']['db_user']
self.pwd = configs['db']['db_pass']
self.db = configs['db']['db_db']
self.conn = None
try:
self.conn = MySQLdb.Connect(host=self.host, user=self.user, passwd=self.pwd, db=self.db, charset="utf8", use_unicode="True")
except MySQLdb.Error, e:
print e
print "can't connect database error num %d the desc is %s" % (e[0], e[1])
def execute(self):
StaticUtil.remove(self.filename)
configs = Config.getConfig()
dowLoadNum = configs['sys']['download.num']
DownloadHandler.Threads_lock.acquire()
try:
#self.logger.info("Download Num is %d , MaxNum is %d ."%(DownloadHandler.dowNum,dowLoadNum));
if DownloadHandler.dowNum >= dowLoadNum:
TaskRun.getInstance().submit(self)
#self.logger.info("Now file[%s] download go back ."%self.filename);
return
DownloadHandler.dowNum += 1
finally:
DownloadHandler.Threads_lock.release()
try:
self.logger.info("Download file[%s] start , %s !"%(self.filename,self.source))
download = Download(self.url, self.filename)
download.executeDown();
if self.source == GcpConstant.Source.Bigfish:
parse = lib.gcp.handler.bigfish.FeatureParserHandler.FeatureParserHandler(self.filename, self.source,self.conf);
TaskRun.getInstance().submit(parse)
elif self.source == GcpConstant.Source.Alawar:
parse = lib.gcp.handler.alawar.FeatureParserHandler.FeatureParserHandler(self.filename, self.source,self.conf);
TaskRun.getInstance().submit(parse)
except Exception , e:
self.logger.error("Download file[%s] error !"%self.filename)
#re load
self.logger.info("Reload filename[%s] cfg[%s]",(self.filename,self.conf))
download = DownloadHandler(self.conf, self.filename, self.source)
TaskRun.getInstance().submit(download)
self.logger.exception(e)
from conf import Config
from lib.gcp.util.Logger import LoggerFactory
from lib.core.TaskRun import TaskRun
import lib.gcp.handler.bigfish.UrlParserHandler
import lib.gcp.handler.alawar.UrlParserHandler
from lib.core.TimeScheduler import TimeScheduler
from apscheduler.threadpool import ThreadPool
from lib.gcp.handler.TagHandler import TagHandler
from lib.gcp.handler.GenreEnnameHandler import GenreEnnameHandler
if __name__ == '__main__':
logger = LoggerFactory.getLogger()
#thread pool
configs = Config.getConfig()
poolmin = configs['sys']['threadpool.min']
poolmax = configs['sys']['threadpool.max']
keep = configs['sys']['threadpool.keep']
pool = ThreadPool(poolmin,poolmax,keep);
logger.info("Ready to start Task !");
TaskRun.getInstance().start(treadpool= pool);
logger.info("Start Task end!");
logger.info("Ready to start Scheduler !");
TimeScheduler.getInstance().init(pool);
TimeScheduler.getInstance().registerInterval(lib.gcp.handler.alawar.UrlParserHandler.UrlParserHandler(),
days = 1,start_date ='2014-11-02 00:00:00')
TimeScheduler.getInstance().registerInterval(lib.gcp.handler.bigfish.UrlParserHandler.UrlParserHandler('game'),
def resolveEntity(self, publicId, systemId):
configs = Config.getConfig()
return configs['game.bigfish']['dtd'][0] + systemId
def resolveEntity(self, publicId, systemId):
configs = Config.getConfig()
return configs['game.alawar']['dtd']+systemId