def execute(self, appconfig, query, args):
ret = {}
start, end, timezone = util.parseTime(args)
self.parallel = util.get(query, 'parallel', 'false')
self.threads = []
query['timezone'] = timezone
query['conditions'] = self.get_additional_conditions(args, query)
offset_minutes = util.get(query, 'offset_minutes', 0)
self.fetch_data(appconfig, query, args, start, end,
util.total_seconds(timedelta(minutes=offset_minutes)),
ret, 'today')
if 'true' == query.get('dod'):
self.fetch_data(
appconfig, query, args, start, end,
util.total_seconds(timedelta(days=1, minutes=offset_minutes)),
ret, 'ystd')
if 'true' == query.get('wow'):
self.fetch_data(
appconfig, query, args, start, end,
util.total_seconds(timedelta(days=7, minutes=offset_minutes)),
ret, 'lastwk')
if self.parallel and len(self.threads) > 0:
for t in self.threads:
t.join()
return ret
def execute(appconfig, query, args, fetcher):
start, end, timezone = util.parseTime(args)
offset = util.getOffsetSeconds(start, timezone)
start, end = start - offset, end - offset
offset_seconds = 0
if args.get('offsetdate') == 'yesterday':
offset_seconds = util.total_seconds(timedelta(days=1))
elif args.get('offsetdate') == 'lastweek':
offset_seconds = util.total_seconds(timedelta(days=7))
start -= offset_seconds
ids = args['id'].split(',')
objtype = '256' if 'campaign' == args['type'] else '257'
ids = [_id + "_" + objtype for _id in ids]
try:
colo = appconfig.get("hbase", "hbase")
except:
colo = 'blue_prod'
keys = ",".join(ids)
if args.get('analysis') == "true":
rets = load_hbase(colo, 'analysis', start, end, keys, offset)
else:
rets = load_hbase(colo, 'cache', start, end, keys, offset)
return rets
def loadGeneralConf(self):
sh = self.wb.sheet_by_name(CONF_SHEET)
confD = {}
for i in range(sh.nrows):
key, val = self.getRow( sh, i, True )
confD[ confKeyMap[key] ] = val
self.matchDuration = parseTime( confD['matchDuration'] )
self.nbMatch = int( confD['nbMatch'] )
self.penDateFactor = confD['penDateFactor']
def loadGeneralConf(self):
sh = self.wb.sheet_by_name(CONF_SHEET)
confD = {}
for i in range(sh.nrows):
key, val = self.getRow(sh, i, True)
confD[confKeyMap[key]] = val
self.matchDuration = parseTime(confD['matchDuration'])
self.nbMatch = int(confD['nbMatch'])
self.penDateFactor = confD['penDateFactor']
def parseUniformity(self):
sh = self.wb.sheet_by_name(UNIF)
unifL = []
for i in range(1,sh.nrows):
restr = Restriction(False, 1, None, None)
key, pen, maxCount, timeMin, timeMax, stadium, field = self.getRow(sh, i, True)
if stadium is not None : restr.stadiumS = set([stadium])
if field is not None : restr.fNameS = set([field])
restr.timeMin = parseTime(timeMin)
restr.timeMax = parseTime(timeMax)
unifL.append( (key, float(pen) / 2**maxCount, restr) )
keySet = set(list(zip( *unifL))[0])
for team in self.league.getTeamL():
if hasattr( team, 'unifCountD'):
for id_ in team.unifCountD.keys():
if id_ not in keySet:
print("Warning : %s not in [%s]" % (id_, ', '.join(list(keySet))))
self.unifL = unifL
def parseUniformity(self):
sh = self.wb.sheet_by_name(UNIF)
unifL = []
for i in range(1, sh.nrows):
restr = Restriction(False, 1, None, None)
key, pen, maxCount, timeMin, timeMax, stadium, field = self.getRow(
sh, i, True)
if stadium is not None: restr.stadiumS = set([stadium])
if field is not None: restr.fNameS = set([field])
restr.timeMin = parseTime(timeMin)
restr.timeMax = parseTime(timeMax)
unifL.append((key, float(pen) / 2**maxCount, restr))
keySet = set(list(zip(*unifL))[0])
for team in self.league.getTeamL():
if hasattr(team, 'unifCountD'):
for id_ in team.unifCountD.keys():
if id_ not in keySet:
print("Warning : %s not in [%s]" %
(id_, ', '.join(list(keySet))))
self.unifL = unifL
def changeJobExecutionTime(job):
new_time = None
while not new_time:
new_time_str = raw_input("New Execution Time: ")
try:
new_time = util.parseTime(new_time_str)
except:
new_time = None
print "Invalid time formate please write HH:MM"
job.setExecutionTime(new_time)
jobs_handler.saveJobs(jobs)
jobProperties(job)
def getRestriction(self):
sh = self.wb.sheet_by_name(RESTR)
typeL, type2idx = self.getType(sh)
restrD = {}
self.teamSrv.lock = True
self.dateSrv.lock = True
for i in range(1, sh.nrows):
valL = self.getRow(sh, i, True)
include = signD[str(valL[0]).strip()]
value = valL[1]
teamL, dateL = self.expandAll(valL, type2idx, (TEAM, DATE))
timeMinL, timeMaxL, stadiumL, fNameL = self.expandAll(
valL, type2idx, (TIME_MIN, TIME_MAX, STADIUM, FNAME))
if dateL is None:
dateL = self.dateSrv.objD.values()
for team in teamL:
for date in dateL:
restr = Restriction(include, value, team, date)
if timeMinL is not None:
restr.timeMin = parseTime(timeMinL[0])
if timeMaxL is not None:
restr.timeMax = parseTime(timeMaxL[0])
if stadiumL is not None: restr.stadiumS = set(stadiumL)
if fNameL is not None: restr.fNameS = set(fNameL)
try:
restrD[(team, date)].append(restr)
except KeyError:
restrD[(team, date)] = [restr]
self.restrD = restrD
def getRestriction(self):
sh = self.wb.sheet_by_name(RESTR)
typeL, type2idx = self.getType( sh )
restrD = {}
self.teamSrv.lock = True
self.dateSrv.lock = True
for i in range(1,sh.nrows):
valL = self.getRow( sh, i, True )
include = signD[ str(valL[0]).strip() ]
value = valL[1]
teamL, dateL = self.expandAll( valL,type2idx,(TEAM,DATE) )
timeMinL, timeMaxL, stadiumL, fNameL = self.expandAll( valL,type2idx,(TIME_MIN,TIME_MAX,STADIUM,FNAME) )
if dateL is None:
dateL = self.dateSrv.objD.values()
for team in teamL:
for date in dateL:
restr = Restriction( include, value, team, date )
if timeMinL is not None: restr.timeMin = parseTime(timeMinL[0])
if timeMaxL is not None: restr.timeMax = parseTime(timeMaxL[0])
if stadiumL is not None: restr.stadiumS = set( stadiumL )
if fNameL is not None: restr.fNameS = set( fNameL )
try: restrD[ (team,date) ].append( restr )
except KeyError : restrD[ (team,date) ] = [restr]
self.restrD = restrD
def endElement(self, name):
attributes = self.attributesList.pop()
if name == "job":
# build job
job = Job(self.recipeRef)
executionType = attributes[u"type"]
executionTime = util.parseTime(attributes[u"time"])
executionDay = ""
if (not executionType == "daily"):
executionDay = attributes[u"day"]
job.setExecution(executionType, executionTime, executionDay)
self.jobs.append(job)
if name == "recipeRef":
self.recipeRef = self.buffer
self.buffer = ""
def endElement(self, name):
attributes = self.attributesList.pop()
if name == "job":
# build job
job = Job(self.recipeRef)
executionType = attributes[u"type"]
executionTime = util.parseTime(attributes[u"time"])
executionDay = ""
if (not executionType == "daily"):
executionDay = attributes[u"day"]
job.setExecution(executionType, executionTime, executionDay)
self.jobs.append(job)
if name == "recipeRef":
self.recipeRef = self.buffer
self.buffer = ""
def __init__(self, recipeRef):
self.id = -1
self.recipeRef = recipeRef.strip()
self.executionType = "daily"
self.executionTime = util.parseTime("11:00")
self.isExecuting = False
def __init__(self, recipeRef):
self.id = -1
self.recipeRef = recipeRef.strip()
self.executionType = "daily"
self.executionTime = util.parseTime("11:00")
self.isExecuting = False