def checkKeyDir(f):
error = []
status = 'OK'
meta = None
key = os.path.basename(f[0])
if 'meta.json' not in f[2]:
error.append("no meta file !")
status = 'ERROR'
else:
meta = util.loadJsonFile(os.path.join(f[0], 'meta.json'))
if 'isIgnore' in meta and meta['isIgnore'] == True:
status = 'IGNORE'
error = []
return status, error
if 'content.html' not in f[2]:
error.append("no content file !")
status = 'ERROR'
if 'content.mp3' not in f[2]:
error.append("no mp3 file !")
status = 'ERROR'
if '__zipPic__.jpg' not in f[2] and '__zipPic__.png' not in f[2]:
error.append("no zipPic file !")
status = 'ERROR'
meta = util.loadJsonFile(os.path.join(f[0], 'meta.json'))
if meta:
#check embPic:
for picName in meta['embPics']:
if util.getUrlFileName(picName) not in f[2]:
error.append("no embPic %s !" % (util.getUrlFileName(picName)))
status = 'ERROR'
return status, error
def main(args):
T0 = time.time()
outputData = []
inFile = util.loadJsonFile(args.input_file)
# Remove any ordering effect from game number
random.shuffle(inFile)
toProcess = len(inFile)
print("{} has {} items".format(args.input_file, toProcess))
if 0 < args.count < toProcess:
toProcess = args.count
print("\tonly parsing {}".format(args.count))
print()
printEvery = toProcess // 15
for gameNum, fileNames in enumerate(inFile, 1):
assert type(fileNames) == list
assert len(fileNames) == 2
# If you had an error on this line re-run Coalesce.py
match = util.loadJsonFile(fileNames[0])
timeline = util.loadJsonFile(fileNames[1])
parsed = parseGameRough(match, timeline)
outputData.append(parsed)
if gameNum % printEvery == 0:
print("parsed {} of {} ({:0.0f}%) ({:.2f}s)".format(
gameNum, toProcess, 100 * gameNum / toProcess,
time.time() - T0))
if gameNum == args.count:
print()
print("Stopping after {} games (like you asked with -c)".format(
args.count))
break
debugStats(args, outputData, T0)
exampleLines = random.sample(range(toProcess), args.examples)
for exampleLine in sorted(exampleLines):
gameStr = str(outputData[exampleLine])
example = util.abbreviateString(gameStr,
10000 if args.full_examples else 70)
print()
print("line {}: {}".format(exampleLine, example))
if args.examples > 0:
util.writeJsonFile('example-feature.json', outputData[exampleLines[0]])
if not args.dry_run:
util.writeJsonFile(args.output_file, outputData)
def addStockListByFile(self, fname):
stockList = util.loadJsonFile(fname)
for stock in stockList:
code = stock['code']
name = stock['name']
if code not in self.importStockDict:
self.importStockDict[code] = {}
self.importStockDict[code]['name'] = name
self.importStockDict[code]['nameCn'] = name
def addStockListByFile(self, fname):
stockList = util.loadJsonFile(fname)
for stock in stockList:
code = stock['code']
name = stock['name']
if code not in self.importStockDict:
self.importStockDict[code] = {}
self.importStockDict[code]['name'] = name
self.importStockDict[code]['nameCn'] = name
def check(dirName):
#find today's keyfiles
tsDir = dirName
report = {}
okKeys = []
for f in os.walk(os.path.join(outputPath, tsDir)):
if os.path.basename(f[0]).startswith('_content_') and f[1] == []:
error = []
print f
meta = None
key = os.path.basename(f[0])
if 'meta.json' not in f[2]:
error.append("no meta file !")
else:
meta = util.loadJsonFile(os.path.join(f[0], 'meta.json'))
if 'isIgnore' in meta and meta['isIgnore'] == True:
report[key] = [
"ignored:%s" % meta['ignoreMsg'],
]
okKeys.append(key)
continue
if 'content.html' not in f[2]:
error.append("no content file !")
if 'content.mp3' not in f[2]:
error.append("no mp3 file !")
if '__zipPic__.jpg' not in f[2] and '__zipPic__.png' not in f[2]:
error.append("no zipPic file !")
meta = util.loadJsonFile(os.path.join(f[0], 'meta.json'))
if meta:
#check embPic:
for picName in meta['embPics']:
if util.getUrlFileName(picName) not in f[2]:
error.append("no embPic %s !" %
(util.getUrlFileName(picName)))
if error != []:
report[key] = error
else:
okKeys.append(key)
return (report, okKeys)
def check(dirName):
#find today's keyfiles
tsDir = dirName
report = {}
okKeys = []
for f in os.walk(os.path.join(outputPath, tsDir)):
if os.path.basename(f[0]).startswith('_content_') and f[1] == []:
error = []
print f
meta = None
key = os.path.basename(f[0])
if 'meta.json' not in f[2]:
error.append("no meta file !")
else:
meta = util.loadJsonFile(os.path.join(f[0], 'meta.json'))
if 'isIgnore' in meta and meta['isIgnore'] == True:
report[key] = ["ignored:%s" % meta['ignoreMsg'], ]
okKeys.append(key)
continue
if 'content.html' not in f[2]:
error.append("no content file !")
if 'content.mp3' not in f[2]:
error.append("no mp3 file !")
if '__zipPic__.jpg' not in f[2] and '__zipPic__.png' not in f[2]:
error.append("no zipPic file !")
meta = util.loadJsonFile(os.path.join(f[0], 'meta.json'))
if meta:
#check embPic:
for picName in meta['embPics']:
if util.getUrlFileName(picName) not in f[2]:
error.append("no embPic %s !" % (util.getUrlFileName(picName)))
if error != []:
report[key] = error
else:
okKeys.append(key)
return (report, okKeys)
def update():
global _db, _all_classifications
#find today's keyfiles
date = util.getDateStamp(theTime)
for f in os.walk(outputPath):
if os.path.basename(f[0]).startswith('_content_') and f[1] == []:
#print f
key = os.path.basename(f[0])
meta = util.loadJsonFile(os.path.join(f[0], 'meta.json'))
if meta:
#got cha
classification = 'VOA, ' + meta['siteTitle']
cId = 0
#check classification
if classification not in _all_classifications:
cId = insertClassification(classification, classification)
_all_classifications = getAllClassification()
else:
cId = _all_classifications[classification]['id']
name = meta['title']
url = os.path.join(monUrl, key)
ori_url = meta['url']
ori_pub_date = meta['date']
if re.match("\d+/\d+/\d+", ori_pub_date):
ori_pub_date = util.timeStr2timeStr(ori_pub_date, "%m/%d/%Y", "%Y-%m-%d")
if 'content.jpg' in f[2]:
contentPic = 'content.jpg'
contentPicCaption = meta['contentPicCaptions'][0]
elif 'content.png' in f[2]:
contentPic = 'content.png'
contentPicCaption = meta['contentPicCaptions'][0]
else:
contentPic = None;
contentPicCaption = None;
#for v in (classification, cId, name, url, ori_url, ori_pub_date, contentPic, contentPicCaption):
# #print v
# pass
# cp
dest = os.path.join(copyToPath, key)
src = f[0]
if os.path.isdir(dest):
shutil.rmtree(dest)
shutil.copytree(f[0], dest)
# insert it
#sql = "insert into %s(name, classification, url, ori_url, ori_pub_date, contentPic, contentPicCaption) "\
# "values('%s', '%s', '%s', '%s', '%s', '%s', '%s')" % \
# ('iz', name, classification, url, ori_url, ori_pub_date, contentPic, contentPicCaption)
oriSql = "insert into iz_article(name, classification, classification_id, url, ori_url, ori_pub_date, "\
"contentPic, contentPicCaption) "\
" values(%s, %s, %s, %s, %s, %s, %s, %s)"
_db.execute(oriSql, name, classification, cId, url, ori_url, ori_pub_date, contentPic, contentPicCaption)
def genKlist(theDir):
report = {}
okKeys = []
tsDir = os.path.dirname(theDir)
for f in os.walk(theDir):
if f[1] != []:
continue
error = []
meta = None
key = os.path.basename(f[0])
if 'meta.json' not in f[2]:
error.append("no meta file !")
else:
meta = util.loadJsonFile(os.path.join(f[0], 'meta.json'))
if 'isIgnore' in meta and meta['isIgnore'] == True:
report[key] = ["ignored:%s" % meta['ignoreMsg'], ]
okKeys.append(key)
continue
if 'content.html' not in f[2]:
error.append("no content file !")
if 'content.mp3' not in f[2]:
error.append("no mp3 file !")
if '__zipPic__.jpg' not in f[2] and '__zipPic__.png' not in f[2]:
error.append("no zipPic file !")
#if meta:
# #check embPic:
# for picName in meta['embPics']:
# if util.getUrlFileName(picName) not in f[2]:
# error.append("no embPic %s !" % (util.getUrlFileName(picName)))
if error != []:
report[key] = error
else:
okKeys.append(key)
return (report, okKeys)
def __load_components(self,path):
components = util.loadJsonFile(self.get_path());
print(components);
def getRawGameData(args):
fileName = args.input_file
numGames = args.num_games
requiredRank = util.rankOrdering(args.rank)
filtered = defaultdict(int)
if args.filter_weird_games:
print()
print("FILTERING WEIRD_GAMES")
print("INVALIDATES STATS PROBABLY")
print("INVESTIGATE IF YOU ARE USING THIS")
print()
games = []
goals = []
outputData = util.loadJsonFile(fileName)
for dataI, data in enumerate(outputData):
# Filtering remakes and stuff (sudo valid beacuse know at t ~= 0)
if data['debug']['duration'] < 600:
filtered['short_game'] += 1
continue
# Filter out low rank games (valid because know at t = 0)
lowerRanked = len([
1 for c in data['features']['champs']
if util.rankOrdering(c['approxRank']) < requiredRank
])
if lowerRanked >= 2:
filtered['rank'] += 1
continue
if args.filter_weird_games:
if data['debug']['surrendered']:
filtered['surrendered'] += 1
continue
anyOtherPositions = False
for champ in data['features']['champs']:
position = util.guessPosition(champ)
if position == "OTHER":
anyOtherPositions = True
break
if anyOtherPositions:
filtered['bad_position'] += 1
continue
goal = data['goal']
assert goal in (True, False)
games.append(data)
goals.append(goal)
if len(games) == numGames:
break
filterCount = len(outputData) - len(goals)
assert filterCount == sum(filtered.values())
print("Loaded {} games (filtered {} = {:.1f}%)".format(
len(goals), filterCount, 100 * filterCount / len(outputData)))
for reason, count in sorted(filtered.items()):
print("\t{}: {}".format(reason, count))
print()
return games, goals
def addStockInfoByFile(self, fname):
stockInfo = util.loadJsonFile(fname)
code = stockInfo['code']
if code not in self.importStockDict:
self.importStockDict[code] = {}
self.importStockDict[code].update(stockInfo)
def addStockInfoByFile(self, fname):
stockInfo = util.loadJsonFile(fname)
code = stockInfo['code']
if code not in self.importStockDict:
self.importStockDict[code] = {}
self.importStockDict[code].update(stockInfo)
def load_options(self):
options = util.loadJsonFile(self.path)
#Add line per params
self.x = options["x"]
self.y = options["y"]
self.windowType = options["wType"]
