def checkKeyDir(f): error = [] status = 'OK' meta = None key = os.path.basename(f[0]) if 'meta.json' not in f[2]: error.append("no meta file !") status = 'ERROR' else: meta = util.loadJsonFile(os.path.join(f[0], 'meta.json')) if 'isIgnore' in meta and meta['isIgnore'] == True: status = 'IGNORE' error = [] return status, error if 'content.html' not in f[2]: error.append("no content file !") status = 'ERROR' if 'content.mp3' not in f[2]: error.append("no mp3 file !") status = 'ERROR' if '__zipPic__.jpg' not in f[2] and '__zipPic__.png' not in f[2]: error.append("no zipPic file !") status = 'ERROR' meta = util.loadJsonFile(os.path.join(f[0], 'meta.json')) if meta: #check embPic: for picName in meta['embPics']: if util.getUrlFileName(picName) not in f[2]: error.append("no embPic %s !" % (util.getUrlFileName(picName))) status = 'ERROR' return status, error
def main(args): T0 = time.time() outputData = [] inFile = util.loadJsonFile(args.input_file) # Remove any ordering effect from game number random.shuffle(inFile) toProcess = len(inFile) print("{} has {} items".format(args.input_file, toProcess)) if 0 < args.count < toProcess: toProcess = args.count print("\tonly parsing {}".format(args.count)) print() printEvery = toProcess // 15 for gameNum, fileNames in enumerate(inFile, 1): assert type(fileNames) == list assert len(fileNames) == 2 # If you had an error on this line re-run Coalesce.py match = util.loadJsonFile(fileNames[0]) timeline = util.loadJsonFile(fileNames[1]) parsed = parseGameRough(match, timeline) outputData.append(parsed) if gameNum % printEvery == 0: print("parsed {} of {} ({:0.0f}%) ({:.2f}s)".format( gameNum, toProcess, 100 * gameNum / toProcess, time.time() - T0)) if gameNum == args.count: print() print("Stopping after {} games (like you asked with -c)".format( args.count)) break debugStats(args, outputData, T0) exampleLines = random.sample(range(toProcess), args.examples) for exampleLine in sorted(exampleLines): gameStr = str(outputData[exampleLine]) example = util.abbreviateString(gameStr, 10000 if args.full_examples else 70) print() print("line {}: {}".format(exampleLine, example)) if args.examples > 0: util.writeJsonFile('example-feature.json', outputData[exampleLines[0]]) if not args.dry_run: util.writeJsonFile(args.output_file, outputData)
def addStockListByFile(self, fname): stockList = util.loadJsonFile(fname) for stock in stockList: code = stock['code'] name = stock['name'] if code not in self.importStockDict: self.importStockDict[code] = {} self.importStockDict[code]['name'] = name self.importStockDict[code]['nameCn'] = name
def check(dirName): #find today's keyfiles tsDir = dirName report = {} okKeys = [] for f in os.walk(os.path.join(outputPath, tsDir)): if os.path.basename(f[0]).startswith('_content_') and f[1] == []: error = [] print f meta = None key = os.path.basename(f[0]) if 'meta.json' not in f[2]: error.append("no meta file !") else: meta = util.loadJsonFile(os.path.join(f[0], 'meta.json')) if 'isIgnore' in meta and meta['isIgnore'] == True: report[key] = [ "ignored:%s" % meta['ignoreMsg'], ] okKeys.append(key) continue if 'content.html' not in f[2]: error.append("no content file !") if 'content.mp3' not in f[2]: error.append("no mp3 file !") if '__zipPic__.jpg' not in f[2] and '__zipPic__.png' not in f[2]: error.append("no zipPic file !") meta = util.loadJsonFile(os.path.join(f[0], 'meta.json')) if meta: #check embPic: for picName in meta['embPics']: if util.getUrlFileName(picName) not in f[2]: error.append("no embPic %s !" % (util.getUrlFileName(picName))) if error != []: report[key] = error else: okKeys.append(key) return (report, okKeys)
def check(dirName): #find today's keyfiles tsDir = dirName report = {} okKeys = [] for f in os.walk(os.path.join(outputPath, tsDir)): if os.path.basename(f[0]).startswith('_content_') and f[1] == []: error = [] print f meta = None key = os.path.basename(f[0]) if 'meta.json' not in f[2]: error.append("no meta file !") else: meta = util.loadJsonFile(os.path.join(f[0], 'meta.json')) if 'isIgnore' in meta and meta['isIgnore'] == True: report[key] = ["ignored:%s" % meta['ignoreMsg'], ] okKeys.append(key) continue if 'content.html' not in f[2]: error.append("no content file !") if 'content.mp3' not in f[2]: error.append("no mp3 file !") if '__zipPic__.jpg' not in f[2] and '__zipPic__.png' not in f[2]: error.append("no zipPic file !") meta = util.loadJsonFile(os.path.join(f[0], 'meta.json')) if meta: #check embPic: for picName in meta['embPics']: if util.getUrlFileName(picName) not in f[2]: error.append("no embPic %s !" % (util.getUrlFileName(picName))) if error != []: report[key] = error else: okKeys.append(key) return (report, okKeys)
def update(): global _db, _all_classifications #find today's keyfiles date = util.getDateStamp(theTime) for f in os.walk(outputPath): if os.path.basename(f[0]).startswith('_content_') and f[1] == []: #print f key = os.path.basename(f[0]) meta = util.loadJsonFile(os.path.join(f[0], 'meta.json')) if meta: #got cha classification = 'VOA, ' + meta['siteTitle'] cId = 0 #check classification if classification not in _all_classifications: cId = insertClassification(classification, classification) _all_classifications = getAllClassification() else: cId = _all_classifications[classification]['id'] name = meta['title'] url = os.path.join(monUrl, key) ori_url = meta['url'] ori_pub_date = meta['date'] if re.match("\d+/\d+/\d+", ori_pub_date): ori_pub_date = util.timeStr2timeStr(ori_pub_date, "%m/%d/%Y", "%Y-%m-%d") if 'content.jpg' in f[2]: contentPic = 'content.jpg' contentPicCaption = meta['contentPicCaptions'][0] elif 'content.png' in f[2]: contentPic = 'content.png' contentPicCaption = meta['contentPicCaptions'][0] else: contentPic = None; contentPicCaption = None; #for v in (classification, cId, name, url, ori_url, ori_pub_date, contentPic, contentPicCaption): # #print v # pass # cp dest = os.path.join(copyToPath, key) src = f[0] if os.path.isdir(dest): shutil.rmtree(dest) shutil.copytree(f[0], dest) # insert it #sql = "insert into %s(name, classification, url, ori_url, ori_pub_date, contentPic, contentPicCaption) "\ # "values('%s', '%s', '%s', '%s', '%s', '%s', '%s')" % \ # ('iz', name, classification, url, ori_url, ori_pub_date, contentPic, contentPicCaption) oriSql = "insert into iz_article(name, classification, classification_id, url, ori_url, ori_pub_date, "\ "contentPic, contentPicCaption) "\ " values(%s, %s, %s, %s, %s, %s, %s, %s)" _db.execute(oriSql, name, classification, cId, url, ori_url, ori_pub_date, contentPic, contentPicCaption)
def genKlist(theDir): report = {} okKeys = [] tsDir = os.path.dirname(theDir) for f in os.walk(theDir): if f[1] != []: continue error = [] meta = None key = os.path.basename(f[0]) if 'meta.json' not in f[2]: error.append("no meta file !") else: meta = util.loadJsonFile(os.path.join(f[0], 'meta.json')) if 'isIgnore' in meta and meta['isIgnore'] == True: report[key] = ["ignored:%s" % meta['ignoreMsg'], ] okKeys.append(key) continue if 'content.html' not in f[2]: error.append("no content file !") if 'content.mp3' not in f[2]: error.append("no mp3 file !") if '__zipPic__.jpg' not in f[2] and '__zipPic__.png' not in f[2]: error.append("no zipPic file !") #if meta: # #check embPic: # for picName in meta['embPics']: # if util.getUrlFileName(picName) not in f[2]: # error.append("no embPic %s !" % (util.getUrlFileName(picName))) if error != []: report[key] = error else: okKeys.append(key) return (report, okKeys)
def __load_components(self,path): components = util.loadJsonFile(self.get_path()); print(components);
def getRawGameData(args): fileName = args.input_file numGames = args.num_games requiredRank = util.rankOrdering(args.rank) filtered = defaultdict(int) if args.filter_weird_games: print() print("FILTERING WEIRD_GAMES") print("INVALIDATES STATS PROBABLY") print("INVESTIGATE IF YOU ARE USING THIS") print() games = [] goals = [] outputData = util.loadJsonFile(fileName) for dataI, data in enumerate(outputData): # Filtering remakes and stuff (sudo valid beacuse know at t ~= 0) if data['debug']['duration'] < 600: filtered['short_game'] += 1 continue # Filter out low rank games (valid because know at t = 0) lowerRanked = len([ 1 for c in data['features']['champs'] if util.rankOrdering(c['approxRank']) < requiredRank ]) if lowerRanked >= 2: filtered['rank'] += 1 continue if args.filter_weird_games: if data['debug']['surrendered']: filtered['surrendered'] += 1 continue anyOtherPositions = False for champ in data['features']['champs']: position = util.guessPosition(champ) if position == "OTHER": anyOtherPositions = True break if anyOtherPositions: filtered['bad_position'] += 1 continue goal = data['goal'] assert goal in (True, False) games.append(data) goals.append(goal) if len(games) == numGames: break filterCount = len(outputData) - len(goals) assert filterCount == sum(filtered.values()) print("Loaded {} games (filtered {} = {:.1f}%)".format( len(goals), filterCount, 100 * filterCount / len(outputData))) for reason, count in sorted(filtered.items()): print("\t{}: {}".format(reason, count)) print() return games, goals
def addStockInfoByFile(self, fname): stockInfo = util.loadJsonFile(fname) code = stockInfo['code'] if code not in self.importStockDict: self.importStockDict[code] = {} self.importStockDict[code].update(stockInfo)
def load_options(self): options = util.loadJsonFile(self.path) #Add line per params self.x = options["x"] self.y = options["y"] self.windowType = options["wType"]