def parseAndDownloadTeamYearlyStandings(self):
files = findExt(self.getSeasonDir(), ext=".p", debug=False)
for ifile in files:
year = getBaseFilename(ifile)
htmldata = getFile(ifile)
bsdata = getHTML(htmldata)
idVals = {}
links = bsdata.findAll("a")
for link in links:
attrs = link.attrs
if attrs.get("data-clubhouse-uid") is not None:
href = attrs['href']
name = getBasename(href)
idval = getBasename(getDirname(href))
if idVals.get(idval) is not None:
if idVals[idval] != name:
raise ValueError("Error in ID for this year!")
idVals[idval] = name
for idVal, name in idVals.items():
self.downloadTeamDataByYear(idVal,
name,
season=str(year),
debug=True)
def main(args):
cwd = getcwd()
albumSegments = {}
discSegments = {}
for ifile in findPattern("./", pattern="."):
mid = mp3ID(ifile)
try:
mid = mp3ID(ifile)
except:
print("Error reading file {0}".format(ifile))
continue
album = mid.getAlbum()
print("Album: {0}".format(album))
if album is not None:
album = album[0]
if albumSegments.get(album) is None:
albumSegments[album] = []
albumSegments[album].append(ifile)
disc = mid.getDiscNumber()
if disc is not None:
disc = disc[0]
if discSegments.get(disc) is None:
discSegments[disc] = []
discSegments[disc].append(ifile)
if args.album is True:
print("Album Segments: {0}".format(albumSegments.keys()))
for album, albumFiles in albumSegments.items():
albumDir = setDir(cwd, album)
mkDir(albumDir)
for ifile in albumFiles:
src = ifile
dst = setFile(albumDir, getBasename(ifile))
print("Moving [{0}] to [{1}]".format(src, dst))
moveFile(src, dst, debug=True)
if args.disc is True:
print("Disc Segments: {0}".format(discSegments.keys()))
for disc, discFiles in discSegments.items():
discDir = setDir(cwd, "Disc {0}".format(disc))
mkDir(discDir)
for ifile in discFiles:
src = ifile
dst = setFile(discDir, getBasename(ifile))
#print("Moving [{0}] to [{1}]".format(src, dst))
moveFile(src, dst, debug=True)
def main(args):
files = findExt(getcwd(), ext=".mp3")
for ifile in files:
fname = getBasename(ifile)
dname = getDirname(ifile)
fname = fname.replace(args.remove, "").strip()
dst = join(dname, fname)
if ifile != dst:
moveFile(ifile, dst, debug=True)
def moveExtArtistFilesToNewModValue(self, newModVal):
artistsDir = self.getArtistsDir()
extArtistsDir = self.getArtistsExtraDir()
dutils = discogsUtils()
files = glob(join(extArtistsDir, "*.p"))
print("Moving {0} files".format(len(files)))
for ifile in files:
fbasics = getFileBasics(ifile)
fname = getBasename(ifile)
discID = fbasics[1].split('-')[0]
modValue = dutils.getDiscIDHashMod(
discID=discID, modval=newModVal) #disc.getMaxModVal())
src = ifile
dst = join(artistsDir, str(modValue), fname)
moveFile(src, dst)
def moveArtistFilesToNewModValue(self, newModValue, oldModValue):
filedir = self.getArtistsDir()
dutils = discogsUtils()
for modVal in range(oldModValue):
modValue = dutils.getDiscIDHashMod(
discID=modVal, modval=newModVal) #disc.getMaxModVal())
if modVal == modValue:
sleep(1)
continue
else:
files = glob(join(filedir, str(modVal), "*.p"))
print("Moving {0} files from {1} to {2}".format(
len(files), modVal, modValue))
for ifile in files:
fbasics = getFileBasics(ifile)
fname = getBasename(ifile)
src = ifile
dst = join(artistsDir, str(modValue), fname)
moveFile(src, dst)
def getPaths(self, ifile, errors='ignore'):
if self.debug:
print("Start: {0}".format(ifile))
print("isFile: {0}".format(isFile(ifile)))
mfile = ifile
self.stripBase(ifile)
if self.debug:
print("Class: {0}".format(self.mClass))
print("File: {0}".format(self.mFile))
filename = None
ext = None
## Try Artist/Album/File.mp3
dirval, fval = getDirname(self.mFile), getBasename(self.mFile)
if getExt(fval) in self.musicext:
filename = fval
ext = getExt(fval)
if self.debug:
print("Found File: {0}".format(fval))
print("Upstream: {0}".format(dirval))
## Try Artist/Album
if dirval is not None:
artval, albval = getDirname(dirval), getBasename(dirval)
if self.debug:
print("Artval/Albval: {0}\t{1}".format(artval, albval))
print("Artval/Albval: {0}\t{1}".format(
getBasename(artval), getBasename(albval)))
print("Artval/Albval: {0}\t{1}".format(
getDirname(artval), getDirname(albval)))
if not all([artval, albval]):
## Assume single path is artist
artval = albval
self.mArtist = artval
elif not any([getDirname(artval), getDirname(albval)]):
## Assume artist, album path
if self.debug:
print("Artist: {0}".format(artval))
print("Album: {0}".format(albval))
self.mArtist = artval
self.mAlbum = albval
pass
else:
## Assume last path is the disc
self.mDisc = albval
artval, albval = getDirname(artval), getBasename(artval)
if self.debug:
print("Artval/Albval: {0}\t{1}".format(
artval, albval))
print("Artval/Albval: {0}\t{1}".format(
getBasename(artval), getBasename(albval)))
print("Artval/Albval: {0}\t{1}".format(
getDirname(artval), getDirname(albval)))
if not all([artval, albval]):
## Assume single path is artist
artval = albval
self.mArtist = artval
elif not any([getDirname(artval), getDirname(albval)]):
## Assume artist, album path
if self.debug:
print("Artist: {0}".format(artval))
print("Album: {0}".format(albval))
self.mArtist = artval
self.mAlbum = albval
pass
else:
artval1, artval2 = getDirname(artval), getBasename(
artval)
if not all([artval1, artval2]):
raise ValueError(
"This shouldn't happen: {0}".format(artval))
else:
self.mArtist = artval2
self.mAlbum = self.mDisc
if errors != 'ignore':
raise ValueError(
"Not sure what to do with {0}, {1}".format(
artval1, artval2))
else:
print("Ext: {0}".format(getExt(fval)))
raise ValueError("Not sure how to parse file: {0}".format(ifile))
if self.debug:
print("Dirname: {0}".format(dirval))
print("Filename: {0}".format(filename))
return pbClass(self.mClass, self.mArtist, self.mAlbum, self.mDisc,
filename, ext)
def main(args):
print(args)
args = addDefault(args)
if args.artist:
print('Artist = {!r}'.format(args.artist))
outdir = mkDir(setDir(args.dir, "Artist"))
if args.album:
print('Album = {!r}'.format(args.album))
outdir = mkDir(setDir(args.dir, "Album"))
if args.albumartist:
print('AlbumArtist = {!r}'.format(args.albumartist))
outdir = mkDir(setDir(args.dir, "AlbumArtist"))
if args.disc:
print('Disc = {!r}'.format(args.disc))
outdir = mkDir(setDir(args.dir, "Disc"))
if args.track:
print('Track = {!r}'.format(args.track))
outdir = mkDir(setDir(args.dir, "Track"))
if args.dir:
print('Dir = {!r}'.format(args.dir))
tomove = []
files = findWalk(args.dir)
for idir, ifiles in files.items():
for jf, ifile in enumerate(ifiles):
results = MusicID(ifile, debug=args.debug)
if results.skip is True:
continue
tags = results.getInfo()
if args.artist:
if args.artist == tags["Artist"][0]:
tomove.append(ifile)
continue
if args.albumartist:
if args.albumartist == tags["AlbumArtist"][0]:
tomove.append(ifile)
continue
if args.album:
if args.album == tags["Album"][0]:
tomove.append(ifile)
continue
if args.disc:
print(args.disc, tags["DiscNo"][0])
if args.disc == tags["DiscNo"][0]:
tomove.append(ifile)
continue
if args.track:
if args.track == tags["TrackNo"][0]:
tomove.append(ifile)
continue
if len(tomove) > 0:
for ifile in tomove:
src = ifile
fname = getBasename(ifile)
dst = setFile(outdir, fname)
moveFile(src, dst)
print("Moved {0} ===> {1}".format(src, dst))
else:
removeDir(outdir)
print("Didn't find any matching files...")
def getPaths(self, ifile, errors='ignore'):
if self.debug:
print("Name: {0}".format(ifile))
print("isFile: {0}".format(isFile(ifile)))
mfile = ifile
self.stripBase(ifile)
filename = None
ext = None
##
## Test for Album/File Structure
##
if self.known == "Artist":
dirval,fileval = getDirname(self.mFile),getBasename(self.mFile)
nextDirVal,nextFileVal = getDirname(dirval),getBasename(dirval)
## Get File Info
filename = fileval
ext = getExt(fileval)
if len(nextDirVal) == 0 and nextFileVal == dirval:
self.mAlbum = dirval
else:
self.mAlbum = dirval
self.mDisc = nextDirVal
##
## Test for Artist/Album/File Structure
##
if self.known == "Class":
dirval,fileval = getDirname(self.mFile),getBasename(self.mFile)
nextDirval,nextFileval = getDirname(dirval),getBasename(dirval)
nextNextDirval,nextNextFileval = getDirname(nextDirval),getBasename(nextDirval)
## Get File Info
filename = fileval
ext = getExt(fileval)
#ifile = "/Volumes/Music/Matched/test/AC-DC/Let There Be Rock - Live In Paris/CD1/01 -Live Wire.mp3"
#{'pbClass': 'test', 'pbArtist': 'AC-DC/Let There Be Rock - Live In Paris', 'pbAlbum': 'CD1', 'pbDisc': None, 'pbFile': '01 -Live Wire.mp3', 'pbExt': '.mp3'}
if len(nextNextDirval) == 0 and nextNextFileval == nextDirval:
self.mArtist = nextDirval
self.mAlbum = nextFileval
else:
self.mArtist = nextNextDirval
self.mAlbum = nextNextFileval
self.mDisc = nextFileval
##
## Test for File Structure
##
if self.known == "Album":
dirval,fileval = getDirname(self.mFile),getBasename(self.mFile)
nextDirVal,nextFileVal = getDirname(dirval),getBasename(dirval)
## Get File Info
filename = fileval
ext = getExt(fileval)
if len(dirval) == 0:
pass
else:
self.mDisc = dirval
return pbClass(self.mClass, self.mArtist, self.mAlbum, self.mDisc, filename, ext)
def parseTeamYearlyStandings(self,
startYear=2003,
endYear=2018,
debug=False,
verydebug=False):
for year in range(startYear, endYear + 1):
seasonDir = self.getYearlySeasonDir(year)
files = findExt(seasonDir, ext=".p", debug=False)
seasonData = season(year)
for ifile in files:
nameyear = getBaseFilename(ifile)
htmldata = getFile(ifile)
bsdata = getHTML(htmldata)
teamName = nameyear.replace("-{0}".format(year), "")
metadata = bsdata.find("meta", {"property": "og:url"})
if metadata is None:
raise ValueError(
"Could not find basic team meta data for this file! {0}"
.format(ifile))
try:
content = metadata.attrs['content']
year = getBasename(content)
teamID = getBasename(getDirname(getDirname(content)))
except:
raise ValueError(
"Could not get team year and ID from meta data: {0}".
format(metadata))
if verydebug:
print(year, '\t', teamID, '\t', ifile)
## Create Team Object
teamData = team(year=year,
teamName=teamName,
teamMascot=None,
teamID=teamID)
tables = bsdata.findAll("table", {"class": "Table2__table"})
if verydebug:
print("\tFound {0} game tables".format(len(tables)))
for it, table in enumerate(tables):
trs = table.findAll("tr")
headers = trs[1]
headers = [
x.text for x in headers.findAll("td") if x is not None
]
gameRows = trs[2:]
totalGames = len(gameRows)
if verydebug:
print("\tFound {0} potential games".format(totalGames))
for ig, tr in enumerate(gameRows):
tds = tr.findAll("td")
gameData = dict(zip(headers, tds))
extra = {"OT": False, "Bowl": False}
## Get the Date
try:
date = gameData["Date"]
except:
print(ifile)
raise ValueError(
"No date for this game! {0}".format(gameData))
date = date.text
## Only Keep Games With Regular Dates
try:
dateval = "{0} {1}".format(
date.split(", ")[-1], year)
date = getDateTime(dateval)
except:
date = None
if date is None:
continue
## Check for January Games (in the following year)
if date.month == 1:
date = addMonths(date, 12)
## Get the Opponent
try:
opponent = gameData["Opponent"]
except:
raise ValueError(
"No opponent for this game! {0}".format(game))
try:
oppolink = opponent.find("a")
oppohref = oppolink.attrs['href']
opponame = getBasename(oppohref)
oppoID = getBasename(getDirname(oppohref))
except:
opponame = opponent.text
oppoID = 0
#raise ValueError("Could not find href in link! {0}".format(opponent))
try:
gamespan = opponent.find("span", {"class": "pr2"})
gametype = gamespan.text
except:
raise ValueError(
"Could not find game type from {0}".format(
opponent))
if gametype == "vs":
location = teamID
elif gametype == "@":
location = oppoID
else:
raise ValueError(
"Location --> {0}".format(gametype))
if verydebug:
print("\t{0}/{1}\t{2}\t{3: <4}{4: <50}".format(
ig, totalGames, printDateTime(date), gametype,
opponame),
end="\t")
## Get the Result
try:
result = gameData["Result"]
except:
raise ValueError(
"No result for this game! {0}".format(game))
spans = result.findAll("span")
if len(spans) == 0:
continue
if len(spans) != 2:
raise ValueError(
"There are {0} spans in this row!: {1}".format(
len(spans), result))
outcome = spans[0].text.strip()
score = spans[1].text.strip()
if score.endswith("OT"):
extra = {"OT": True}
score = score[:-3].strip()
try:
scores = [int(x) for x in score.split('-')]
except:
raise ValueError(
"Could not create integer scores from {0}".
format(spans))
if outcome == 'W':
teamScore = scores[0]
oppoScore = scores[1]
teamResult = "W"
oppoResult = "L"
elif outcome == "L":
teamScore = scores[1]
oppoScore = scores[0]
teamResult = "L"
oppoResult = "W"
elif outcome == "T":
teamScore = scores[0]
oppoScore = scores[1]
teamResult = "T"
oppoResult = "T"
else:
raise ValueError(
"Did not recognize game outcome {0}".format(
outcome))
## Get the Game
try:
gamelink = result.find("a")
gamehref = gamelink.attrs['href']
except:
raise ValueError(
"Could not find href in link! {0}".format(
result))
if verydebug:
print("{0} {1}".format(
teamResult, "-".join(
str(x) for x in [teamScore, oppoScore])))
## Create game object
gameData = game(gameID=gameID,
date=date,
teamA=teamID,
teamB=oppoID,
teamAResult=teamResult,
teamBResult=oppoResult,
teamAScore=teamScore,
teamBScore=oppoScore,
location=location)
## Append game to team data
teamData.addGame(gameData)
## Show Summary
teamData.setStatistics()
if debug:
teamData.summary()
if teamData.ngames == 0:
removeFile(ifile, debug=True)
seasonData.addTeam(teamData)
#http://www.espn.com/college-football/team/schedule/_/id/201/season/2005"
savename = setFile(self.getSeasonResultsDir(),
"{0}.p".format(year))
saveFile(idata=seasonData, ifile=savename, debug=True)