def parseAndDownloadTeamYearlyStandings(self):
files = findExt(self.getSeasonDir(), ext=".p", debug=False)
for ifile in files:
year = getBaseFilename(ifile)
htmldata = getFile(ifile)
bsdata = getHTML(htmldata)
idVals = {}
links = bsdata.findAll("a")
for link in links:
attrs = link.attrs
if attrs.get("data-clubhouse-uid") is not None:
href = attrs['href']
name = getBasename(href)
idval = getBasename(getDirname(href))
if idVals.get(idval) is not None:
if idVals[idval] != name:
raise ValueError("Error in ID for this year!")
idVals[idval] = name
for idVal, name in idVals.items():
self.downloadTeamDataByYear(idVal,
name,
season=str(year),
debug=True)
def main(args):
if args.dir is None:
args.dir = getcwd()
pb = pathBasics(albumDir=args.dir)
files = pb.getFiles()
artistDir = getDirname(args.dir)
for albumDir, filevals in files.items():
retval = genMIDTags(albumDir, artistDir, files=filevals, args=args)
def main(args):
files = findExt(getcwd(), ext=".mp3")
for ifile in files:
fname = getBasename(ifile)
dname = getDirname(ifile)
fname = fname.replace(args.remove, "").strip()
dst = join(dname, fname)
if ifile != dst:
moveFile(ifile, dst, debug=True)
def rmIDFiles(self, artistID):
print("Removing files artistID {0}".format(artistID))
savename = self.getArtistSavename(artistID)
if isFile(savename):
files = [savename]
else:
files = []
from glob import glob
from os.path import join
from fileUtils import getDirname
files += glob(join(getDirname(savename), "extra", "{0}-*.p".format(artistID)))
print("Found {0} files to delete.".format(len(files)))
from fsUtils import removeFile
for ifile in files:
removeFile(ifile)
print("Removed File {0}".format(ifile))
def main(args):
args = addDefault(args)
print('Artist = {!r}'.format(args.artist))
print('Album = {!r}'.format(args.album))
print('Class = {!r}'.format(args.cls))
print('Dir = {!r}'.format(args.dir))
if args.album is True:
pb = pathBasics(albumDir=args.dir)
files = pb.getFiles()
artistDir = getDirname(args.dir)
for albumDir, filevals in files.items():
retval = testAlbum(albumDir, artistDir, files=filevals)
actionOnAlbum(albumDir, artistDir, retval)
#print(retval)
if args.artist is True:
artistDir = args.dir
pb = pathBasics(artistDir=artistDir)
files = pb.getFiles()
print("\n")
print("=" * 60)
print("===", artistDir, "===")
print("=" * 60)
for albumDir, filevals in files.items():
retval = testAlbum(albumDir, artistDir, files=filevals)
actionOnAlbum(albumDir, artistDir, retval)
#print(retval)
if args.cls is True:
artistDirs = findDirs(args.dir)
for artistDir in artistDirs:
pb = pathBasics(artistDir=artistDir)
files = pb.getFiles()
print("\n")
print("=" * 60)
print("===", artistDir, "===")
print("=" * 60)
for albumDir, filevals in files.items():
retval = testAlbum(albumDir, artistDir, files=filevals)
actionOnAlbum(albumDir, artistDir, retval)
def moveAlbumFilesToNewModValue(self, newModValue, oldModValue):
filedir = self.getAlbumsDir()
dutils = discogsUtils()
for modVal in range(oldModValue):
modValue = dutils.getDiscIDHashMod(
discID=modVal, modval=newModValue) #disc.getMaxModVal())
if modVal == modValue:
sleep(1)
continue
else:
dirs = glob(join(filedir, str(modVal), "*"))
print("Moving {0} directories from {1} to {2}".format(
len(dirs), modVal, modValue))
for idir in dirs:
dname = getDirname(idir)
src = idir
dst = join(filedir, str(modValue), dname)
print(src)
print(dst)
1 / 0
moveDir(src, dst)
def getPaths(self, ifile, errors='ignore'):
if self.debug:
print("Start: {0}".format(ifile))
print("isFile: {0}".format(isFile(ifile)))
mfile = ifile
self.stripBase(ifile)
if self.debug:
print("Class: {0}".format(self.mClass))
print("File: {0}".format(self.mFile))
filename = None
ext = None
## Try Artist/Album/File.mp3
dirval, fval = getDirname(self.mFile), getBasename(self.mFile)
if getExt(fval) in self.musicext:
filename = fval
ext = getExt(fval)
if self.debug:
print("Found File: {0}".format(fval))
print("Upstream: {0}".format(dirval))
## Try Artist/Album
if dirval is not None:
artval, albval = getDirname(dirval), getBasename(dirval)
if self.debug:
print("Artval/Albval: {0}\t{1}".format(artval, albval))
print("Artval/Albval: {0}\t{1}".format(
getBasename(artval), getBasename(albval)))
print("Artval/Albval: {0}\t{1}".format(
getDirname(artval), getDirname(albval)))
if not all([artval, albval]):
## Assume single path is artist
artval = albval
self.mArtist = artval
elif not any([getDirname(artval), getDirname(albval)]):
## Assume artist, album path
if self.debug:
print("Artist: {0}".format(artval))
print("Album: {0}".format(albval))
self.mArtist = artval
self.mAlbum = albval
pass
else:
## Assume last path is the disc
self.mDisc = albval
artval, albval = getDirname(artval), getBasename(artval)
if self.debug:
print("Artval/Albval: {0}\t{1}".format(
artval, albval))
print("Artval/Albval: {0}\t{1}".format(
getBasename(artval), getBasename(albval)))
print("Artval/Albval: {0}\t{1}".format(
getDirname(artval), getDirname(albval)))
if not all([artval, albval]):
## Assume single path is artist
artval = albval
self.mArtist = artval
elif not any([getDirname(artval), getDirname(albval)]):
## Assume artist, album path
if self.debug:
print("Artist: {0}".format(artval))
print("Album: {0}".format(albval))
self.mArtist = artval
self.mAlbum = albval
pass
else:
artval1, artval2 = getDirname(artval), getBasename(
artval)
if not all([artval1, artval2]):
raise ValueError(
"This shouldn't happen: {0}".format(artval))
else:
self.mArtist = artval2
self.mAlbum = self.mDisc
if errors != 'ignore':
raise ValueError(
"Not sure what to do with {0}, {1}".format(
artval1, artval2))
else:
print("Ext: {0}".format(getExt(fval)))
raise ValueError("Not sure how to parse file: {0}".format(ifile))
if self.debug:
print("Dirname: {0}".format(dirval))
print("Filename: {0}".format(filename))
return pbClass(self.mClass, self.mArtist, self.mAlbum, self.mDisc,
filename, ext)
def main(args):
args = addDefault(args)
print('Artist = {!r}'.format(args.artist))
print('Album = {!r}'.format(args.album))
print('Class = {!r}'.format(args.cls))
print('Dir = {!r}'.format(args.dir))
if args.artist is True:
pb = pathBasics(artistDir=args.dir)
elif args.album is True:
pb = pathBasics(albumDir=args.dir)
elif args.cls is True:
pb = pathBasics(classDir=args.dir)
else:
raise ValueError("Can only run with -artist, -album, or -class!")
actions = {}
files = pb.getFiles()
for i, (dirval, filevals) in enumerate(files.items()):
print("\nDirectory: {0}".format(dirval))
header()
j = 0
errs = {}
for jf, ifile in enumerate(filevals):
results = MusicID(ifile, debug=args.debug)
if results.skip is True:
continue
tags = results.getInfo()
r = tags
pbc = pb.getPaths(ifile).getDict()
errs[ifile] = {}
j += 1
p([
j, r["DiscNo"], r["TrackNo"], r["AlbumArtist"], r["Artist"],
r["Album"], r["Title"], r["Size"]
])
if args.showpath is True:
p([
j, pbc["pbDisc"], None, pbc["pbArtist"], pbc["pbArtist"],
pbc["pbAlbum"], pbc["pbFile"], None
])
#print(pbc)
if r["AlbumArtist"][0] == pbc["pbArtist"]:
pass
else:
if pbc["pbAlbum"] in skipDirs():
pass
elif tags["AlbumArtist"][0].replace("/",
"-") == pbc["pbArtist"]:
pass
else:
errs[ifile]["Artist"] = {
"Tag": r["Artist"][0],
"Path": pbc["pbArtist"]
}
if r["Album"][0] == pbc["pbAlbum"]:
pass
else:
if pbc["pbAlbum"] in skipDirs():
pass
else:
errs[ifile]["Album"] = {
"Tag": r["Album"][0],
"Path": pbc["pbAlbum"]
}
if args.force is not None:
for ifile, fileData in errs.items():
if actions.get(dirval) is not None:
continue
if not isFile(ifile):
continue
for category, errData in fileData.items():
if args.force.title() == category:
print("Mismatch: {0}".format(args.force.title()))
print("File: {0}".format(ifile))
print("Tag: [{0}]".format(errData["Tag"]))
print("Path: [{0}]".format(errData["Path"]))
if args.usetag is True:
srcdir = dirval
if "/" in errData["Tag"]:
print("Found / in Tag. Converting to -")
errData["Tag"] = errData["Tag"].replace(
"/", "-")
if "\"" in errData["Tag"]:
print("Found \" in Tag. Converting to -")
errData["Tag"] = errData["Tag"].replace(
"\"", "")
dstdir = setDir(getDirname(dirval), errData["Tag"])
print(
"Set option to move directory based on the {0} tag"
.format(category))
print("Moving {0}".format(srcdir))
if isDir(dstdir):
print("=" * 40)
print(
"== Not moving because there is already a directory with that name..."
)
print("=" * 40)
break
print(" to {0}".format(dstdir))
if "/" in errData["Path"]:
print("=" * 40)
print(
"== Not moving because there is a sub directory..."
)
print("=" * 40)
break
if args.script is False:
print("Sleeping for 3 seconds")
for i in range(3):
print(".", end="")
sys.stdout.flush()
sleep(1)
print("\n")
moveDir(srcdir, dstdir)
break
else:
actions[srcdir] = dstdir
break
else:
return
if args.script is False:
print("\n", "=" * 60, "\n")
print("Looks Good")
print("\n\n")
else:
if len(actions) > 0:
print("#!/bin/sh\n", file=open("actions.sh", "w"))
for srcdir, dstdir in actions.items():
print("mv \"{0}\" \"{1}\"".format(srcdir, dstdir),
file=open("actions.sh", "a"))
from os import chmod
import stat
chmod("actions.sh", stat.S_IRWXU)
print("\n", "=" * 60, "\n")
print("Looks Good")
print("\n\n")
def testAlbum(albumDir, artistDir, files):
retval = {
"Track": False,
"Album": False,
"Title": False,
"Multi": False,
"Skip": False,
"Extra": False,
"Mix": False
}
artistName = getDirBasics(artistDir)[-1]
if artistName in skipDirs():
retval["Skip"] = True
#print("artistDir",artistDir)
#print("albumDir",albumDir)
#print("artistName",artistName)
albumName = albumDir.replace(artistDir, "")[1:]
if "/" in albumName:
retval["Extra"] = True
albumDirs = albumName.split("/")
if albumDirs[0] in skipDirs():
retval["Skip"] = True
if albumName in skipDirs():
retval["Skip"] = True
#print("albumName",albumName)
j = 0
tags = {}
print("\t-----> Album Info: {0} / {1} \t ==> {2} Songs".format(
artistName, albumName, len(files)))
if retval["Extra"] is True:
return retval
if retval["Skip"] is True:
return retval
ifiles = []
for jf, ifile in enumerate(files):
results = MusicID(ifile, debug=args.debug)
if results.skip is True:
continue
tags[j] = results.getInfo()
ifiles.append(ifile)
#pbcs[j] = pb.getPaths(ifile).getDict()
j += 1
nfiles = j
## Track Tests
testTrackNo = True
trackCheckSum = sum(range(1, nfiles + 1))
trackTrackSum = 0
## Album Tests
testAlbum = testCD(albumName) or testDisc(albumName)
retval["Multi"] = testAlbum
retval["Mix"] = testMix(albumName)
## Title Tests
testTitle = True
for j in range(nfiles):
ifile = ifiles[j]
tag = tags[j]
#pbc = pbcs[j]
###############################################################################################
## Album Tests
###############################################################################################
albumTag = tag.get("Album")
if albumTag is None:
print("Album Name Error ==> [{0}]".format("No Album Tag"))
retval["Album"] = True
break
try:
albumName = albumTag[0]
albumName = albumName.replace("/", " ")
except:
print("Track Number Error ==> [{0}]".format("No Value"))
trackNo = ""
if len(albumName) == 0:
retval["Album"] = True
break
dirvals = getDirBasics(getDirname(ifile))
if albumName not in dirvals:
retval["Album"] = True
###############################################################################################
## Track Number Tests
###############################################################################################
trkTag = tag.get("TrackNo")
if trkTag is None:
print("Track Number Error ==> [{0}]".format("No TrackNo Tag"))
retval["Track"] = True
break
try:
trackNo = trkTag[0]
except:
print("Track Number Error ==> [{0}]".format("No Value"))
trackNo = ""
if len(trackNo) == 0:
retval["Track"] = True
break
trackNumberValue = None
try:
trackNumberValue = int(trackNo)
except:
try:
trackVals = [int(x) for x in trackNo.split("/")]
trackNumberValue = trackVals[0]
except:
print("Track Number Error ==> [{0}]".format(trackNo))
try:
trackTrackSum += trackNumberValue
except:
pass
###############################################################################################
## Title Number Tests
###############################################################################################
titleTag = tag.get("Title")
if titleTag is None:
print("Title Error ==> [{0}]".format("No Title Tag"))
retval["Title"] = True
break
try:
title = titleTag[0]
except:
print("Title Error ==> [{0}]".format(titleTag))
testTitle = False
break
if testMix(title):
print("Possible Mix ==> [{0}]".format(title))
retval["Mix"] = True
break
fileName = getBaseFilename(ifile)
if not any([title in fileName, fileName in title]):
print("Title Error ==> [{0}] not [{1}]".format(title, fileName))
retval["Title"] = True
break
if sum(retval.values()) == 0:
if trackTrackSum != trackCheckSum:
print("Problem with track numbering...")
print(" Expected {0} and found {1}".format(
trackCheckSum, trackTrackSum))
retval["Track"] = True
#print(retval)
return retval
def getPaths(self, ifile, errors='ignore'):
if self.debug:
print("Name: {0}".format(ifile))
print("isFile: {0}".format(isFile(ifile)))
mfile = ifile
self.stripBase(ifile)
filename = None
ext = None
##
## Test for Album/File Structure
##
if self.known == "Artist":
dirval,fileval = getDirname(self.mFile),getBasename(self.mFile)
nextDirVal,nextFileVal = getDirname(dirval),getBasename(dirval)
## Get File Info
filename = fileval
ext = getExt(fileval)
if len(nextDirVal) == 0 and nextFileVal == dirval:
self.mAlbum = dirval
else:
self.mAlbum = dirval
self.mDisc = nextDirVal
##
## Test for Artist/Album/File Structure
##
if self.known == "Class":
dirval,fileval = getDirname(self.mFile),getBasename(self.mFile)
nextDirval,nextFileval = getDirname(dirval),getBasename(dirval)
nextNextDirval,nextNextFileval = getDirname(nextDirval),getBasename(nextDirval)
## Get File Info
filename = fileval
ext = getExt(fileval)
#ifile = "/Volumes/Music/Matched/test/AC-DC/Let There Be Rock - Live In Paris/CD1/01 -Live Wire.mp3"
#{'pbClass': 'test', 'pbArtist': 'AC-DC/Let There Be Rock - Live In Paris', 'pbAlbum': 'CD1', 'pbDisc': None, 'pbFile': '01 -Live Wire.mp3', 'pbExt': '.mp3'}
if len(nextNextDirval) == 0 and nextNextFileval == nextDirval:
self.mArtist = nextDirval
self.mAlbum = nextFileval
else:
self.mArtist = nextNextDirval
self.mAlbum = nextNextFileval
self.mDisc = nextFileval
##
## Test for File Structure
##
if self.known == "Album":
dirval,fileval = getDirname(self.mFile),getBasename(self.mFile)
nextDirVal,nextFileVal = getDirname(dirval),getBasename(dirval)
## Get File Info
filename = fileval
ext = getExt(fileval)
if len(dirval) == 0:
pass
else:
self.mDisc = dirval
return pbClass(self.mClass, self.mArtist, self.mAlbum, self.mDisc, filename, ext)
def parseTeamYearlyStandings(self,
startYear=2003,
endYear=2018,
debug=False,
verydebug=False):
for year in range(startYear, endYear + 1):
seasonDir = self.getYearlySeasonDir(year)
files = findExt(seasonDir, ext=".p", debug=False)
seasonData = season(year)
for ifile in files:
nameyear = getBaseFilename(ifile)
htmldata = getFile(ifile)
bsdata = getHTML(htmldata)
teamName = nameyear.replace("-{0}".format(year), "")
metadata = bsdata.find("meta", {"property": "og:url"})
if metadata is None:
raise ValueError(
"Could not find basic team meta data for this file! {0}"
.format(ifile))
try:
content = metadata.attrs['content']
year = getBasename(content)
teamID = getBasename(getDirname(getDirname(content)))
except:
raise ValueError(
"Could not get team year and ID from meta data: {0}".
format(metadata))
if verydebug:
print(year, '\t', teamID, '\t', ifile)
## Create Team Object
teamData = team(year=year,
teamName=teamName,
teamMascot=None,
teamID=teamID)
tables = bsdata.findAll("table", {"class": "Table2__table"})
if verydebug:
print("\tFound {0} game tables".format(len(tables)))
for it, table in enumerate(tables):
trs = table.findAll("tr")
headers = trs[1]
headers = [
x.text for x in headers.findAll("td") if x is not None
]
gameRows = trs[2:]
totalGames = len(gameRows)
if verydebug:
print("\tFound {0} potential games".format(totalGames))
for ig, tr in enumerate(gameRows):
tds = tr.findAll("td")
gameData = dict(zip(headers, tds))
extra = {"OT": False, "Bowl": False}
## Get the Date
try:
date = gameData["Date"]
except:
print(ifile)
raise ValueError(
"No date for this game! {0}".format(gameData))
date = date.text
## Only Keep Games With Regular Dates
try:
dateval = "{0} {1}".format(
date.split(", ")[-1], year)
date = getDateTime(dateval)
except:
date = None
if date is None:
continue
## Check for January Games (in the following year)
if date.month == 1:
date = addMonths(date, 12)
## Get the Opponent
try:
opponent = gameData["Opponent"]
except:
raise ValueError(
"No opponent for this game! {0}".format(game))
try:
oppolink = opponent.find("a")
oppohref = oppolink.attrs['href']
opponame = getBasename(oppohref)
oppoID = getBasename(getDirname(oppohref))
except:
opponame = opponent.text
oppoID = 0
#raise ValueError("Could not find href in link! {0}".format(opponent))
try:
gamespan = opponent.find("span", {"class": "pr2"})
gametype = gamespan.text
except:
raise ValueError(
"Could not find game type from {0}".format(
opponent))
if gametype == "vs":
location = teamID
elif gametype == "@":
location = oppoID
else:
raise ValueError(
"Location --> {0}".format(gametype))
if verydebug:
print("\t{0}/{1}\t{2}\t{3: <4}{4: <50}".format(
ig, totalGames, printDateTime(date), gametype,
opponame),
end="\t")
## Get the Result
try:
result = gameData["Result"]
except:
raise ValueError(
"No result for this game! {0}".format(game))
spans = result.findAll("span")
if len(spans) == 0:
continue
if len(spans) != 2:
raise ValueError(
"There are {0} spans in this row!: {1}".format(
len(spans), result))
outcome = spans[0].text.strip()
score = spans[1].text.strip()
if score.endswith("OT"):
extra = {"OT": True}
score = score[:-3].strip()
try:
scores = [int(x) for x in score.split('-')]
except:
raise ValueError(
"Could not create integer scores from {0}".
format(spans))
if outcome == 'W':
teamScore = scores[0]
oppoScore = scores[1]
teamResult = "W"
oppoResult = "L"
elif outcome == "L":
teamScore = scores[1]
oppoScore = scores[0]
teamResult = "L"
oppoResult = "W"
elif outcome == "T":
teamScore = scores[0]
oppoScore = scores[1]
teamResult = "T"
oppoResult = "T"
else:
raise ValueError(
"Did not recognize game outcome {0}".format(
outcome))
## Get the Game
try:
gamelink = result.find("a")
gamehref = gamelink.attrs['href']
except:
raise ValueError(
"Could not find href in link! {0}".format(
result))
if verydebug:
print("{0} {1}".format(
teamResult, "-".join(
str(x) for x in [teamScore, oppoScore])))
## Create game object
gameData = game(gameID=gameID,
date=date,
teamA=teamID,
teamB=oppoID,
teamAResult=teamResult,
teamBResult=oppoResult,
teamAScore=teamScore,
teamBScore=oppoScore,
location=location)
## Append game to team data
teamData.addGame(gameData)
## Show Summary
teamData.setStatistics()
if debug:
teamData.summary()
if teamData.ngames == 0:
removeFile(ifile, debug=True)
seasonData.addTeam(teamData)
#http://www.espn.com/college-football/team/schedule/_/id/201/season/2005"
savename = setFile(self.getSeasonResultsDir(),
"{0}.p".format(year))
saveFile(idata=seasonData, ifile=savename, debug=True)
