class LDSClient(object):
vals = ValueGetters()
def __init__(self, root, username, password):
self._session = None
self.username = username
self.password = password
self.root = FilePath(root)
self.raw_root = self.root.child('raw')
if not self.root.exists():
self.root.makedirs()
if not self.raw_root.exists():
self.raw_root.makedirs()
self.photo_root = self.root.child('photos')
if not self.photo_root.exists():
self.photo_root.makedirs()
def assertOk(self, response):
if not response.ok:
sys.stderr.write('not okay: %r\n' % (response, ))
sys.stderr.write(repr(response.text)[:200] + '\n')
raise Exception('response not okay', response)
def log(self, message):
sys.stderr.write(message + '\n')
def authenticate(self):
if self._session:
return self._session
self.log('Signing in...')
s = self._session = requests.session()
r = s.get('https://ident.lds.org/sso/UI/Login')
r = s.post('https://ident.lds.org/sso/UI/Login',
params={
'IDToken1': self.username,
'IDToken2': self.password,
'IDButton': 'Log In',
})
self.assertOk(r)
return self._session
def storeRawValue(self, filename, value):
self.raw_root.child(filename).setContent(json.dumps(value))
def hasRawValue(self, filename):
fp = self.raw_root.child(filename)
if fp.exists():
return fp
return None
def getRawValue(self, filename, default_value=None):
fp = self.hasRawValue(filename)
if fp:
return json.loads(fp.getContent())
else:
return default_value
def updateRawData(self):
for name, func in self.vals.raws.items():
if self.hasRawValue(name):
# already has a value; do not recompute
self.log('[%s] data already present' % (name, ))
continue
self.log('[%s] fetching...' % (name, ))
data = func(self)
self.storeRawValue(name, data)
@vals.raw('unit_number')
def get_unitNumber(self):
s = self.authenticate()
r = s.get('https://www.lds.org/mls/mbr/records/member-list?lang=eng')
self.assertOk(r)
# this is probably pretty fragile...
re_unit_number = re.compile(r"window.unitNumber\s=\s'(.*?)';")
m = re_unit_number.search(r.text)
return m.groups()[0]
@vals.raw('member_list')
def get_memberList(self):
s = self.authenticate()
unit_number = self.getRawValue('unit_number')
r = s.get('https://www.lds.org/mls/mbr/services/report/member-list',
params={
'lang': 'eng',
'unitNumber': unit_number,
})
self.assertOk(r)
return r.json()
@vals.raw('members_with_callings')
def get_membersWithCallings(self):
s = self.authenticate()
unit_number = self.getRawValue('unit_number')
r = s.get(
'https://www.lds.org/mls/mbr/services/report/members-with-callings',
params={
'lang': 'eng',
'unitNumber': unit_number,
},
headers={
'Accept': 'application/json',
})
self.assertOk(r)
return r.json()
@vals.raw('members_without_callings')
def get_membersWithoutCallings(self):
s = self.authenticate()
unit_number = self.getRawValue('unit_number')
r = s.get(
'https://www.lds.org/mls/mbr/services/orgs/members-without-callings',
params={
'lang': 'eng',
'unitNumber': unit_number,
},
headers={
'Accept': 'application/json',
})
self.assertOk(r)
return r.json()
#----------------------------
# photos
#----------------------------
def _memberPhotoFilePath(self, member_id, size='large', ext='jpg'):
"""
Valid size options are:
- large
- medium
- original
- thumbnail
"""
return self.photo_root.child('solo-%s-%s.%s' % (member_id, size, ext))
def _memberIDsWithNoPhoto(self, size='large'):
members = self.getRawValue('member_list')
for member in members:
member_id = member['id']
photo_fp = self._memberPhotoFilePath(member_id, size)
if photo_fp.exists():
continue
yield member_id
def updatePhotos(self, size='large'):
s = self.authenticate()
self.log('Getting photos...')
for member_ids in xAtATime(self._memberIDsWithNoPhoto(size), 19):
if not member_ids:
continue
try:
r = s.get(
'https://www.lds.org/directory/services/ludrs/photo/url/' +
','.join(map(str, member_ids)) + '/individual')
data = r.json()
except ValueError:
print 'Error on', member_ids
raise
for member_id, result in zip(member_ids, data):
fp = self._memberPhotoFilePath(member_id, size)
uri = result[size + 'Uri']
if uri:
print 'fetching photo for', member_id
uri = 'https://www.lds.org' + uri
image_data = s.get(uri)
content_type = image_data.headers['content-type']
if content_type != 'image/jpeg':
print 'NON-JPEG: ', content_type, member_id
continue
fp.setContent(image_data.content)
else:
print 'no photo for', member_id
time.sleep(0.5)
def tagfiles(albumdir, album, options):
'''Rename and tag files using freedb information for
the specified album.'''
# XXX: doesn't really belong here
missing = album.validate()
if len(missing) > 0:
if options.strict:
amiss = ",".join(missing)
raise TagIncompleteWarning(amiss)
else:
for miss in missing:
print TagIncompleteWarning(miss)
print
album.ignoreMissing(True)
localalbum = LocalAlbumInfo(albumdir)
filelist = localalbum.getfilelist(options.sysencoding)
a = len(album.tracks)
b = len(filelist)
if a != b:
raise NamingMuseError('there are %d files, but %d metainfo tracks' % (b,a))
namebinder = get_namebinder(options, filelist)
tracks = namebinder(filelist, album, options.sysencoding)
if not sortedcmp(tracks, album.tracks):
options.dryrun = True
print NamingMuseError("binding was not exact, forcing dry run")
print "Tagging album: %s, %s - %s, %s.\n" % \
(album.year, album.artist, album.title, album.genre)
# Process files
renamealbum = True
renameTempDir = FilePath(albumdir, 'rename-tmp')
if renameTempDir.exists():
raise NamingMuseError('%s exists!' % renameTempDir)
renameTempDir.mkdir()
# a list of tuples (from, to)
# we use temporary renames to avoid filename collision on filename swaps
# this holds the list of final renames to be executed
finalrenames = []
# a list of tuples (from, to)
# used to rollback renames in case of error
rollback = []
renamesign = "->"
rollbacksign = '<-'
if options.tagonly:
renamesign = "-tag->"
if options.dryrun:
renamesign = "-dry->"
try:
for i in range(0, len(filelist)):
fpath = filelist[i]
# XXX: move bug check to freedbalbuminfo parser
#if album.isVarious:
# if not "/" in title and "-" in title:
# # workaround: this is a bug in the freedb entry
# # (according to submission guidelines)
# trackartist, title = title.split("-")
# print NamingMuseWarning("bugged database entry with - instead of /")
# else:
# trackartist, title = title.split("/")
# trackartist, title = trackartist.strip(), title.strip()
#else:
# trackartist = albumartist
track = tracks[i]
tofile = policy.genfilename(filelist[i], album, track)
tofile = tofile.encode(options.sysencoding)
totmpfile = FilePath(renameTempDir, tofile, encoding=options.sysencoding)
tofinalfile = FilePath(albumdir, tofile, encoding=options.sysencoding)
# Tag and rename file
print fpath.getName()
print "\t", colorize(renamesign), tofinalfile.getName()
if not options.dryrun:
# Tag file
#preserve stat
fd = tempfile.NamedTemporaryFile()
tmpfilename = fd.name
shutil.copystat(str(fpath), tmpfilename)
# tag the file
tagfile(fpath, album, track, options)
# restore filestat
shutil.copystat(tmpfilename, str(fpath))
# deletes tempfile
fd.close()
# Rename file to temporary name
if not options.tagonly:
if totmpfile.exists():
raise NamingMuseError('tried to rename file over existing: %s' % str(totmpfile))
if fpath != totmpfile:
fpath.rename(totmpfile)
rollback.append((fpath, totmpfile))
finalrenames.append((totmpfile, tofinalfile))
except Exception, e:
print
print colorize('Error: an error occurred. rolling back %d renames.' % len(rollback))
for frompath, topath in reversed(rollback):
print frompath.getName()
print "\t", colorize(rollbacksign), topath.getName()
topath.rename(frompath)
renameTempDir.rmdir()
raise
newalbumdir = FilePath(albumdir.getParent(), newalbum, \
encoding=options.sysencoding)
# Make parent directory of albumdir if needed
parent = newalbumdir.getParent()
if not parent.isdir():
parent.mkdir()
# Rename album (if no "manual" mp3 files in that dir)
rollbacksign = '<-'
renamesign = "->"
if options.dryrun or options.tagonly:
renamesign = "-dry->"
if not (options.dryrun or options.tagonly) and renamealbum \
and str(albumdir) != str(newalbumdir):
if newalbumdir.exists():
raise NamingMuseWarning("Directory already exists (dup album?): " +
str(newalbumdir))
try:
albumdir.rename(newalbumdir)
except OSError, err:
raise NamingMuseWarning(str(err))
# Print rename message
print "\n", albumdir.getName()
print "\t", colorize(renamesign),
if needartistdirmove:
print FilePath(album.artist.encode(options.sysencoding), newalbumdir.getName())
else:
print newalbumdir.getName()
def tagfiles(albumdir, album, options):
'''Rename and tag files using freedb information for
the specified album.'''
# XXX: doesn't really belong here
missing = album.validate()
if len(missing) > 0:
if options.strict:
amiss = ",".join(missing)
raise TagIncompleteWarning(amiss)
else:
for miss in missing:
print TagIncompleteWarning(miss)
print
album.ignoreMissing(True)
localalbum = LocalAlbumInfo(albumdir)
filelist = localalbum.getfilelist(options.sysencoding)
a = len(album.tracks)
b = len(filelist)
if a != b:
raise NamingMuseError('there are %d files, but %d metainfo tracks' %
(b, a))
namebinder = get_namebinder(options, filelist)
tracks = namebinder(filelist, album, options.sysencoding)
if not sortedcmp(tracks, album.tracks):
options.dryrun = True
print NamingMuseError("binding was not exact, forcing dry run")
print "Tagging album: %s, %s - %s, %s.\n" % \
(album.year, album.artist, album.title, album.genre)
# Process files
renamealbum = True
renameTempDir = FilePath(albumdir, 'rename-tmp')
if renameTempDir.exists():
raise NamingMuseError('%s exists!' % renameTempDir)
renameTempDir.mkdir()
# a list of tuples (from, to)
# we use temporary renames to avoid filename collision on filename swaps
# this holds the list of final renames to be executed
finalrenames = []
# a list of tuples (from, to)
# used to rollback renames in case of error
rollback = []
renamesign = "->"
rollbacksign = '<-'
if options.tagonly:
renamesign = "-tag->"
if options.dryrun:
renamesign = "-dry->"
try:
for i in range(0, len(filelist)):
fpath = filelist[i]
# XXX: move bug check to freedbalbuminfo parser
#if album.isVarious:
# if not "/" in title and "-" in title:
# # workaround: this is a bug in the freedb entry
# # (according to submission guidelines)
# trackartist, title = title.split("-")
# print NamingMuseWarning("bugged database entry with - instead of /")
# else:
# trackartist, title = title.split("/")
# trackartist, title = trackartist.strip(), title.strip()
#else:
# trackartist = albumartist
track = tracks[i]
tofile = policy.genfilename(filelist[i], album, track)
tofile = tofile.encode(options.sysencoding)
totmpfile = FilePath(renameTempDir,
tofile,
encoding=options.sysencoding)
tofinalfile = FilePath(albumdir,
tofile,
encoding=options.sysencoding)
# Tag and rename file
print fpath.getName()
print "\t", colorize(renamesign), tofinalfile.getName()
if not options.dryrun:
# Tag file
#preserve stat
fd = tempfile.NamedTemporaryFile()
tmpfilename = fd.name
shutil.copystat(str(fpath), tmpfilename)
# tag the file
tagfile(fpath, album, track, options)
# restore filestat
shutil.copystat(tmpfilename, str(fpath))
# deletes tempfile
fd.close()
# Rename file to temporary name
if not options.tagonly:
if totmpfile.exists():
raise NamingMuseError(
'tried to rename file over existing: %s' %
str(totmpfile))
if fpath != totmpfile:
fpath.rename(totmpfile)
rollback.append((fpath, totmpfile))
finalrenames.append((totmpfile, tofinalfile))
except Exception, e:
print
print colorize('Error: an error occurred. rolling back %d renames.' %
len(rollback))
for frompath, topath in reversed(rollback):
print frompath.getName()
print "\t", colorize(rollbacksign), topath.getName()
topath.rename(frompath)
renameTempDir.rmdir()
raise
newalbumdir = FilePath(albumdir.getParent(), newalbum, \
encoding=options.sysencoding)
# Make parent directory of albumdir if needed
parent = newalbumdir.getParent()
if not parent.isdir():
parent.mkdir()
# Rename album (if no "manual" mp3 files in that dir)
rollbacksign = '<-'
renamesign = "->"
if options.dryrun or options.tagonly:
renamesign = "-dry->"
if not (options.dryrun or options.tagonly) and renamealbum \
and str(albumdir) != str(newalbumdir):
if newalbumdir.exists():
raise NamingMuseWarning("Directory already exists (dup album?): " +
str(newalbumdir))
try:
albumdir.rename(newalbumdir)
except OSError, err:
raise NamingMuseWarning(str(err))
# Print rename message
print "\n", albumdir.getName()
print "\t", colorize(renamesign),
if needartistdirmove:
print FilePath(album.artist.encode(options.sysencoding),
newalbumdir.getName())
else:
print newalbumdir.getName()
class LDSClient(object):
vals = ValueGetters()
def __init__(self, root, username, password):
self._session = None
self.username = username
self.password = password
self.root = FilePath(root)
self.raw_root = self.root.child('raw')
if not self.root.exists():
self.root.makedirs()
if not self.raw_root.exists():
self.raw_root.makedirs()
self.photo_root = self.root.child('photos')
if not self.photo_root.exists():
self.photo_root.makedirs()
def assertOk(self, response):
if not response.ok:
sys.stderr.write('not okay: %r\n' % (response,))
sys.stderr.write(repr(response.text)[:200] + '\n')
raise Exception('response not okay', response)
def log(self, message):
sys.stderr.write(message + '\n')
def authenticate(self):
if self._session:
return self._session
self.log('Signing in...')
s = self._session = requests.session()
r = s.get('https://ident.lds.org/sso/UI/Login')
r = s.post('https://ident.lds.org/sso/UI/Login', params={
'IDToken1': self.username,
'IDToken2': self.password,
'IDButton': 'Log In',
})
self.assertOk(r)
return self._session
def storeRawValue(self, filename, value):
self.raw_root.child(filename).setContent(json.dumps(value))
def hasRawValue(self, filename):
fp = self.raw_root.child(filename)
if fp.exists():
return fp
return None
def getRawValue(self, filename, default_value=None):
fp = self.hasRawValue(filename)
if fp:
return json.loads(fp.getContent())
else:
return default_value
def updateRawData(self):
for name, func in self.vals.raws.items():
if self.hasRawValue(name):
# already has a value; do not recompute
self.log('[%s] data already present' % (name,))
continue
self.log('[%s] fetching...' % (name,))
data = func(self)
self.storeRawValue(name, data)
@vals.raw('unit_number')
def get_unitNumber(self):
s = self.authenticate()
r = s.get('https://www.lds.org/mls/mbr/records/member-list?lang=eng')
self.assertOk(r)
# this is probably pretty fragile...
re_unit_number = re.compile(r"window.unitNumber\s=\s'(.*?)';")
m = re_unit_number.search(r.text)
return m.groups()[0]
@vals.raw('member_list')
def get_memberList(self):
s = self.authenticate()
unit_number = self.getRawValue('unit_number')
r = s.get('https://www.lds.org/mls/mbr/services/report/member-list', params={
'lang': 'eng',
'unitNumber': unit_number,
})
self.assertOk(r)
return r.json()
@vals.raw('members_with_callings')
def get_membersWithCallings(self):
s = self.authenticate()
unit_number = self.getRawValue('unit_number')
r = s.get('https://www.lds.org/mls/mbr/services/report/members-with-callings', params={
'lang': 'eng',
'unitNumber': unit_number,
}, headers={
'Accept': 'application/json',
})
self.assertOk(r)
return r.json()
@vals.raw('members_without_callings')
def get_membersWithoutCallings(self):
s = self.authenticate()
unit_number = self.getRawValue('unit_number')
r = s.get('https://www.lds.org/mls/mbr/services/orgs/members-without-callings', params={
'lang': 'eng',
'unitNumber': unit_number,
}, headers={
'Accept': 'application/json',
})
self.assertOk(r)
return r.json()
#----------------------------
# photos
#----------------------------
def _memberPhotoFilePath(self, member_id, size='large', ext='jpg'):
"""
Valid size options are:
- large
- medium
- original
- thumbnail
"""
return self.photo_root.child('solo-%s-%s.%s' % (member_id, size, ext))
def _memberIDsWithNoPhoto(self, size='large'):
members = self.getRawValue('member_list')
for member in members:
member_id = member['id']
photo_fp = self._memberPhotoFilePath(member_id, size)
if photo_fp.exists():
continue
yield member_id
def updatePhotos(self, size='large'):
s = self.authenticate()
self.log('Getting photos...')
for member_ids in xAtATime(self._memberIDsWithNoPhoto(size), 19):
if not member_ids:
continue
try:
r = s.get('https://www.lds.org/directory/services/ludrs/photo/url/'+','.join(map(str, member_ids))+'/individual')
data = r.json()
except ValueError:
print 'Error on', member_ids
raise
for member_id, result in zip(member_ids, data):
fp = self._memberPhotoFilePath(member_id, size)
uri = result[size + 'Uri']
if uri:
print 'fetching photo for', member_id
uri = 'https://www.lds.org' + uri
image_data = s.get(uri)
content_type = image_data.headers['content-type']
if content_type != 'image/jpeg':
print 'NON-JPEG: ', content_type, member_id
continue
fp.setContent(image_data.content)
else:
print 'no photo for', member_id
time.sleep(0.5)
