class LDSClient(object): vals = ValueGetters() def __init__(self, root, username, password): self._session = None self.username = username self.password = password self.root = FilePath(root) self.raw_root = self.root.child('raw') if not self.root.exists(): self.root.makedirs() if not self.raw_root.exists(): self.raw_root.makedirs() self.photo_root = self.root.child('photos') if not self.photo_root.exists(): self.photo_root.makedirs() def assertOk(self, response): if not response.ok: sys.stderr.write('not okay: %r\n' % (response, )) sys.stderr.write(repr(response.text)[:200] + '\n') raise Exception('response not okay', response) def log(self, message): sys.stderr.write(message + '\n') def authenticate(self): if self._session: return self._session self.log('Signing in...') s = self._session = requests.session() r = s.get('https://ident.lds.org/sso/UI/Login') r = s.post('https://ident.lds.org/sso/UI/Login', params={ 'IDToken1': self.username, 'IDToken2': self.password, 'IDButton': 'Log In', }) self.assertOk(r) return self._session def storeRawValue(self, filename, value): self.raw_root.child(filename).setContent(json.dumps(value)) def hasRawValue(self, filename): fp = self.raw_root.child(filename) if fp.exists(): return fp return None def getRawValue(self, filename, default_value=None): fp = self.hasRawValue(filename) if fp: return json.loads(fp.getContent()) else: return default_value def updateRawData(self): for name, func in self.vals.raws.items(): if self.hasRawValue(name): # already has a value; do not recompute self.log('[%s] data already present' % (name, )) continue self.log('[%s] fetching...' % (name, )) data = func(self) self.storeRawValue(name, data) @vals.raw('unit_number') def get_unitNumber(self): s = self.authenticate() r = s.get('https://www.lds.org/mls/mbr/records/member-list?lang=eng') self.assertOk(r) # this is probably pretty fragile... re_unit_number = re.compile(r"window.unitNumber\s=\s'(.*?)';") m = re_unit_number.search(r.text) return m.groups()[0] @vals.raw('member_list') def get_memberList(self): s = self.authenticate() unit_number = self.getRawValue('unit_number') r = s.get('https://www.lds.org/mls/mbr/services/report/member-list', params={ 'lang': 'eng', 'unitNumber': unit_number, }) self.assertOk(r) return r.json() @vals.raw('members_with_callings') def get_membersWithCallings(self): s = self.authenticate() unit_number = self.getRawValue('unit_number') r = s.get( 'https://www.lds.org/mls/mbr/services/report/members-with-callings', params={ 'lang': 'eng', 'unitNumber': unit_number, }, headers={ 'Accept': 'application/json', }) self.assertOk(r) return r.json() @vals.raw('members_without_callings') def get_membersWithoutCallings(self): s = self.authenticate() unit_number = self.getRawValue('unit_number') r = s.get( 'https://www.lds.org/mls/mbr/services/orgs/members-without-callings', params={ 'lang': 'eng', 'unitNumber': unit_number, }, headers={ 'Accept': 'application/json', }) self.assertOk(r) return r.json() #---------------------------- # photos #---------------------------- def _memberPhotoFilePath(self, member_id, size='large', ext='jpg'): """ Valid size options are: - large - medium - original - thumbnail """ return self.photo_root.child('solo-%s-%s.%s' % (member_id, size, ext)) def _memberIDsWithNoPhoto(self, size='large'): members = self.getRawValue('member_list') for member in members: member_id = member['id'] photo_fp = self._memberPhotoFilePath(member_id, size) if photo_fp.exists(): continue yield member_id def updatePhotos(self, size='large'): s = self.authenticate() self.log('Getting photos...') for member_ids in xAtATime(self._memberIDsWithNoPhoto(size), 19): if not member_ids: continue try: r = s.get( 'https://www.lds.org/directory/services/ludrs/photo/url/' + ','.join(map(str, member_ids)) + '/individual') data = r.json() except ValueError: print 'Error on', member_ids raise for member_id, result in zip(member_ids, data): fp = self._memberPhotoFilePath(member_id, size) uri = result[size + 'Uri'] if uri: print 'fetching photo for', member_id uri = 'https://www.lds.org' + uri image_data = s.get(uri) content_type = image_data.headers['content-type'] if content_type != 'image/jpeg': print 'NON-JPEG: ', content_type, member_id continue fp.setContent(image_data.content) else: print 'no photo for', member_id time.sleep(0.5)
def tagfiles(albumdir, album, options): '''Rename and tag files using freedb information for the specified album.''' # XXX: doesn't really belong here missing = album.validate() if len(missing) > 0: if options.strict: amiss = ",".join(missing) raise TagIncompleteWarning(amiss) else: for miss in missing: print TagIncompleteWarning(miss) print album.ignoreMissing(True) localalbum = LocalAlbumInfo(albumdir) filelist = localalbum.getfilelist(options.sysencoding) a = len(album.tracks) b = len(filelist) if a != b: raise NamingMuseError('there are %d files, but %d metainfo tracks' % (b,a)) namebinder = get_namebinder(options, filelist) tracks = namebinder(filelist, album, options.sysencoding) if not sortedcmp(tracks, album.tracks): options.dryrun = True print NamingMuseError("binding was not exact, forcing dry run") print "Tagging album: %s, %s - %s, %s.\n" % \ (album.year, album.artist, album.title, album.genre) # Process files renamealbum = True renameTempDir = FilePath(albumdir, 'rename-tmp') if renameTempDir.exists(): raise NamingMuseError('%s exists!' % renameTempDir) renameTempDir.mkdir() # a list of tuples (from, to) # we use temporary renames to avoid filename collision on filename swaps # this holds the list of final renames to be executed finalrenames = [] # a list of tuples (from, to) # used to rollback renames in case of error rollback = [] renamesign = "->" rollbacksign = '<-' if options.tagonly: renamesign = "-tag->" if options.dryrun: renamesign = "-dry->" try: for i in range(0, len(filelist)): fpath = filelist[i] # XXX: move bug check to freedbalbuminfo parser #if album.isVarious: # if not "/" in title and "-" in title: # # workaround: this is a bug in the freedb entry # # (according to submission guidelines) # trackartist, title = title.split("-") # print NamingMuseWarning("bugged database entry with - instead of /") # else: # trackartist, title = title.split("/") # trackartist, title = trackartist.strip(), title.strip() #else: # trackartist = albumartist track = tracks[i] tofile = policy.genfilename(filelist[i], album, track) tofile = tofile.encode(options.sysencoding) totmpfile = FilePath(renameTempDir, tofile, encoding=options.sysencoding) tofinalfile = FilePath(albumdir, tofile, encoding=options.sysencoding) # Tag and rename file print fpath.getName() print "\t", colorize(renamesign), tofinalfile.getName() if not options.dryrun: # Tag file #preserve stat fd = tempfile.NamedTemporaryFile() tmpfilename = fd.name shutil.copystat(str(fpath), tmpfilename) # tag the file tagfile(fpath, album, track, options) # restore filestat shutil.copystat(tmpfilename, str(fpath)) # deletes tempfile fd.close() # Rename file to temporary name if not options.tagonly: if totmpfile.exists(): raise NamingMuseError('tried to rename file over existing: %s' % str(totmpfile)) if fpath != totmpfile: fpath.rename(totmpfile) rollback.append((fpath, totmpfile)) finalrenames.append((totmpfile, tofinalfile)) except Exception, e: print print colorize('Error: an error occurred. rolling back %d renames.' % len(rollback)) for frompath, topath in reversed(rollback): print frompath.getName() print "\t", colorize(rollbacksign), topath.getName() topath.rename(frompath) renameTempDir.rmdir() raise
newalbumdir = FilePath(albumdir.getParent(), newalbum, \ encoding=options.sysencoding) # Make parent directory of albumdir if needed parent = newalbumdir.getParent() if not parent.isdir(): parent.mkdir() # Rename album (if no "manual" mp3 files in that dir) rollbacksign = '<-' renamesign = "->" if options.dryrun or options.tagonly: renamesign = "-dry->" if not (options.dryrun or options.tagonly) and renamealbum \ and str(albumdir) != str(newalbumdir): if newalbumdir.exists(): raise NamingMuseWarning("Directory already exists (dup album?): " + str(newalbumdir)) try: albumdir.rename(newalbumdir) except OSError, err: raise NamingMuseWarning(str(err)) # Print rename message print "\n", albumdir.getName() print "\t", colorize(renamesign), if needartistdirmove: print FilePath(album.artist.encode(options.sysencoding), newalbumdir.getName()) else: print newalbumdir.getName()
def tagfiles(albumdir, album, options): '''Rename and tag files using freedb information for the specified album.''' # XXX: doesn't really belong here missing = album.validate() if len(missing) > 0: if options.strict: amiss = ",".join(missing) raise TagIncompleteWarning(amiss) else: for miss in missing: print TagIncompleteWarning(miss) print album.ignoreMissing(True) localalbum = LocalAlbumInfo(albumdir) filelist = localalbum.getfilelist(options.sysencoding) a = len(album.tracks) b = len(filelist) if a != b: raise NamingMuseError('there are %d files, but %d metainfo tracks' % (b, a)) namebinder = get_namebinder(options, filelist) tracks = namebinder(filelist, album, options.sysencoding) if not sortedcmp(tracks, album.tracks): options.dryrun = True print NamingMuseError("binding was not exact, forcing dry run") print "Tagging album: %s, %s - %s, %s.\n" % \ (album.year, album.artist, album.title, album.genre) # Process files renamealbum = True renameTempDir = FilePath(albumdir, 'rename-tmp') if renameTempDir.exists(): raise NamingMuseError('%s exists!' % renameTempDir) renameTempDir.mkdir() # a list of tuples (from, to) # we use temporary renames to avoid filename collision on filename swaps # this holds the list of final renames to be executed finalrenames = [] # a list of tuples (from, to) # used to rollback renames in case of error rollback = [] renamesign = "->" rollbacksign = '<-' if options.tagonly: renamesign = "-tag->" if options.dryrun: renamesign = "-dry->" try: for i in range(0, len(filelist)): fpath = filelist[i] # XXX: move bug check to freedbalbuminfo parser #if album.isVarious: # if not "/" in title and "-" in title: # # workaround: this is a bug in the freedb entry # # (according to submission guidelines) # trackartist, title = title.split("-") # print NamingMuseWarning("bugged database entry with - instead of /") # else: # trackartist, title = title.split("/") # trackartist, title = trackartist.strip(), title.strip() #else: # trackartist = albumartist track = tracks[i] tofile = policy.genfilename(filelist[i], album, track) tofile = tofile.encode(options.sysencoding) totmpfile = FilePath(renameTempDir, tofile, encoding=options.sysencoding) tofinalfile = FilePath(albumdir, tofile, encoding=options.sysencoding) # Tag and rename file print fpath.getName() print "\t", colorize(renamesign), tofinalfile.getName() if not options.dryrun: # Tag file #preserve stat fd = tempfile.NamedTemporaryFile() tmpfilename = fd.name shutil.copystat(str(fpath), tmpfilename) # tag the file tagfile(fpath, album, track, options) # restore filestat shutil.copystat(tmpfilename, str(fpath)) # deletes tempfile fd.close() # Rename file to temporary name if not options.tagonly: if totmpfile.exists(): raise NamingMuseError( 'tried to rename file over existing: %s' % str(totmpfile)) if fpath != totmpfile: fpath.rename(totmpfile) rollback.append((fpath, totmpfile)) finalrenames.append((totmpfile, tofinalfile)) except Exception, e: print print colorize('Error: an error occurred. rolling back %d renames.' % len(rollback)) for frompath, topath in reversed(rollback): print frompath.getName() print "\t", colorize(rollbacksign), topath.getName() topath.rename(frompath) renameTempDir.rmdir() raise
class LDSClient(object): vals = ValueGetters() def __init__(self, root, username, password): self._session = None self.username = username self.password = password self.root = FilePath(root) self.raw_root = self.root.child('raw') if not self.root.exists(): self.root.makedirs() if not self.raw_root.exists(): self.raw_root.makedirs() self.photo_root = self.root.child('photos') if not self.photo_root.exists(): self.photo_root.makedirs() def assertOk(self, response): if not response.ok: sys.stderr.write('not okay: %r\n' % (response,)) sys.stderr.write(repr(response.text)[:200] + '\n') raise Exception('response not okay', response) def log(self, message): sys.stderr.write(message + '\n') def authenticate(self): if self._session: return self._session self.log('Signing in...') s = self._session = requests.session() r = s.get('https://ident.lds.org/sso/UI/Login') r = s.post('https://ident.lds.org/sso/UI/Login', params={ 'IDToken1': self.username, 'IDToken2': self.password, 'IDButton': 'Log In', }) self.assertOk(r) return self._session def storeRawValue(self, filename, value): self.raw_root.child(filename).setContent(json.dumps(value)) def hasRawValue(self, filename): fp = self.raw_root.child(filename) if fp.exists(): return fp return None def getRawValue(self, filename, default_value=None): fp = self.hasRawValue(filename) if fp: return json.loads(fp.getContent()) else: return default_value def updateRawData(self): for name, func in self.vals.raws.items(): if self.hasRawValue(name): # already has a value; do not recompute self.log('[%s] data already present' % (name,)) continue self.log('[%s] fetching...' % (name,)) data = func(self) self.storeRawValue(name, data) @vals.raw('unit_number') def get_unitNumber(self): s = self.authenticate() r = s.get('https://www.lds.org/mls/mbr/records/member-list?lang=eng') self.assertOk(r) # this is probably pretty fragile... re_unit_number = re.compile(r"window.unitNumber\s=\s'(.*?)';") m = re_unit_number.search(r.text) return m.groups()[0] @vals.raw('member_list') def get_memberList(self): s = self.authenticate() unit_number = self.getRawValue('unit_number') r = s.get('https://www.lds.org/mls/mbr/services/report/member-list', params={ 'lang': 'eng', 'unitNumber': unit_number, }) self.assertOk(r) return r.json() @vals.raw('members_with_callings') def get_membersWithCallings(self): s = self.authenticate() unit_number = self.getRawValue('unit_number') r = s.get('https://www.lds.org/mls/mbr/services/report/members-with-callings', params={ 'lang': 'eng', 'unitNumber': unit_number, }, headers={ 'Accept': 'application/json', }) self.assertOk(r) return r.json() @vals.raw('members_without_callings') def get_membersWithoutCallings(self): s = self.authenticate() unit_number = self.getRawValue('unit_number') r = s.get('https://www.lds.org/mls/mbr/services/orgs/members-without-callings', params={ 'lang': 'eng', 'unitNumber': unit_number, }, headers={ 'Accept': 'application/json', }) self.assertOk(r) return r.json() #---------------------------- # photos #---------------------------- def _memberPhotoFilePath(self, member_id, size='large', ext='jpg'): """ Valid size options are: - large - medium - original - thumbnail """ return self.photo_root.child('solo-%s-%s.%s' % (member_id, size, ext)) def _memberIDsWithNoPhoto(self, size='large'): members = self.getRawValue('member_list') for member in members: member_id = member['id'] photo_fp = self._memberPhotoFilePath(member_id, size) if photo_fp.exists(): continue yield member_id def updatePhotos(self, size='large'): s = self.authenticate() self.log('Getting photos...') for member_ids in xAtATime(self._memberIDsWithNoPhoto(size), 19): if not member_ids: continue try: r = s.get('https://www.lds.org/directory/services/ludrs/photo/url/'+','.join(map(str, member_ids))+'/individual') data = r.json() except ValueError: print 'Error on', member_ids raise for member_id, result in zip(member_ids, data): fp = self._memberPhotoFilePath(member_id, size) uri = result[size + 'Uri'] if uri: print 'fetching photo for', member_id uri = 'https://www.lds.org' + uri image_data = s.get(uri) content_type = image_data.headers['content-type'] if content_type != 'image/jpeg': print 'NON-JPEG: ', content_type, member_id continue fp.setContent(image_data.content) else: print 'no photo for', member_id time.sleep(0.5)