def eventually_rename_child(child, dir_path, rename_non_utf8=True): print ("eventually_rename_child") # # To manage non-utf8 filenames # the easiest thing is to rename # paths in utf. # # This may cause issues for collections # stored on windows or vfat filesystem. # # This is the KISS-siest approach # that avoids continuously encode # and decode of the filenames. # if not isinstance(child, unicode): if not rename_non_utf8: log.warn( "skipping non unicode path: %s " % to_unicode(child)) raise ValueError("Unsupported non utf-8 encoding") # guess the right encoding # then preserve the encoded string # while changing encoding child_new = to_unicode(child) os.rename( b'%s/%s' % (dir_path.encode('utf-8'), child), b'%s/%s' % ( dir_path.encode('utf-8'), child_new.encode('utf-8')) ) child = child_new return child
def eventually_rename_child(child, dir_path, rename_non_utf8=True): print("eventually_rename_child") # # To manage non-utf8 filenames # the easiest thing is to rename # paths in utf. # # This may cause issues for collections # stored on windows or vfat filesystem. # # This is the KISS-siest approach # that avoids continuously encode # and decode of the filenames. # if not isinstance(child, unicode): if not rename_non_utf8: log.warn("skipping non unicode path: %s " % to_unicode(child)) raise ValueError("Unsupported non utf-8 encoding") # guess the right encoding # then preserve the encoded string # while changing encoding child_new = to_unicode(child) os.rename( b"%s/%s" % (dir_path.encode("utf-8"), child), b"%s/%s" % (dir_path.encode("utf-8"), child_new.encode("utf-8")), ) child = child_new return child
def walk_music_folder(iposonic): log.info("Start walker thread") def add_or_log(path, album=False): try: iposonic.add_path(path, album) except Exception as e: iposonic.log.error(e) for music_folder in iposonic.get_music_folders(): log.info("Walking into: %s" % music_folder) # Assume artist names in utf-8 artists_local = [x for x in os.listdir(music_folder) if os.path.isdir(join("/", music_folder, x))] log.info("Local artists: %s" % artists_local) # index all artists for a in artists_local: try: iposonic.log.info("scanning artist: %s" % a) except: iposonic.log.info("cannot read object: %s" % a.__class__) if a: a = eventually_rename_child(a, music_folder) path = join("/", music_folder, a) add_or_log(path) # # Scan recurrently only if not refresh_always # try: for dirpath, dirnames, filenames in os.walk(path): for d in dirnames: try: d = eventually_rename_child(d, dirpath) d = join("/", path, dirpath, d) add_or_log(d, album=True) except: iposonic.log.info("error: %s" % stringutils.to_unicode(d)) for f in filenames: try: p = join("/", path.encode("utf-8"), dirpath.encode("utf-8"), f.encode("utf-8")).decode( "utf-8" ) iposonic.log.info("p: %s" % stringutils.to_unicode(p)) add_or_log(p) except: iposonic.log.info("error: %s" % stringutils.to_unicode(f)) except: iposonic.log.warn("error traversing: %s" % path) # do something when the app signals something while True: item = q.get() log.exception("Do something with that messages: %s" % item) q.task_done()
def walk_music_folder(iposonic): log.info("Start walker thread") def add_or_log(path, album=False): try: iposonic.add_path(path, album) except Exception as e: iposonic.log.error(e) for music_folder in iposonic.get_music_folders(): log.info("Walking into: %s" % music_folder) # Assume artist names in utf-8 artists_local = [x for x in os.listdir( music_folder) if os.path.isdir(join("/", music_folder, x))] log.info("Local artists: %s" % artists_local) #index all artists for a in artists_local: try: iposonic.log.info(u"scanning artist: %s" % a) except: iposonic.log.info(u'cannot read object: %s' % a.__class__) if a: a = eventually_rename_child(a, music_folder) path = join("/", music_folder, a) add_or_log(path) # # Scan recurrently only if not refresh_always # try: for dirpath, dirnames, filenames in os.walk(path): for d in dirnames: try: d = eventually_rename_child(d, dirpath) d = join("/", path, dirpath, d) add_or_log(d, album=True) except: iposonic.log.info("error: %s" % stringutils.to_unicode(d)) for f in filenames: try: p = join("/", path.encode('utf-8'), dirpath.encode('utf-8'), f.encode('utf-8')).decode('utf-8') iposonic.log.info("p: %s" % stringutils.to_unicode(p)) add_or_log(p) except: iposonic.log.info("error: %s" % stringutils.to_unicode(f)) except: iposonic.log.warn("error traversing: %s" % path) # do something when the app signals something while True: item = q.get() log.exception("Do something with that messages: %s" % item) q.task_done()
def test_unicode(self): for f in os.listdir("/opt/music/"): # f is a byte sequence returned by the # filesystem and should be converted # to a unicode object f_u = stringutils.to_unicode(f) print f.__class__, "%s" % f_u
def add_path(self, path, album=False, session=None): self.log.info("add_path: %s, album=%s" % (path, album)) assert session eid = None record = None record_a = None if not isinstance(path, unicode): path_u = to_unicode(path) else: path_u = path if os.path.isdir(path): eid = MediaManager.uuid(path) if album: record = self.Album(path) else: record = self.Artist(path) self.log.info("adding directory: %s, %s " % (eid, path_u)) elif MediaManager.is_allowed_extension(path_u): try: record = self.Media(path) # Create a virtual album using a mock album id # every song with the same virtual album (artist,album) # is tied to it. if record.album != basename(path) and record.artist and record.album: vpath = join("/", record.artist, record.album) record_a = self.Album(vpath) record.albumId = MediaManager.uuid(vpath) eid = record.id self.log.info("adding file: %s, %s " % ( eid, path_u)) except UnsupportedMediaError, e: raise IposonicException(e)
def add_path(self, path, album=False): """Create an entry from path and add it to the DB.""" if os.path.isdir(path): self.log.warn( "Adding %s: %s " % ("album" if album else "artist", stringutils.to_unicode(path))) eid = MediaManager.uuid(path) if album: self.albums[eid] = IposonicDB.Album(path) else: self.artists[eid] = IposonicDB.Artist(path) self.log.info(u"adding directory: %s, %s " % (eid, stringutils.to_unicode(path))) return eid elif MediaManager.is_allowed_extension(path): try: info = MediaManager.get_info(path) info.update({ 'coverArt': MediaManager.cover_art_uuid(info) }) self.songs[info['id']] = info self.log.info("adding file: %s, %s " % (info['id'], path)) return info['id'] except UnsupportedMediaError as e: raise IposonicException(e) raise IposonicException("Path not found or bad extension: %s " % path)
def add_path(self, path, album=False): """Create an entry from path and add it to the DB.""" if os.path.isdir(path): self.log.warn( "Adding %s: %s " % ("album" if album else "artist", stringutils.to_unicode(path))) eid = MediaManager.uuid(path) if album: self.albums[eid] = IposonicDB.Album(path) else: self.artists[eid] = IposonicDB.Artist(path) self.log.info(u"adding directory: %s, %s " % (eid, stringutils.to_unicode(path))) return eid elif MediaManager.is_allowed_extension(path): try: info = MediaManager.get_info(path) info.update({'coverArt': MediaManager.cover_art_uuid(info)}) self.songs[info['id']] = info self.log.info("adding file: %s, %s " % (info['id'], path)) return info['id'] except UnsupportedMediaError as e: raise IposonicException(e) raise IposonicException("Path not found or bad extension: %s " % path)
def get_info(self, path): """TODO use path_u directly.""" eid = MediaManager.uuid(path) path_u = stringutils.to_unicode(path) parent = dirname(path) dirname_u = MediaManager.get_album_name(path_u) return { 'id': eid, 'name': dirname_u, 'isDir': 'true', 'path': path_u, 'title': dirname_u, 'parent': MediaManager.uuid(parent), 'album': dirname_u, 'artist': basename(parent), 'coverArt': eid }
def transact(self, *args, **kwds): session = self.Session() kwds['session'] = session try: ret = fn(self, *args, **kwds) session.commit() return ret except (ProgrammingError, OperationalError) as e: session.rollback() self.log.exception( "Corrupted database: removing and recreating") self.reset() except Exception as e: session.rollback() if len(args): ret = to_unicode(args[0]) else: ret = "" self.log.exception( u"error: string: %s, ex: %s" % (ret.__class__, e)) raise
def connect(self, *args, **kwds): session = self.Session() kwds['session'] = session try: ret = fn(self, *args, **kwds) return ret except (ProgrammingError, OperationalError) as e: self.log.exception( "Corrupted database: removing and recreating", e) self.reset() except orm.exc.NoResultFound as e: # detailed logging for NoResultFound isn't needed. # just propagate the exception raise EntryNotFoundException(e) except Exception as e: if len(args): ret = to_unicode(args[0]) else: ret = "" self.log.exception( u"error: string: %s, ex: %s" % (ret.__class__, e)) raise
def get_music_directory_view(): """Return the content of a directory. params: - id=-493506601 - xml response 1: <directory id="1" name="ABBA"> <child id="11" parent="1" title="Arrival" artist="ABBA" isDir="true" coverArt="22"/> <child id="12" parent="1" title="Super Trouper" artist="ABBA" isDir="true" coverArt="23"/> </directory> xml response 2: <directory id="11" parent="1" name="Arrival"> <child id="111" parent="11" title="Dancing Queen" isDir="false" album="Arrival" artist="ABBA" track="7" year="1978" genre="Pop" coverArt="24" size="8421341" contentType="audio/mpeg" suffix="mp3" duration="146" bitRate="128" path="ABBA/Arrival/Dancing Queen.mp3"/> <child id="112" parent="11" ... # se above contentType="audio/flac" suffix="flac" transcodedContentType="audio/mpeg" transcodedSuffix="mp3" duration="208" bitRate="128" /> </directory> jsonp response """ (u, p, v, c, f, callback, dir_id) = map(request.args.get, ['u', 'p', 'v', 'c', 'f', 'callback', 'id']) if not dir_id: raise SubsonicProtocolException( "Missing required parameter: 'id' in getMusicDirectory.view") (path, dir_path) = app.iposonic.get_directory_path_by_id(dir_id) mf = app.iposonic.db.music_folders[0] dir_path = os.path.join("/", mf, dir_path) log.info("Getting entries in path: %s" % dir_path) children = [] artist = app.iposonic.db.Artist(dir_path) # # if nothing changed before our last visit # or is a virtual path (eg. uniexistent) # don't rescan # try: last_modified = os.stat(dir_path).st_ctime except: last_modified = -1 if last_modified == -1: print("Getting items from valbum.") children = app.iposonic.get_songs(query={'albumId': dir_id}) elif fs_cache.get(dir_id, 0) == last_modified: print("Getting items from cache.") children = app.iposonic.get_songs(query={'parent': dir_id}) children.extend(app.iposonic.get_albums(query={'parent': dir_id})) else: for child in os.listdir(unicode(dir_path)): # TODO find a way to support non-unicode directories and # folders. The easiest way is to simply RENAME THEM! # ................ print("checking string type: ", type(child)) #child = to_unicode(child) if child[0] in ['.', '_']: continue # # To manage non-utf8 filenames # the easiest thing is to rename # paths in utf. # # This may cause issues for collections # stored on windows or vfat filesystem. # # This is the KISS-siest approach # that avoids continuously encode # and decode of the filenames. # if not isinstance(child, unicode): if not app.config.get('rename_non_utf8'): log.warn("skipping non unicode path: %s " % to_unicode(child)) continue child_new = to_unicode(child) os.rename( b'%s/%s' % (dir_path.encode('utf-8'), child), b'%s/%s' % (dir_path.encode('utf-8'), child_new.encode('utf-8'))) child = child_new path = join(dir_path, child) try: child_j = {} is_dir = isdir(path) # This is a Lazy Indexing. It should not be there # unless a cache is set # XXX eid = MediaManager.uuid(path) try: child_j = app.iposonic.get_entry_by_id(eid) except IposonicException: app.iposonic.add_path(path, album=is_dir) child_j = app.iposonic.get_entry_by_id(eid) children.append(child_j) except IposonicException as e: log.info(e) fs_cache.setdefault(dir_id, last_modified) def _track_or_die(x): try: return int(x['track']) except: return 0 # Sort songs by track id, if possible children = sorted(children, key=_track_or_die) return request.formatter({ 'directory': { 'id': dir_id, 'name': artist.get('name'), 'child': children } })
def get_music_directory_view(): """Return the content of a directory. params: - id=-493506601 - xml response 1: <directory id="1" name="ABBA"> <child id="11" parent="1" title="Arrival" artist="ABBA" isDir="true" coverArt="22"/> <child id="12" parent="1" title="Super Trouper" artist="ABBA" isDir="true" coverArt="23"/> </directory> xml response 2: <directory id="11" parent="1" name="Arrival"> <child id="111" parent="11" title="Dancing Queen" isDir="false" album="Arrival" artist="ABBA" track="7" year="1978" genre="Pop" coverArt="24" size="8421341" contentType="audio/mpeg" suffix="mp3" duration="146" bitRate="128" path="ABBA/Arrival/Dancing Queen.mp3"/> <child id="112" parent="11" ... # se above contentType="audio/flac" suffix="flac" transcodedContentType="audio/mpeg" transcodedSuffix="mp3" duration="208" bitRate="128" /> </directory> jsonp response """ (u, p, v, c, f, callback, dir_id) = map( request.args.get, ['u', 'p', 'v', 'c', 'f', 'callback', 'id']) if not dir_id: raise SubsonicProtocolException( "Missing required parameter: 'id' in getMusicDirectory.view") (path, dir_path) = app.iposonic.get_directory_path_by_id(dir_id) mf = app.iposonic.db.music_folders[0] dir_path = os.path.join("/", mf, dir_path) log.info("Getting entries in path: %s" % dir_path) children = [] artist = app.iposonic.db.Artist(dir_path) # # if nothing changed before our last visit # or is a virtual path (eg. uniexistent) # don't rescan # try: last_modified = os.stat(dir_path).st_ctime except: last_modified = -1 if last_modified == -1: print("Getting items from valbum.") children = app.iposonic.get_songs(query={'albumId': dir_id}) elif fs_cache.get(dir_id, 0) == last_modified: print("Getting items from cache.") children = app.iposonic.get_songs(query={'parent': dir_id}) children.extend(app.iposonic.get_albums(query={'parent': dir_id})) else: for child in os.listdir(unicode(dir_path)): # TODO find a way to support non-unicode directories and # folders. The easiest way is to simply RENAME THEM! # ................ print("checking string type: ", type(child)) #child = to_unicode(child) if child[0] in ['.', '_']: continue # # To manage non-utf8 filenames # the easiest thing is to rename # paths in utf. # # This may cause issues for collections # stored on windows or vfat filesystem. # # This is the KISS-siest approach # that avoids continuously encode # and decode of the filenames. # if not isinstance(child, unicode): if not app.config.get('rename_non_utf8'): log.warn( "skipping non unicode path: %s " % to_unicode(child)) continue child_new = to_unicode(child) os.rename( b'%s/%s' % (dir_path.encode('utf-8'), child), b'%s/%s' % ( dir_path.encode('utf-8'), child_new.encode('utf-8')) ) child = child_new path = join(dir_path, child) try: child_j = {} is_dir = isdir(path) # This is a Lazy Indexing. It should not be there # unless a cache is set # XXX eid = MediaManager.uuid(path) try: child_j = app.iposonic.get_entry_by_id(eid) except IposonicException: app.iposonic.add_path(path, album=is_dir) child_j = app.iposonic.get_entry_by_id(eid) children.append(child_j) except IposonicException as e: log.info(e) fs_cache.setdefault(dir_id, last_modified) def _track_or_die(x): try: return int(x['track']) except: return 0 # Sort songs by track id, if possible children = sorted(children, key=_track_or_die) return request.formatter( {'directory': { 'id': dir_id, 'name': artist.get('name'), 'child': children } })
def jsonp2xml(json): """Convert a json structure to xml. The game is trivial. Nesting uses the [] parenthesis. ex. { 'musicFolder': {'id': 1234, 'name': "sss" } } ex. { 'musicFolder': [{'id': 1234, 'name': "sss" }, {'id': 456, 'name': "aaa" }]} ex. { 'musicFolders': {'musicFolder' : [{'id': 1234, 'name': "sss" }, {'id': 456, 'name': "aaa" }] } } ex. { 'index': [{'name': 'A', 'artist': [{'id': '517674445', 'name': 'Antonello Venditti'}] }] } ex. {"subsonic-response": { "musicFolders": {"musicFolder": [{ "id": 0,"name": "Music"}]}, "status": "ok","version": "1.7.0","xmlns": "http://subsonic.org/restapi"}} """ ret = "" content = None for c in [str, int, unicode]: if isinstance(json, c): return str(json) if not isinstance(json, dict): raise Exception("class type: %s" % json) # every tag is a dict. # its value can be a string, a list or a dict for tag in json.keys(): tag_list = json[tag] # if tag_list is a list, then it represent a list of elements # ex. {index: [{ 'a':'1'} , {'a':'2'} ] } # --> <index a="1" /> <index b="2" /> if isinstance(tag_list, list): for t in tag_list: # for every element, get the attributes # and embed them in the tag named attributes = "" content = "" if not isinstance(t, dict): ret += "%s" % t else: for (attr, value) in t.iteritems(): # only serializable values are attributes if value.__class__.__name__ in 'str': attributes = """%s %s="%s" """ % ( attributes, attr, cgi.escape(stringutils.to_unicode(value), quote=None)) elif value.__class__.__name__ in [ 'int', 'unicode', 'bool', 'long' ]: attributes = """%s %s="%s" """ % (attributes, attr, value) # other values are content elif isinstance(value, dict): content += ResponseHelper.jsonp2xml(value) elif isinstance(value, list): content += ResponseHelper.jsonp2xml( {attr: value}) if content: ret += "<%s%s>%s</%s>" % (tag, attributes, content, tag) else: ret += "<%s%s/>" % (tag, attributes) elif isinstance(tag_list, dict): attributes = "" content = "" for (attr, value) in tag_list.iteritems(): # only string values are attributes if not isinstance(value, dict) and not isinstance( value, list): attributes = """%s %s="%s" """ % (attributes, attr, value) else: content += ResponseHelper.jsonp2xml({attr: value}) if content: ret += "<%s%s>%s</%s>" % (tag, attributes, content, tag) else: ret += "<%s%s/>" % (tag, attributes) # Log the source and destination of the response ResponseHelper.log.debug("ret object is %s" % ret.__class__) if dump_response: ResponseHelper.log.debug("\n\njsonp2xml: %s\n--->\n%s \n\n" % (json, ret)) return ret.replace("isDir=\"True\"", "isDir=\"true\"")