Example #1
0
    def complete_release_meta(self, r, it):
        includes = [
            "artists", "labels", "recordings", "release-groups", "media",
            "artist-credits", "discids", "puids", "isrcs", "artist-rels",
            "label-rels", "recording-rels", "release-rels",
            "release-group-rels", "url-rels", "work-rels",
            "recording-level-rels", "work-level-rels"
        ]
        mb_release = musicbrainzngs.get_release_by_id(id=it['mb_release_id'],
                                                      includes=includes)

        mbr = mb_release['release']

        print '***************************************'

        if 'status' in mbr:
            print mbr['status']

        if 'title' in mbr:
            print mbr['title']

        if 'url-relation-list' in mbr:
            # print mbr['url-relation-list']

            for rel in mbr['url-relation-list']:
                print rel
                if rel['type'] == 'discogs':
                    print 'DISCOGS: %s' % rel['target']

                    try:
                        # pass
                        rel = Relation(content_object=r, url=rel['target'])
                        rel.save()
                    except Exception, e:
                        print 'RELATION EXCEPTION'
                        print e

                    try:
                        discogs_image = discogs_image_by_url(rel['target'])
                        img = filer_extra.url_to_file(discogs_image, r.folder)
                        r.main_image = img
                    except:
                        pass
Example #2
0
    def complete_release_meta(self, r, it):
        includes = [
            "artists", "labels", "recordings", "release-groups", "media",
            "artist-credits", "discids", "puids", "isrcs", "artist-rels",
            "label-rels", "recording-rels", "release-rels",
            "release-group-rels", "url-rels", "work-rels",
            "recording-level-rels", "work-level-rels"
        ]
        mb_release = musicbrainzngs.get_release_by_id(
            id=it['mb_release_id'], includes=includes)

        mbr = mb_release['release']

        print '***************************************'

        if 'status' in mbr:
            print mbr['status']

        if 'title' in mbr:
            print mbr['title']

        if 'url-relation-list' in mbr:
            # print mbr['url-relation-list']

            for rel in mbr['url-relation-list']:
                print rel
                if rel['type'] == 'discogs':
                    print 'DISCOGS: %s' % rel['target']

                    try:
                        # pass
                        rel = Relation(content_object=r, url=rel['target'])
                        rel.save()
                    except Exception, e:
                        print 'RELATION EXCEPTION'
                        print e

                    try:
                        discogs_image = discogs_image_by_url(rel['target'])
                        img = filer_extra.url_to_file(discogs_image, r.folder)
                        r.main_image = img
                    except:
                        pass
Example #3
0
    def format_master_releases(self, master_releases):
        
        
        print
        print '***************************************************'
        print 'format_master_releases'
        print
        print master_releases
        print
        print '***************************************************'
        print
        
        
        releases = []
        
        for re in master_releases:
            
            release = re['release']
            recording = re['recording']
            relations = re['relations']
            
            print release
            
            print 'recording:'
            print recording
            
            print 'relations:'
            print relations
            
            r = {}
            
            r['mb_id'] = None
            r['name'] = None
            r['releasedate'] = None
            r['asin'] = None
            r['barcode'] = None
            r['status'] = None
            r['country'] = None

            # mapping
            try:
                r['mb_id'] = release['id']
            except:
                pass
            
            try:
                r['name'] = release['title']
            except:
                pass
            
            try:
                r['releasedate'] = release['date']
            except:
                pass
            
            try:
                r['asin'] = release['asin']
            except:
                pass
            
            try:
                r['barcode'] = release['barcode']
            except:
                pass
            
            try:
                r['status'] = release['status']
            except:
                pass
            
            try:
                r['country'] = release['country']
            except:
                pass
            
            
            # track mapping
            m = {}
            m['mb_id'] = None
            m['name'] = None
            m['duration'] = None
        
            try:
                m['mb_id'] = recording['recording']['id']
            except:
                pass

            try:
                m['name'] = recording['recording']['title']
            except:
                pass
            
            try:
                m['duration'] = recording['recording']['length']
            except:
                pass
            
            
            # try to get media position
            if 'medium-list' in release:
                print
                print 'got medium list'
                print '*************************************************************'
                print release['medium-list']
                print '*************************************************************'
                for el in release['medium-list'][0]['track-list']:
                    print
                    print el
                    print
                print '*************************************************************'
                print '*************************************************************'
            
            
            r['media'] = m
            
            # artist mapping
            a = {}
            a['mb_id'] = None
            a['name'] = None
            
            try:
                artist = recording['recording']['artist-credit'][0]['artist']
                print artist
                
                try:
                    a['mb_id'] = artist['id']
                except:
                    pass
                
                try:
                    a['name'] = artist['name']
                except:
                    pass
            except:
                pass
            
            
            r['artist'] = a
            
            
            # label related mapping
            l = {}
            l['mb_id'] = None
            l['name'] = 'Unknown'
            l['code'] = None
            l['catalognumber'] = None
            
            try:
                label = release['label-info-list'][0]['label']
                print label
                
                try:
                    l['mb_id'] = label['id']
                except:
                    pass
                
                try:
                    l['name'] = label['name']
                except:
                    pass
                
                try:
                    l['code'] = label['label-code']
                except:
                    pass
                
                try:
                    l['catalognumber'] = release['label-info-list'][0]['catalog-number']
                except:
                    pass
                
            except Exception, e:
                print e
                pass
            
            
            
            r['label'] = l
            
            
            # relation mapping
            rel = {}
            rel['discogs_url'] = None
            rel['discogs_image'] = None
            
            try:
                try:
                    for relation in relations:
                        if relation['type'] == 'discogs':
                            rel['discogs_url'] = relation['target']
                            rel['discogs_image'] = discogs_image_by_url(relation['target'], 'uri150')
                            
                    
                except Exception, e:
                    print e
                    pass
                
            except Exception, e:
                print e
                pass
Example #4
0
    def complete_releases(self, releases):
        

        log = logging.getLogger('importer.process.complete_releases')
        log.info('got %s releases to complete' % len(releases))
        
        completed_releases = []
        
        for release in releases:
            if release['id'] in completed_releases:
                log.debug('already completed release with id: %s' % release['id'])
                releases.remove(release)
                
            else:
                log.debug('complete release with id: %s' % release['id'])
                
                r_id = release['id']
                rg_id = release['release-group']['id']
                
                print 'r_id: %s' % r_id
                print 'rg_id: %s' % rg_id
                
                
                release['label'] = None
                release['catalog-number'] = None
                release['discogs_url'] = None
                release['discogs_master_url'] = None
                release['discogs_image'] = None
                
                
                """
                get release details
                """
                inc = ('labels', 'artists', 'url-rels', 'label-rels',)
                url = 'http://%s/ws/2/release/%s/?fmt=json&inc=%s' % (MUSICBRAINZ_HOST, r_id, "+".join(inc))
                
                r = requests.get(url)
                result = r.json()
                #self.pp.pprint(result)

                # only apply label info if unique
                if 'label-info' in result and len(result['label-info']) == 1:
                    if 'label' in result['label-info'][0]:
                        release['label'] = result['label-info'][0]['label']
                    if 'catalog-number' in result['label-info'][0]:
                        release['catalog-number'] = result['label-info'][0]['catalog-number']
                    
                # try to get discogs url
                if 'relations' in result:
                    for relation in result['relations']:
                        if relation['type'] == 'discogs':
                            log.debug('got discogs url from release: %s' % relation['url'])
                            release['discogs_url'] = relation['url']['resource']
                
                
                
                """
                get release-group details
                """
                inc = ('url-rels',)
                url = 'http://%s/ws/2/release-group/%s/?fmt=json&inc=%s' % (MUSICBRAINZ_HOST, rg_id, "+".join(inc))
                
                r = requests.get(url)
                result = r.json()
                #self.pp.pprint(result)
                    
                # try to get discogs master-url
                if 'relations' in result:
                    for relation in result['relations']:
                        if relation['type'] == 'discogs':
                            log.debug('got discogs url from release: %s' % relation['url'])
                            release['discogs_master_url'] = relation['url']['resource']
                        
                        
                        
                        
                """
                assign cover image (if available)
                """
                if release['discogs_url']:
                    try:
                        release['discogs_image'] = discogs_image_by_url(release['discogs_url'], 'uri150')
                    except:
                        pass
                    
                if not release['discogs_image']:
                    try:
                        release['discogs_image'] = discogs_image_by_url(release['discogs_master_url'], 'uri150')
                    except:
                        pass
                """
                finally try to get image from coverartarchive.org
                """
                if not release['discogs_image']:
                    url = 'http://coverartarchive.org/release/%s' % r_id
                    try:    
                        r = requests.get(url)
                        result = r.json()
                        release['discogs_image'] = result['images'][0]['image']
                    except:
                        pass
                    

                
                completed_releases.append(release['id'])
            
        
        
        return releases
Example #5
0
    def mb_complete_artist(self, obj, mb_id):
        
        log = logging.getLogger('util.importer.mb_complete_artist')
        log.info('complete artist, a: %s | mb_id: %s' % (obj.name, mb_id))
        
        self.mb_completed.append(mb_id)
        
        inc = ('url-rels', 'tags')
        url = 'http://%s/ws/2/artist/%s/?fmt=json&inc=%s' % (MUSICBRAINZ_HOST, mb_id, "+".join(inc))
        
        r = requests.get(url)
        result = r.json()
        
        
        
        print '#########################################################################'
        self.pp.pprint(result)
        
        
        discogs_url = None
        discogs_image = None
        
        valid_relations = ('wikipedia', 'allmusic', 'BBC Music page', 'social network', 'official homepage', 'youtube', 'myspace',)
        
        relations = result.get('relations', ())

        for relation in relations:
            
            if relation['type'] == 'discogs':
                log.debug('got discogs url for artist: %s' % relation['url'])
                discogs_url = relation['url']
                
            if relation['type'] in valid_relations:
                log.debug('got %s url for artist: %s' % (relation['type'], relation['url']))
                

                try:
                    rel = Relation.objects.get(object_id=obj.pk, url=relation['url'])
                except:
                    rel = Relation(content_object=obj, url=relation['url'])
                    
                    if relation['type'] == 'official homepage':
                        rel.service = 'official'
                    
                    rel.save()

            
            
            
        if discogs_url:
            
            try:
                rel = Relation.objects.get(object_id=obj.pk, url=discogs_url)
            except:
                rel = Relation(content_object=obj, url=discogs_url)
                rel.save()
                
            # try to get image
            try:
                discogs_image = discogs_image_by_url(discogs_url, 'resource_url')
                log.debug('discogs image located at: %s' % discogs_image)
            except:
                pass
            
            
        # try to load & assign image
        if discogs_image:
            try:
                img = filer_extra.url_to_file(discogs_image, obj.folder)
                obj.main_image = img
                obj.save()
            except:
                log.info('unable to assign discogs image')
                
                
        if discogs_url:

            discogs_id = None
            try:
                # TODO: not sure if always working
                discogs_id = discogs_id_by_url(discogs_url)
                log.info('extracted discogs id: %s' % discogs_id)
            except:
                pass
            
            if discogs_id:
                url = 'http://api.discogs.com/artists/%s' % discogs_id
                r = requests.get(url)
                dgs_result = r.json()
                
                self.pp.pprint(dgs_result)
  
                """                  
                styles = dgs_result.get('styles', ())
                for style in styles:
                    log.debug('got style: %s' % (style))
                    Tag.objects.add_tag(obj, '"%s"' % style)
                """ 
                profile = dgs_result.get('profile', None)
                if profile:
                    obj.biography = profile
                    
                realname = dgs_result.get('realname', None)
                if realname:
                    obj.real_name = realname
                    
                """
                verry hackish part here, just as proof-of-concept
                """
                aliases = dgs_result.get('aliases', ())
                for alias in aliases:
                    try:
                        log.debug('got alias: %s' % alias['name'])
                        # TODO: improve! handle duplicates!
                        time.sleep(1.1)
                        r = requests.get(alias['resource_url'])
                        aa_result = r.json()
                        aa_discogs_url = aa_result.get('uri', None)
                        aa_name = aa_result.get('name', None)
                        aa_profile = aa_result.get('profile', None)
                        if aa_discogs_url and aa_name:
                            
                            l_as = lookup.artist_by_relation_url(aa_discogs_url)
                            l_a = None
                                
                            if len(l_as) < 1:
                                l_a = Artist(name=aa_name, biography=aa_profile)
                                l_a.save()
    
                                rel = Relation(content_object=l_a, url=aa_discogs_url)
                                rel.save()
                                
                            if len(l_as) == 1:
                                l_a = l_as[0]
                                print l_as[0]
                                
                            if l_a:
                                obj.aliases.add(l_a)
                    except:
                        pass
                    
                """
                verry hackish part here, just as proof-of-concept
                """
                members = dgs_result.get('members', ())
                for member in members:
                    try:
                        log.debug('got member: %s' % member['name'])
                        # TODO: improve! handle duplicates!
                        time.sleep(1.1)
                        r = requests.get(member['resource_url'])
                        ma_result = r.json()
                        ma_discogs_url = ma_result.get('uri', None)
                        ma_name = ma_result.get('name', None)
                        ma_profile = ma_result.get('profile', None)
                        if ma_discogs_url and ma_name:
                            
                            l_as = lookup.artist_by_relation_url(ma_discogs_url)
                            l_a = None
                                
                            if len(l_as) < 1:
                                l_a = Artist(name=ma_name, biography=ma_profile)
                                l_a.save()
    
                                rel = Relation(content_object=l_a, url=ma_discogs_url)
                                rel.save()
                                
                            if len(l_as) == 1:
                                l_a = l_as[0]
                                print l_as[0]
                                
                            if l_a:                                
                                ma = ArtistMembership.objects.get_or_create(parent=obj, child=l_a)
                                
                    except:
                        pass
                        
                
            
        type = result.get('type', None)
        if type:
            log.debug('got type: %s' % (type))
            obj.type = type
            
        disambiguation = result.get('disambiguation', None)
        if disambiguation:
            log.debug('got disambiguation: %s' % (disambiguation))
            obj.disambiguation = disambiguation
                    
        tags = result.get('tags', ())

        for tag in tags:
            log.debug('got tag: %s' % (tag['name']))
            Tag.objects.add_tag(obj, '"%s"' % tag['name'])
            
                    
        # add mb relation
        mb_url = 'http://musicbrainz.org/artist/%s' % (mb_id)
        try:
            rel = Relation.objects.get(object_id=obj.pk, url=mb_url)
        except:
            log.debug('relation not here yet, add it: %s' % (mb_url))
            rel = Relation(content_object=obj, url=mb_url)
            rel.save()
        
        obj.save()
        
        
        return obj
Example #6
0
    def mb_complete_release(self, obj, mb_id):
        
        log = logging.getLogger('util.importer.mb_complete_release')
        log.info('complete release, r: %s | mb_id: %s' % (obj.name, mb_id))
        
        inc = ('artists', 'url-rels', 'aliases', 'tags', 'recording-rels', 'work-rels', 'work-level-rels', 'artist-credits', 'labels', 'label-rels', 'release-groups')
        url = 'http://%s/ws/2/release/%s/?fmt=json&inc=%s' % (MUSICBRAINZ_HOST, mb_id, "+".join(inc))
        
        r = requests.get(url)
        result = r.json()
        
        self.pp.pprint(result)
        
        rg_id = None
        release_group = result.get('release-group', None)
        if release_group:
            rg_id = release_group.get('id', None)
            
        log.debug('release-group id: %s' % rg_id)
        
        discogs_url = None
        discogs_master_url = None
        discogs_image = None
        # try to get relations
        if 'relations' in result:
            for relation in result['relations']:
                
                if relation['type'] == 'discogs':
                    log.debug('got discogs url for release: %s' % relation['url'])
                    discogs_url = relation['url']
                    
                    # obj.save()
                    
                if relation['type'] == 'purchase for download':
                    log.debug('got purchase url for release: %s' % relation['url'])

                    try:
                        rel = Relation.objects.get(object_id=obj.pk, url=relation['url'])
                    except:
                        rel = Relation(content_object=obj, url=relation['url'])
                        rel.save()
                        
        

            
        if rg_id:
            # try to get discogs master url
            inc = ('url-rels',)
            url = 'http://%s/ws/2/release-group/%s/?fmt=json&inc=%s' % (MUSICBRAINZ_HOST, rg_id, "+".join(inc))
            
            r = requests.get(url)
            rg_result = r.json()
            
            print "*******************************************************************"
            self.pp.pprint(rg_result)

            # try to get relations from master
            if 'relations' in rg_result:
                for relation in rg_result['relations']:
                    
                    if relation['type'] == 'discogs':
                        log.debug('got discogs master-url for release: %s' % relation['url'])
                        discogs_master_url = relation['url']

                        
                    if relation['type'] == 'wikipedia':
                        log.debug('got wikipedia url for release: %s' % relation['url'])
    
                        try:
                            rel = Relation.objects.get(object_id=obj.pk, url=relation['url'])
                        except:
                            rel = Relation(content_object=obj, url=relation['url'])
                            rel.save()

                        
                    if relation['type'] == 'lyrics':
                        log.debug('got lyrics url for release: %s' % relation['url'])
    
                        try:
                            rel = Relation.objects.get(object_id=obj.pk, url=relation['url'])
                        except:
                            rel = Relation(content_object=obj, url=relation['url'])
                            rel.save()

                        
                    if relation['type'] == 'allmusic':
                        log.debug('got allmusic url for release: %s' % relation['url'])
    
                        try:
                            rel = Relation.objects.get(object_id=obj.pk, url=relation['url'])
                        except:
                            rel = Relation(content_object=obj, url=relation['url'])
                            rel.save()

                        
                    if relation['type'] == 'review':
                        log.debug('got review url for release: %s' % relation['url'])
    
                        try:
                            rel = Relation.objects.get(object_id=obj.pk, url=relation['url'])
                        except:
                            rel = Relation(content_object=obj, url=relation['url'])
                            rel.save()
        
        
        if discogs_url:
            
            try:
                rel = Relation.objects.get(object_id=obj.pk, url=discogs_url)
            except:
                rel = Relation(content_object=obj, url=discogs_url)
                rel.save()
                
            # try to get image
            try:
                discogs_image = discogs_image_by_url(discogs_url, 'resource_url')
                log.debug('discogs image located at: %s' % discogs_image)
            except:
                pass
            
        if discogs_master_url:
            
            try:
                rel = Relation.objects.get(object_id=obj.pk, url=discogs_master_url)
            except:
                rel = Relation(content_object=obj, url=discogs_master_url)
                rel.save()
                
            # try to get image from master
            if not discogs_image:
                try:
                    discogs_image = discogs_image_by_url(discogs_master_url, 'resource_url')
                    log.debug('discogs image located at: %s' % discogs_master_url)
                except:
                    pass
            
            
            
        # try to load & assign image
        if discogs_image:
            try:
                img = filer_extra.url_to_file(discogs_image, obj.folder)
                obj.main_image = img
                obj.save()
            except:
                log.info('unable to assign discogs image')
                
        else:
            # try at coverartarchive...
            url = 'http://coverartarchive.org/release/%s' % mb_id
            try:    
                r = requests.get(url)
                ca_result = r.json()
                ca_url = ca_result['images'][0]['image']
                img = filer_extra.url_to_file(ca_url, obj.folder)
                obj.main_image = img
                obj.save()
            except:
                pass
            
            
        # try to get some additional information from discogs
        if discogs_url:
            discogs_id = None
            try:
                discogs_id = re.findall(r'\d+', discogs_url)[0]
                log.info('extracted discogs id: %s' % discogs_id)
            except:
                pass
            
            if discogs_id:
                url = 'http://api.discogs.com/releases/%s' % discogs_id
                r = requests.get(url)
                dgs_result = r.json()
                    
                styles = dgs_result.get('styles', None)
                for style in styles:
                    log.debug('got style: %s' % (style))
                    Tag.objects.add_tag(obj, '"%s"' % style)
                    
                genres = dgs_result.get('genres', None)
                for genre in genres:
                    log.debug('got genre: %s' % (genre))
                    Tag.objects.add_tag(obj, '"%s"' % genre)
                    
                notes = dgs_result.get('notes', None)
                if notes:
                    obj.description = notes
                    
        if discogs_master_url:
            discogs_id = None
            try:
                discogs_id = re.findall(r'\d+', discogs_master_url)[0]
                log.info('extracted discogs id: %s' % discogs_id)
            except:
                pass
            
            if discogs_id:
                url = 'http://api.discogs.com/masters/%s' % discogs_id
                r = requests.get(url)
                dgs_result = r.json()
                    
                styles = dgs_result.get('styles', None)
                for style in styles:
                    log.debug('got style: %s' % (style))
                    Tag.objects.add_tag(obj, '"%s"' % style)
                    
                genres = dgs_result.get('genres', None)
                for genre in genres:
                    log.debug('got genre: %s' % (genre))
                    Tag.objects.add_tag(obj, '"%s"' % genre)
                    
                notes = dgs_result.get('notes', None)
                if notes:
                    obj.description = notes
                

                    
        tags = result.get('tags', ())
        for tag in tags:
            log.debug('got tag: %s' % (tag['name']))
            Tag.objects.add_tag(obj, '"%s"' % tag['name'])
            
        status = result.get('status', None)
        if status:
            log.debug('got status: %s' % (status))
            obj.releasestatus = status
            
        country = result.get('country', None)
        if country:
            log.debug('got country: %s' % (country))
            obj.release_country = country
            
        date = result.get('date', None)
        if date:
            log.debug('got date: %s' % (date))
            # TODO: rework field
            if len(date) == 4:
                date = '%s-00-00' % (date)
            elif len(date) == 7:
                date = '%s-00' % (date)
            elif len(date) == 10:
                date = '%s' % (date)
                
            re_date = re.compile('^\d{4}-\d{2}-\d{2}$')
            if re_date.match(date):
                obj.releasedate_approx = '%s' % date
            
            
        asin = result.get('asin', None)
        if asin:
            log.debug('got asin: %s' % (asin))
            obj.asin = asin
            
        barcode = result.get('barcode', None)
        if barcode:
            log.debug('got barcode: %s' % (barcode))
            # obj.barcode = barcode
            
                    
        # add mb relation
        mb_url = 'http://musicbrainz.org/release/%s' % (mb_id)
        try:
            rel = Relation.objects.get(object_id=obj.pk, url=mb_url)
        except:
            log.debug('relation not here yet, add it: %s' % (mb_url))
            rel = Relation(content_object=obj, url=mb_url)
            rel.save()

        obj.save()
        
        
        return obj
Example #7
0
    def mb_complete_artist(self, obj, mb_id):

        log = logging.getLogger('util.importer.mb_complete_artist')
        log.info('complete artist, a: %s | mb_id: %s' % (obj.name, mb_id))

        self.mb_completed.append(mb_id)

        inc = ('url-rels', 'tags')
        url = 'http://%s/ws/2/artist/%s/?fmt=json&inc=%s' % (
            MUSICBRAINZ_HOST, mb_id, "+".join(inc))

        r = requests.get(url)
        result = r.json()

        print '#########################################################################'
        self.pp.pprint(result)

        discogs_url = None
        discogs_image = None

        valid_relations = (
            'wikipedia',
            'allmusic',
            'BBC Music page',
            'social network',
            'official homepage',
            'youtube',
            'myspace',
        )

        relations = result.get('relations', ())

        for relation in relations:

            if relation['type'] == 'discogs':
                log.debug('got discogs url for artist: %s' % relation['url'])
                discogs_url = relation['url']

            if relation['type'] in valid_relations:
                log.debug('got %s url for artist: %s' %
                          (relation['type'], relation['url']))

                try:
                    rel = Relation.objects.get(object_id=obj.pk,
                                               url=relation['url'])
                except:
                    rel = Relation(content_object=obj, url=relation['url'])

                    if relation['type'] == 'official homepage':
                        rel.service = 'official'

                    rel.save()

        if discogs_url:

            try:
                rel = Relation.objects.get(object_id=obj.pk, url=discogs_url)
            except:
                rel = Relation(content_object=obj, url=discogs_url)
                rel.save()

            # try to get image
            try:
                discogs_image = discogs_image_by_url(discogs_url,
                                                     'resource_url')
                log.debug('discogs image located at: %s' % discogs_image)
            except:
                pass

        # try to load & assign image
        if discogs_image:
            try:
                img = filer_extra.url_to_file(discogs_image, obj.folder)
                obj.main_image = img
                obj.save()
            except:
                log.info('unable to assign discogs image')

        if discogs_url:

            discogs_id = None
            try:
                # TODO: not sure if always working
                discogs_id = discogs_id_by_url(discogs_url)
                log.info('extracted discogs id: %s' % discogs_id)
            except:
                pass

            if discogs_id:
                url = 'http://api.discogs.com/artists/%s' % discogs_id
                r = requests.get(url)
                dgs_result = r.json()

                self.pp.pprint(dgs_result)
                """                  
                styles = dgs_result.get('styles', ())
                for style in styles:
                    log.debug('got style: %s' % (style))
                    Tag.objects.add_tag(obj, '"%s"' % style)
                """
                profile = dgs_result.get('profile', None)
                if profile:
                    obj.biography = profile

                realname = dgs_result.get('realname', None)
                if realname:
                    obj.real_name = realname
                """
                verry hackish part here, just as proof-of-concept
                """
                aliases = dgs_result.get('aliases', ())
                for alias in aliases:
                    try:
                        log.debug('got alias: %s' % alias['name'])
                        # TODO: improve! handle duplicates!
                        time.sleep(1.1)
                        r = requests.get(alias['resource_url'])
                        aa_result = r.json()
                        aa_discogs_url = aa_result.get('uri', None)
                        aa_name = aa_result.get('name', None)
                        aa_profile = aa_result.get('profile', None)
                        if aa_discogs_url and aa_name:

                            l_as = lookup.artist_by_relation_url(
                                aa_discogs_url)
                            l_a = None

                            if len(l_as) < 1:
                                l_a = Artist(name=aa_name,
                                             biography=aa_profile)
                                l_a.save()

                                rel = Relation(content_object=l_a,
                                               url=aa_discogs_url)
                                rel.save()

                            if len(l_as) == 1:
                                l_a = l_as[0]
                                print l_as[0]

                            if l_a:
                                obj.aliases.add(l_a)
                    except:
                        pass
                """
                verry hackish part here, just as proof-of-concept
                """
                members = dgs_result.get('members', ())
                for member in members:
                    try:
                        log.debug('got member: %s' % member['name'])
                        # TODO: improve! handle duplicates!
                        time.sleep(1.1)
                        r = requests.get(member['resource_url'])
                        ma_result = r.json()
                        ma_discogs_url = ma_result.get('uri', None)
                        ma_name = ma_result.get('name', None)
                        ma_profile = ma_result.get('profile', None)
                        if ma_discogs_url and ma_name:

                            l_as = lookup.artist_by_relation_url(
                                ma_discogs_url)
                            l_a = None

                            if len(l_as) < 1:
                                l_a = Artist(name=ma_name,
                                             biography=ma_profile)
                                l_a.save()

                                rel = Relation(content_object=l_a,
                                               url=ma_discogs_url)
                                rel.save()

                            if len(l_as) == 1:
                                l_a = l_as[0]
                                print l_as[0]

                            if l_a:
                                ma = ArtistMembership.objects.get_or_create(
                                    parent=obj, child=l_a)

                    except:
                        pass

        type = result.get('type', None)
        if type:
            log.debug('got type: %s' % (type))
            obj.type = type

        disambiguation = result.get('disambiguation', None)
        if disambiguation:
            log.debug('got disambiguation: %s' % (disambiguation))
            obj.disambiguation = disambiguation

        tags = result.get('tags', ())

        for tag in tags:
            log.debug('got tag: %s' % (tag['name']))
            Tag.objects.add_tag(obj, '"%s"' % tag['name'])

        # add mb relation
        mb_url = 'http://musicbrainz.org/artist/%s' % (mb_id)
        try:
            rel = Relation.objects.get(object_id=obj.pk, url=mb_url)
        except:
            log.debug('relation not here yet, add it: %s' % (mb_url))
            rel = Relation(content_object=obj, url=mb_url)
            rel.save()

        obj.save()

        return obj
Example #8
0
    def mb_complete_release(self, obj, mb_id):

        log = logging.getLogger('util.importer.mb_complete_release')
        log.info('complete release, r: %s | mb_id: %s' % (obj.name, mb_id))

        inc = ('artists', 'url-rels', 'aliases', 'tags', 'recording-rels',
               'work-rels', 'work-level-rels', 'artist-credits', 'labels',
               'label-rels', 'release-groups')
        url = 'http://%s/ws/2/release/%s/?fmt=json&inc=%s' % (
            MUSICBRAINZ_HOST, mb_id, "+".join(inc))

        r = requests.get(url)
        result = r.json()

        self.pp.pprint(result)

        rg_id = None
        release_group = result.get('release-group', None)
        if release_group:
            rg_id = release_group.get('id', None)

        log.debug('release-group id: %s' % rg_id)

        discogs_url = None
        discogs_master_url = None
        discogs_image = None
        # try to get relations
        if 'relations' in result:
            for relation in result['relations']:

                if relation['type'] == 'discogs':
                    log.debug('got discogs url for release: %s' %
                              relation['url'])
                    discogs_url = relation['url']

                    # obj.save()

                if relation['type'] == 'purchase for download':
                    log.debug('got purchase url for release: %s' %
                              relation['url'])

                    try:
                        rel = Relation.objects.get(object_id=obj.pk,
                                                   url=relation['url'])
                    except:
                        rel = Relation(content_object=obj, url=relation['url'])
                        rel.save()

        if rg_id:
            # try to get discogs master url
            inc = ('url-rels', )
            url = 'http://%s/ws/2/release-group/%s/?fmt=json&inc=%s' % (
                MUSICBRAINZ_HOST, rg_id, "+".join(inc))

            r = requests.get(url)
            rg_result = r.json()

            print "*******************************************************************"
            self.pp.pprint(rg_result)

            # try to get relations from master
            if 'relations' in rg_result:
                for relation in rg_result['relations']:

                    if relation['type'] == 'discogs':
                        log.debug('got discogs master-url for release: %s' %
                                  relation['url'])
                        discogs_master_url = relation['url']

                    if relation['type'] == 'wikipedia':
                        log.debug('got wikipedia url for release: %s' %
                                  relation['url'])

                        try:
                            rel = Relation.objects.get(object_id=obj.pk,
                                                       url=relation['url'])
                        except:
                            rel = Relation(content_object=obj,
                                           url=relation['url'])
                            rel.save()

                    if relation['type'] == 'lyrics':
                        log.debug('got lyrics url for release: %s' %
                                  relation['url'])

                        try:
                            rel = Relation.objects.get(object_id=obj.pk,
                                                       url=relation['url'])
                        except:
                            rel = Relation(content_object=obj,
                                           url=relation['url'])
                            rel.save()

                    if relation['type'] == 'allmusic':
                        log.debug('got allmusic url for release: %s' %
                                  relation['url'])

                        try:
                            rel = Relation.objects.get(object_id=obj.pk,
                                                       url=relation['url'])
                        except:
                            rel = Relation(content_object=obj,
                                           url=relation['url'])
                            rel.save()

                    if relation['type'] == 'review':
                        log.debug('got review url for release: %s' %
                                  relation['url'])

                        try:
                            rel = Relation.objects.get(object_id=obj.pk,
                                                       url=relation['url'])
                        except:
                            rel = Relation(content_object=obj,
                                           url=relation['url'])
                            rel.save()

        if discogs_url:

            try:
                rel = Relation.objects.get(object_id=obj.pk, url=discogs_url)
            except:
                rel = Relation(content_object=obj, url=discogs_url)
                rel.save()

            # try to get image
            try:
                discogs_image = discogs_image_by_url(discogs_url,
                                                     'resource_url')
                log.debug('discogs image located at: %s' % discogs_image)
            except:
                pass

        if discogs_master_url:

            try:
                rel = Relation.objects.get(object_id=obj.pk,
                                           url=discogs_master_url)
            except:
                rel = Relation(content_object=obj, url=discogs_master_url)
                rel.save()

            # try to get image from master
            if not discogs_image:
                try:
                    discogs_image = discogs_image_by_url(
                        discogs_master_url, 'resource_url')
                    log.debug('discogs image located at: %s' %
                              discogs_master_url)
                except:
                    pass

        # try to load & assign image
        if discogs_image:
            try:
                img = filer_extra.url_to_file(discogs_image, obj.folder)
                obj.main_image = img
                obj.save()
            except:
                log.info('unable to assign discogs image')

        else:
            # try at coverartarchive...
            url = 'http://coverartarchive.org/release/%s' % mb_id
            try:
                r = requests.get(url)
                ca_result = r.json()
                ca_url = ca_result['images'][0]['image']
                img = filer_extra.url_to_file(ca_url, obj.folder)
                obj.main_image = img
                obj.save()
            except:
                pass

        # try to get some additional information from discogs
        if discogs_url:
            discogs_id = None
            try:
                discogs_id = re.findall(r'\d+', discogs_url)[0]
                log.info('extracted discogs id: %s' % discogs_id)
            except:
                pass

            if discogs_id:
                url = 'http://api.discogs.com/releases/%s' % discogs_id
                r = requests.get(url)
                dgs_result = r.json()

                styles = dgs_result.get('styles', None)
                for style in styles:
                    log.debug('got style: %s' % (style))
                    Tag.objects.add_tag(obj, '"%s"' % style)

                genres = dgs_result.get('genres', None)
                for genre in genres:
                    log.debug('got genre: %s' % (genre))
                    Tag.objects.add_tag(obj, '"%s"' % genre)

                notes = dgs_result.get('notes', None)
                if notes:
                    obj.description = notes

        if discogs_master_url:
            discogs_id = None
            try:
                discogs_id = re.findall(r'\d+', discogs_master_url)[0]
                log.info('extracted discogs id: %s' % discogs_id)
            except:
                pass

            if discogs_id:
                url = 'http://api.discogs.com/masters/%s' % discogs_id
                r = requests.get(url)
                dgs_result = r.json()

                styles = dgs_result.get('styles', None)
                for style in styles:
                    log.debug('got style: %s' % (style))
                    Tag.objects.add_tag(obj, '"%s"' % style)

                genres = dgs_result.get('genres', None)
                for genre in genres:
                    log.debug('got genre: %s' % (genre))
                    Tag.objects.add_tag(obj, '"%s"' % genre)

                notes = dgs_result.get('notes', None)
                if notes:
                    obj.description = notes

        tags = result.get('tags', ())
        for tag in tags:
            log.debug('got tag: %s' % (tag['name']))
            Tag.objects.add_tag(obj, '"%s"' % tag['name'])

        status = result.get('status', None)
        if status:
            log.debug('got status: %s' % (status))
            obj.releasestatus = status

        country = result.get('country', None)
        if country:
            log.debug('got country: %s' % (country))
            obj.release_country = country

        date = result.get('date', None)
        if date:
            log.debug('got date: %s' % (date))
            # TODO: rework field
            if len(date) == 4:
                date = '%s-00-00' % (date)
            elif len(date) == 7:
                date = '%s-00' % (date)
            elif len(date) == 10:
                date = '%s' % (date)

            re_date = re.compile('^\d{4}-\d{2}-\d{2}$')
            if re_date.match(date) and date != '0000-00-00':
                obj.releasedate_approx = '%s' % date

        asin = result.get('asin', None)
        if asin:
            log.debug('got asin: %s' % (asin))
            obj.asin = asin

        barcode = result.get('barcode', None)
        if barcode:
            log.debug('got barcode: %s' % (barcode))
            # obj.barcode = barcode

        # add mb relation
        mb_url = 'http://musicbrainz.org/release/%s' % (mb_id)
        try:
            rel = Relation.objects.get(object_id=obj.pk, url=mb_url)
        except:
            log.debug('relation not here yet, add it: %s' % (mb_url))
            rel = Relation(content_object=obj, url=mb_url)
            rel.save()

        obj.save()

        return obj
Example #9
0
    def format_master_releases(self, master_releases):
        
        
        print
        print '***************************************************'
        print 'format_master_releases'
        print
        print master_releases
        print
        print '***************************************************'
        print
        
        
        releases = []
        
        for re in master_releases:
            
            release = re['release']
            recording = re['recording']
            relations = re['relations']
            
            print release
            
            print 'recording:'
            print recording
            
            print 'relations:'
            print relations
            
            r = {}
            
            r['mb_id'] = None
            r['name'] = None
            r['releasedate'] = None
            r['asin'] = None
            r['barcode'] = None
            r['status'] = None
            r['country'] = None

            # mapping
            try:
                r['mb_id'] = release['id']
            except:
                pass
            
            try:
                r['name'] = release['title']
            except:
                pass
            
            try:
                r['releasedate'] = release['date']
            except:
                pass
            
            try:
                r['asin'] = release['asin']
            except:
                pass
            
            try:
                r['barcode'] = release['barcode']
            except:
                pass
            
            try:
                r['status'] = release['status']
            except:
                pass
            
            try:
                r['country'] = release['country']
            except:
                pass
            
            
            # track mapping
            m = {}
            m['mb_id'] = None
            m['name'] = None
            m['duration'] = None
        
            try:
                m['mb_id'] = recording['recording']['id']
            except:
                pass

            try:
                m['name'] = recording['recording']['title']
            except:
                pass
            
            try:
                m['duration'] = recording['recording']['length']
            except:
                pass
            
            
            # try to get media position
            if 'medium-list' in release:
                print
                print 'got medium list'
                print '*************************************************************'
                print release['medium-list']
                print '*************************************************************'
                for el in release['medium-list'][0]['track-list']:
                    print
                    print el
                    print
                print '*************************************************************'
                print '*************************************************************'
            
            
            r['media'] = m
            
            # artist mapping
            a = {}
            a['mb_id'] = None
            a['name'] = None
            
            try:
                artist = recording['recording']['artist-credit'][0]['artist']
                print artist
                
                try:
                    a['mb_id'] = artist['id']
                except:
                    pass
                
                try:
                    a['name'] = artist['name']
                except:
                    pass
            except:
                pass
            
            
            r['artist'] = a
            
            
            # label related mapping
            l = {}
            l['mb_id'] = None
            l['name'] = 'Unknown'
            l['code'] = None
            l['catalognumber'] = None
            
            try:
                label = release['label-info-list'][0]['label']
                print label
                
                try:
                    l['mb_id'] = label['id']
                except:
                    pass
                
                try:
                    l['name'] = label['name']
                except:
                    pass
                
                try:
                    l['code'] = label['label-code']
                except:
                    pass
                
                try:
                    l['catalognumber'] = release['label-info-list'][0]['catalog-number']
                except:
                    pass
                
            except Exception, e:
                print e
                pass
            
            
            
            r['label'] = l
            
            
            # relation mapping
            rel = {}
            rel['discogs_url'] = None
            rel['discogs_image'] = None
            
            try:
                #relations = release['url-relation-list']
                print
                print 'RELATIONS'
                print relations
                print
                print
                try:
                    
                    for relation in relations:
                        if relation['type'] == 'discogs':
                            rel['discogs_url'] = relation['target']
                            rel['discogs_image'] = discogs_image_by_url(relation['target'], 'uri150')
                            
                    
                except Exception, e:
                    print e
                    pass
                
            except Exception, e:
                print e
                pass
Example #10
0
    def complete_releases(self, releases):
        

        log = logging.getLogger('importer.process.complete_releases')
        log.info('Got %s releases to complete' % len(releases))
        
        completed_releases = []
        
        for release in releases:
            if release['id'] in completed_releases:
                log.debug('already completed release with id: %s' % release['id'])
                releases.remove(release)
                
            else:
                log.debug('complete release with id: %s' % release['id'])
                
                r_id = release['id']
                rg_id = release['release-group']['id']
                
                print 'r_id: %s' % r_id
                print 'rg_id: %s' % rg_id
                
                
                release['label'] = None
                release['catalog-number'] = None
                release['discogs_url'] = None
                release['discogs_master_url'] = None
                release['discogs_image'] = None
                
                
                """
                get release details
                """
                inc = ('labels', 'artists', 'url-rels', 'label-rels',)
                url = 'http://%s/ws/2/release/%s/?fmt=json&inc=%s' % (MUSICBRAINZ_HOST, r_id, "+".join(inc))
                
                r = requests.get(url)
                result = r.json()
                #self.pp.pprint(result)

                # only apply label info if unique
                if 'label-info' in result and len(result['label-info']) == 1:
                    if 'label' in result['label-info'][0]:
                        release['label'] = result['label-info'][0]['label']
                    if 'catalog-number' in result['label-info'][0]:
                        release['catalog-number'] = result['label-info'][0]['catalog-number']
                    
                # try to get discogs url
                if 'relations' in result:
                    for relation in result['relations']:
                        if relation['type'] == 'discogs':
                            log.debug('got discogs url from release: %s' % relation['url'])
                            release['discogs_url'] = relation['url']
                
                
                
                """
                get release-group details
                """
                inc = ('url-rels',)
                url = 'http://%s/ws/2/release-group/%s/?fmt=json&inc=%s' % (MUSICBRAINZ_HOST, rg_id, "+".join(inc))
                
                r = requests.get(url)
                result = r.json()
                #self.pp.pprint(result)
                    
                # try to get discogs master-url
                if 'relations' in result:
                    for relation in result['relations']:
                        if relation['type'] == 'discogs':
                            log.debug('got discogs url from release: %s' % relation['url'])
                            release['discogs_master_url'] = relation['url']
                        
                        
                        
                        
                """
                assign cover image (if available)
                """
                if release['discogs_url']:
                    try:
                        release['discogs_image'] = discogs_image_by_url(release['discogs_url'], 'uri150')
                    except:
                        pass
                    
                if not release['discogs_image']:
                    try:
                        release['discogs_image'] = discogs_image_by_url(release['discogs_master_url'], 'uri150')
                    except:
                        pass
                """
                finally try to get image from coverartarchive.org
                """
                if not release['discogs_image']:
                    url = 'http://coverartarchive.org/release/%s' % r_id
                    try:    
                        r = requests.get(url)
                        result = r.json()
                        release['discogs_image'] = result['images'][0]['image']
                    except:
                        pass
                    

                
                completed_releases.append(release['id'])
            
        
        
        return releases