예제 #1
0
    def mb_complete_media(self, obj, mb_id, excludes=()):
        
        log = logging.getLogger('util.importer.mb_complete_media')
        log.info('complete media, m: %s | mb_id: %s' % (obj.name, mb_id))
        
        #raw_input("Press Enter to continue...")
        time.sleep(1.1)
        
        inc = ('artists', 'url-rels', 'aliases', 'tags', 'recording-rels', 'artist-rels', 'work-level-rels', 'artist-credits')
        url = 'http://%s/ws/2/recording/%s/?fmt=json&inc=%s' % (MUSICBRAINZ_HOST, mb_id, "+".join(inc))
        
        r = requests.get(url)
        result = r.json()
        
        
        print '*****************************************************************'
        print '*****************************************************************'
        print '*****************************************************************'
        
        self.pp.pprint(result)


        print '*****************************************************************'
        print '*****************************************************************'
        print '*****************************************************************'
        
        # self.pp.pprint(result)
        if 'relations' in result:
            for relation in result['relations']:
    
                
                # map artists
                if 'artist' in relation:
                    print 'artist: %s' % relation['artist']['name']
                    print 'mb_id:   %s' % relation['artist']['id']
                    print 'role:   %s' % relation['type']
                    print
                    time.sleep(0.1)
                    l_as = lookup.artist_by_mb_id(relation['artist']['id'])
                    l_a = None

                    
                    #if len(l_as) < 1 and relation['artist']['id'] not in self.mb_completed:
                    if len(l_as) < 1 and relation['artist']['id'] not in excludes:
                        self.mb_completed.append(relation['artist']['id'])
                        l_a = Artist(name=relation['artist']['name'])
                        l_a.save()
                            
                        url = 'http://musicbrainz.org/artist/%s' % relation['artist']['id']
                        print 'musicbrainz_url: %s' % url
                        rel = Relation(content_object=l_a, url=url)
                        rel.save()
                        
                        print 'artist created'
                    if len(l_as) == 1:
                        print 'got artist!'
                        l_a = l_as[0]
                        print l_as[0]
                        
                    profession = None
                    if 'type' in relation:
                        profession, created = Profession.objects.get_or_create(name=relation['type'])
                        
                        
                    """"""
                    if l_a:
                        mea, created = MediaExtraartists.objects.get_or_create(artist=l_a, media=obj, profession=profession)
                        l_a = self.mb_complete_artist(l_a, relation['artist']['id'])
                    #self.pp.pprint(relation['artist']['name'])
                    
        tags = result.get('tags', ())
        for tag in tags:
            log.debug('got tag: %s' % (tag['name']))
            Tag.objects.add_tag(obj, '"%s"' % tag['name'])
            

                    
        # add mb relation
        mb_url = 'http://musicbrainz.org/recording/%s' % (mb_id)
        try:
            rel = Relation.objects.get(object_id=obj.pk, url=mb_url)
        except:
            log.debug('relation not here yet, add it: %s' % (mb_url))
            rel = Relation(content_object=obj, url=mb_url)
            rel.save()
        
        
        return obj
예제 #2
0
    def mb_complete_artist(self, obj, mb_id):
        
        log = logging.getLogger('util.importer.mb_complete_artist')
        log.info('complete artist, a: %s | mb_id: %s' % (obj.name, mb_id))
        
        self.mb_completed.append(mb_id)
        
        inc = ('url-rels', 'tags')
        url = 'http://%s/ws/2/artist/%s/?fmt=json&inc=%s' % (MUSICBRAINZ_HOST, mb_id, "+".join(inc))
        
        r = requests.get(url)
        result = r.json()
        
        
        
        print '#########################################################################'
        self.pp.pprint(result)
        
        
        discogs_url = None
        discogs_image = None
        
        valid_relations = ('wikipedia', 'allmusic', 'BBC Music page', 'social network', 'official homepage', 'youtube', 'myspace',)
        
        relations = result.get('relations', ())

        for relation in relations:
            
            if relation['type'] == 'discogs':
                log.debug('got discogs url for artist: %s' % relation['url'])
                discogs_url = relation['url']
                
            if relation['type'] in valid_relations:
                log.debug('got %s url for artist: %s' % (relation['type'], relation['url']))
                

                try:
                    rel = Relation.objects.get(object_id=obj.pk, url=relation['url'])
                except:
                    rel = Relation(content_object=obj, url=relation['url'])
                    
                    if relation['type'] == 'official homepage':
                        rel.service = 'official'
                    
                    rel.save()

            
            
            
        if discogs_url:
            
            try:
                rel = Relation.objects.get(object_id=obj.pk, url=discogs_url)
            except:
                rel = Relation(content_object=obj, url=discogs_url)
                rel.save()
                
            # try to get image
            try:
                discogs_image = discogs_image_by_url(discogs_url, 'resource_url')
                log.debug('discogs image located at: %s' % discogs_image)
            except:
                pass
            
            
        # try to load & assign image
        if discogs_image:
            try:
                img = filer_extra.url_to_file(discogs_image, obj.folder)
                obj.main_image = img
                obj.save()
            except:
                log.info('unable to assign discogs image')
                
                
        if discogs_url:

            discogs_id = None
            try:
                # TODO: not sure if always working
                discogs_id = discogs_id_by_url(discogs_url)
                log.info('extracted discogs id: %s' % discogs_id)
            except:
                pass
            
            if discogs_id:
                url = 'http://api.discogs.com/artists/%s' % discogs_id
                r = requests.get(url)
                dgs_result = r.json()
                
                self.pp.pprint(dgs_result)
  
                """                  
                styles = dgs_result.get('styles', ())
                for style in styles:
                    log.debug('got style: %s' % (style))
                    Tag.objects.add_tag(obj, '"%s"' % style)
                """ 
                profile = dgs_result.get('profile', None)
                if profile:
                    obj.biography = profile
                    
                realname = dgs_result.get('realname', None)
                if realname:
                    obj.real_name = realname
                    
                """
                verry hackish part here, just as proof-of-concept
                """
                aliases = dgs_result.get('aliases', ())
                for alias in aliases:
                    try:
                        log.debug('got alias: %s' % alias['name'])
                        # TODO: improve! handle duplicates!
                        time.sleep(1.1)
                        r = requests.get(alias['resource_url'])
                        aa_result = r.json()
                        aa_discogs_url = aa_result.get('uri', None)
                        aa_name = aa_result.get('name', None)
                        aa_profile = aa_result.get('profile', None)
                        if aa_discogs_url and aa_name:
                            
                            l_as = lookup.artist_by_relation_url(aa_discogs_url)
                            l_a = None
                                
                            if len(l_as) < 1:
                                l_a = Artist(name=aa_name, biography=aa_profile)
                                l_a.save()
    
                                rel = Relation(content_object=l_a, url=aa_discogs_url)
                                rel.save()
                                
                            if len(l_as) == 1:
                                l_a = l_as[0]
                                print l_as[0]
                                
                            if l_a:
                                obj.aliases.add(l_a)
                    except:
                        pass
                    
                """
                verry hackish part here, just as proof-of-concept
                """
                members = dgs_result.get('members', ())
                for member in members:
                    try:
                        log.debug('got member: %s' % member['name'])
                        # TODO: improve! handle duplicates!
                        time.sleep(1.1)
                        r = requests.get(member['resource_url'])
                        ma_result = r.json()
                        ma_discogs_url = ma_result.get('uri', None)
                        ma_name = ma_result.get('name', None)
                        ma_profile = ma_result.get('profile', None)
                        if ma_discogs_url and ma_name:
                            
                            l_as = lookup.artist_by_relation_url(ma_discogs_url)
                            l_a = None
                                
                            if len(l_as) < 1:
                                l_a = Artist(name=ma_name, biography=ma_profile)
                                l_a.save()
    
                                rel = Relation(content_object=l_a, url=ma_discogs_url)
                                rel.save()
                                
                            if len(l_as) == 1:
                                l_a = l_as[0]
                                print l_as[0]
                                
                            if l_a:                                
                                ma = ArtistMembership.objects.get_or_create(parent=obj, child=l_a)
                                
                    except:
                        pass
                        
                
            
        type = result.get('type', None)
        if type:
            log.debug('got type: %s' % (type))
            obj.type = type
            
        disambiguation = result.get('disambiguation', None)
        if disambiguation:
            log.debug('got disambiguation: %s' % (disambiguation))
            obj.disambiguation = disambiguation
                    
        tags = result.get('tags', ())

        for tag in tags:
            log.debug('got tag: %s' % (tag['name']))
            Tag.objects.add_tag(obj, '"%s"' % tag['name'])
            
                    
        # add mb relation
        mb_url = 'http://musicbrainz.org/artist/%s' % (mb_id)
        try:
            rel = Relation.objects.get(object_id=obj.pk, url=mb_url)
        except:
            log.debug('relation not here yet, add it: %s' % (mb_url))
            rel = Relation(content_object=obj, url=mb_url)
            rel.save()
        
        obj.save()
        
        
        return obj
예제 #3
0
            ia = Artist.objects.filter(pk__in=ii_ids, name=artist)
            print ia
            log.info('found artist in import session: %s' % ia)
        except:
            ia = None
        
        if ia and ia.count > 0:
            a = ia[0]
        
        
        # create artist if forced
        if force_artist and not a:
            log.info('artist, force creation: %s' % artist)

            a = Artist(name=artist)
            a.save()
            a_created = True
            
            
        # try to get artist by alibrary_id
        if alibrary_artist_id and not a:
            log.debug('artist, lookup by alibrary_artist_id: %s' % alibrary_artist_id)
            try:
                a = Artist.objects.get(pk=alibrary_artist_id)
                log.debug('got artist: %s by alibrary_artist_id: %s' % (a.pk, alibrary_artist_id))
            except Exception, e:
                # print e
                log.debug('could not get artist by alibrary_artist_id: %s' % alibrary_artist_id)
            
            
        # try to get artist by mb_id
예제 #4
0
    def mb_complete_artist(self, obj, mb_id):

        log = logging.getLogger('util.importer.mb_complete_artist')
        log.info('complete artist, a: %s | mb_id: %s' % (obj.name, mb_id))

        self.mb_completed.append(mb_id)

        inc = ('url-rels', 'tags')
        url = 'http://%s/ws/2/artist/%s/?fmt=json&inc=%s' % (
            MUSICBRAINZ_HOST, mb_id, "+".join(inc))

        r = requests.get(url)
        result = r.json()

        print '#########################################################################'
        self.pp.pprint(result)

        discogs_url = None
        discogs_image = None

        valid_relations = (
            'wikipedia',
            'allmusic',
            'BBC Music page',
            'social network',
            'official homepage',
            'youtube',
            'myspace',
        )

        relations = result.get('relations', ())

        for relation in relations:

            if relation['type'] == 'discogs':
                log.debug('got discogs url for artist: %s' % relation['url'])
                discogs_url = relation['url']

            if relation['type'] in valid_relations:
                log.debug('got %s url for artist: %s' %
                          (relation['type'], relation['url']))

                try:
                    rel = Relation.objects.get(object_id=obj.pk,
                                               url=relation['url'])
                except:
                    rel = Relation(content_object=obj, url=relation['url'])

                    if relation['type'] == 'official homepage':
                        rel.service = 'official'

                    rel.save()

        if discogs_url:

            try:
                rel = Relation.objects.get(object_id=obj.pk, url=discogs_url)
            except:
                rel = Relation(content_object=obj, url=discogs_url)
                rel.save()

            # try to get image
            try:
                discogs_image = discogs_image_by_url(discogs_url,
                                                     'resource_url')
                log.debug('discogs image located at: %s' % discogs_image)
            except:
                pass

        # try to load & assign image
        if discogs_image:
            try:
                img = filer_extra.url_to_file(discogs_image, obj.folder)
                obj.main_image = img
                obj.save()
            except:
                log.info('unable to assign discogs image')

        if discogs_url:

            discogs_id = None
            try:
                # TODO: not sure if always working
                discogs_id = discogs_id_by_url(discogs_url)
                log.info('extracted discogs id: %s' % discogs_id)
            except:
                pass

            if discogs_id:
                url = 'http://api.discogs.com/artists/%s' % discogs_id
                r = requests.get(url)
                dgs_result = r.json()

                self.pp.pprint(dgs_result)
                """                  
                styles = dgs_result.get('styles', ())
                for style in styles:
                    log.debug('got style: %s' % (style))
                    Tag.objects.add_tag(obj, '"%s"' % style)
                """
                profile = dgs_result.get('profile', None)
                if profile:
                    obj.biography = profile

                realname = dgs_result.get('realname', None)
                if realname:
                    obj.real_name = realname
                """
                verry hackish part here, just as proof-of-concept
                """
                aliases = dgs_result.get('aliases', ())
                for alias in aliases:
                    try:
                        log.debug('got alias: %s' % alias['name'])
                        # TODO: improve! handle duplicates!
                        time.sleep(1.1)
                        r = requests.get(alias['resource_url'])
                        aa_result = r.json()
                        aa_discogs_url = aa_result.get('uri', None)
                        aa_name = aa_result.get('name', None)
                        aa_profile = aa_result.get('profile', None)
                        if aa_discogs_url and aa_name:

                            l_as = lookup.artist_by_relation_url(
                                aa_discogs_url)
                            l_a = None

                            if len(l_as) < 1:
                                l_a = Artist(name=aa_name,
                                             biography=aa_profile)
                                l_a.save()

                                rel = Relation(content_object=l_a,
                                               url=aa_discogs_url)
                                rel.save()

                            if len(l_as) == 1:
                                l_a = l_as[0]
                                print l_as[0]

                            if l_a:
                                obj.aliases.add(l_a)
                    except:
                        pass
                """
                verry hackish part here, just as proof-of-concept
                """
                members = dgs_result.get('members', ())
                for member in members:
                    try:
                        log.debug('got member: %s' % member['name'])
                        # TODO: improve! handle duplicates!
                        time.sleep(1.1)
                        r = requests.get(member['resource_url'])
                        ma_result = r.json()
                        ma_discogs_url = ma_result.get('uri', None)
                        ma_name = ma_result.get('name', None)
                        ma_profile = ma_result.get('profile', None)
                        if ma_discogs_url and ma_name:

                            l_as = lookup.artist_by_relation_url(
                                ma_discogs_url)
                            l_a = None

                            if len(l_as) < 1:
                                l_a = Artist(name=ma_name,
                                             biography=ma_profile)
                                l_a.save()

                                rel = Relation(content_object=l_a,
                                               url=ma_discogs_url)
                                rel.save()

                            if len(l_as) == 1:
                                l_a = l_as[0]
                                print l_as[0]

                            if l_a:
                                ma = ArtistMembership.objects.get_or_create(
                                    parent=obj, child=l_a)

                    except:
                        pass

        type = result.get('type', None)
        if type:
            log.debug('got type: %s' % (type))
            obj.type = type

        disambiguation = result.get('disambiguation', None)
        if disambiguation:
            log.debug('got disambiguation: %s' % (disambiguation))
            obj.disambiguation = disambiguation

        tags = result.get('tags', ())

        for tag in tags:
            log.debug('got tag: %s' % (tag['name']))
            Tag.objects.add_tag(obj, '"%s"' % tag['name'])

        # add mb relation
        mb_url = 'http://musicbrainz.org/artist/%s' % (mb_id)
        try:
            rel = Relation.objects.get(object_id=obj.pk, url=mb_url)
        except:
            log.debug('relation not here yet, add it: %s' % (mb_url))
            rel = Relation(content_object=obj, url=mb_url)
            rel.save()

        obj.save()

        return obj
예제 #5
0
    def mb_complete_media(self, obj, mb_id, excludes=()):

        log = logging.getLogger('util.importer.mb_complete_media')
        log.info('complete media, m: %s | mb_id: %s' % (obj.name, mb_id))

        #raw_input("Press Enter to continue...")
        time.sleep(1.1)

        inc = ('artists', 'url-rels', 'aliases', 'tags', 'recording-rels',
               'artist-rels', 'work-level-rels', 'artist-credits')
        url = 'http://%s/ws/2/recording/%s/?fmt=json&inc=%s' % (
            MUSICBRAINZ_HOST, mb_id, "+".join(inc))

        r = requests.get(url)
        result = r.json()

        print '*****************************************************************'
        print '*****************************************************************'
        print '*****************************************************************'

        self.pp.pprint(result)

        print '*****************************************************************'
        print '*****************************************************************'
        print '*****************************************************************'

        # self.pp.pprint(result)
        if 'relations' in result:
            for relation in result['relations']:

                # map artists
                if 'artist' in relation:
                    print 'artist: %s' % relation['artist']['name']
                    print 'mb_id:   %s' % relation['artist']['id']
                    print 'role:   %s' % relation['type']
                    print
                    time.sleep(0.1)
                    l_as = lookup.artist_by_mb_id(relation['artist']['id'])
                    l_a = None

                    #if len(l_as) < 1 and relation['artist']['id'] not in self.mb_completed:
                    if len(l_as
                           ) < 1 and relation['artist']['id'] not in excludes:
                        self.mb_completed.append(relation['artist']['id'])
                        l_a = Artist(name=relation['artist']['name'])
                        l_a.save()

                        url = 'http://musicbrainz.org/artist/%s' % relation[
                            'artist']['id']
                        print 'musicbrainz_url: %s' % url
                        rel = Relation(content_object=l_a, url=url)
                        rel.save()

                        print 'artist created'
                    if len(l_as) == 1:
                        print 'got artist!'
                        l_a = l_as[0]
                        print l_as[0]

                    profession = None
                    if 'type' in relation:
                        profession, created = Profession.objects.get_or_create(
                            name=relation['type'])
                    """"""
                    if l_a:
                        mea, created = MediaExtraartists.objects.get_or_create(
                            artist=l_a, media=obj, profession=profession)
                        l_a = self.mb_complete_artist(l_a,
                                                      relation['artist']['id'])
                    #self.pp.pprint(relation['artist']['name'])

        tags = result.get('tags', ())
        for tag in tags:
            log.debug('got tag: %s' % (tag['name']))
            Tag.objects.add_tag(obj, '"%s"' % tag['name'])

        # add mb relation
        mb_url = 'http://musicbrainz.org/recording/%s' % (mb_id)
        try:
            rel = Relation.objects.get(object_id=obj.pk, url=mb_url)
        except:
            log.debug('relation not here yet, add it: %s' % (mb_url))
            rel = Relation(content_object=obj, url=mb_url)
            rel.save()

        return obj
예제 #6
0
            print ii_ids
            ia = Artist.objects.filter(pk__in=ii_ids, name=artist)
            print ia
            log.info('found artist in import session: %s' % ia)
        except:
            ia = None

        if ia and ia.count > 0:
            a = ia[0]

        # create artist if forced
        if force_artist and not a:
            log.info('artist, force creation: %s' % artist)

            a = Artist(name=artist)
            a.save()
            a_created = True

        # try to get artist by alibrary_id
        if alibrary_artist_id and not a:
            log.debug('artist, lookup by alibrary_artist_id: %s' %
                      alibrary_artist_id)
            try:
                a = Artist.objects.get(pk=alibrary_artist_id)
                log.debug('got artist: %s by alibrary_artist_id: %s' %
                          (a.pk, alibrary_artist_id))
            except Exception, e:
                # print e
                log.debug('could not get artist by alibrary_artist_id: %s' %
                          alibrary_artist_id)
예제 #7
0
def mb_complete_media_task(obj, mb_id, mb_release_id, excludes=()):

    log = logging.getLogger('util.importer.mb_complete_media')
    log.info('complete media, m: %s | mb_id: %s' % (obj.name, mb_id))

    #raw_input("Press Enter to continue...")
    time.sleep(1.1)

    inc = ('artists', 'url-rels', 'aliases', 'tags', 'recording-rels', 'artist-rels', 'work-level-rels', 'artist-credits')
    url = 'http://%s/ws/2/recording/%s/?fmt=json&inc=%s' % (MUSICBRAINZ_HOST, mb_id, "+".join(inc))

    r = requests.get(url)
    result = r.json()

    print '*****************************************************************'
    print url
    print '*****************************************************************'

    # get release based information (to map track- and disc-number)
    inc = ('recordings',)
    url = 'http://%s/ws/2/release/%s/?fmt=json&inc=%s' % (MUSICBRAINZ_HOST, mb_release_id, "+".join(inc))

    r = requests.get(url)
    result_release = r.json()
    print '*****************************************************************'
    print url
    print '*****************************************************************'

    print(result)

    print

    print(result_release)


    print '*****************************************************************'

    if DEBUG_WAIT:
        raw_input("Press Enter to continue...")



    # loop release recordings, trying to get our track...
    if 'media' in result_release:
        disc_index = 0
        media_index = 0
        media_offset = 0
        for disc in result_release['media']:

            for m in disc['tracks']:

                x_mb_id = m['recording']['id']
                x_pos = m['number']

                if x_mb_id == mb_id:
                    """
                    print 'id:  %s' % x_mb_id
                    print 'pos: %s' % x_pos
                    print 'disc_index: %s' % disc_index
                    print 'media_offset: %s' % media_offset
                    print 'final pos: %s' % (int(media_offset) + int(x_pos))
                    """

                    try:
                        obj.tracknumber = (int(media_offset) + int(x_pos))
                    except:
                        pass

                    try:
                        obj.mediamumber = int(disc_index)
                    except:
                        pass

                media_index =+ 1

            disc_index += 1
            media_offset += int(disc['track-count'])



    if DEBUG_WAIT:
        raw_input("Press Enter to continue...")


    if 'relations' in result:
        for relation in result['relations']:


            # map artists
            if 'artist' in relation:
                print 'artist: %s' % relation['artist']['name']
                print 'mb_id:   %s' % relation['artist']['id']
                print 'role:   %s' % relation['type']
                print
                time.sleep(0.1)
                l_as = lookup.artist_by_mb_id(relation['artist']['id'])
                l_a = None


                if len(l_as) < 1 and relation['artist']['id'] not in excludes:
                    #instance.mb_completed.append(relation['artist']['id'])
                    l_a = Artist(name=relation['artist']['name'])
                    l_a.save()

                    url = 'http://musicbrainz.org/artist/%s' % relation['artist']['id']
                    print 'musicbrainz_url: %s' % url
                    rel = Relation(content_object=l_a, url=url)
                    rel.save()

                    print 'artist created'
                if len(l_as) == 1:
                    print 'got artist!'
                    l_a = l_as[0]
                    print l_as[0]

                profession = None
                if 'type' in relation:
                    profession, created = Profession.objects.get_or_create(name=relation['type'])


                """"""
                if l_a:
                    mea, created = MediaExtraartists.objects.get_or_create(artist=l_a, media=obj, profession=profession)

                    if USE_CELERYD:
                        mb_complete_artist_task.delay(l_a, relation['artist']['id'])
                    else:
                        mb_complete_artist_task(l_a, relation['artist']['id'])


    tags = result.get('tags', ())
    for tag in tags:
        log.debug('got tag: %s' % (tag['name']))
        Tag.objects.add_tag(obj, '"%s"' % tag['name'])

    # add mb relation
    mb_url = 'http://musicbrainz.org/recording/%s' % (mb_id)
    try:
        rel = Relation.objects.get(object_id=obj.pk, url=mb_url)
    except:
        log.debug('relation not here yet, add it: %s' % (mb_url))
        rel = Relation(content_object=obj, url=mb_url)
        rel.save()


    return obj