コード例 #1
0
ファイル: migrate_test.py プロジェクト: hzlf/openbroadcast
    def import_image(self):
        print "image importer"

        url = "http://userpage.chemie.fu-berlin.de/~gd/root_broschuere/media/bilder/anclusnan/anclusnan2x.jpg"
        folder = Folder.objects.get(name="01574266-d9ab-11e1-ba53-b8f6b11a3aed")

        filer_extra.url_to_file(url, folder)

        return
コード例 #2
0
ファイル: migrate_test.py プロジェクト: ppp0/openbroadcast
    def import_image(self):
        print 'image importer'

        url = 'http://userpage.chemie.fu-berlin.de/~gd/root_broschuere/media/bilder/anclusnan/anclusnan2x.jpg'
        folder = Folder.objects.get(
            name='01574266-d9ab-11e1-ba53-b8f6b11a3aed')

        filer_extra.url_to_file(url, folder)

        return
コード例 #3
0
    def complete_release_meta(self, r, it):
        includes = [
            "artists", "labels", "recordings", "release-groups", "media",
            "artist-credits", "discids", "puids", "isrcs", "artist-rels",
            "label-rels", "recording-rels", "release-rels",
            "release-group-rels", "url-rels", "work-rels",
            "recording-level-rels", "work-level-rels"
        ]
        mb_release = musicbrainzngs.get_release_by_id(id=it['mb_release_id'],
                                                      includes=includes)

        mbr = mb_release['release']

        print '***************************************'

        if 'status' in mbr:
            print mbr['status']

        if 'title' in mbr:
            print mbr['title']

        if 'url-relation-list' in mbr:
            # print mbr['url-relation-list']

            for rel in mbr['url-relation-list']:
                print rel
                if rel['type'] == 'discogs':
                    print 'DISCOGS: %s' % rel['target']

                    try:
                        # pass
                        rel = Relation(content_object=r, url=rel['target'])
                        rel.save()
                    except Exception, e:
                        print 'RELATION EXCEPTION'
                        print e

                    try:
                        discogs_image = discogs_image_by_url(rel['target'])
                        img = filer_extra.url_to_file(discogs_image, r.folder)
                        r.main_image = img
                    except:
                        pass
コード例 #4
0
ファイル: importer.py プロジェクト: ppp0/openbroadcast
    def complete_release_meta(self, r, it):
        includes = [
            "artists", "labels", "recordings", "release-groups", "media",
            "artist-credits", "discids", "puids", "isrcs", "artist-rels",
            "label-rels", "recording-rels", "release-rels",
            "release-group-rels", "url-rels", "work-rels",
            "recording-level-rels", "work-level-rels"
        ]
        mb_release = musicbrainzngs.get_release_by_id(
            id=it['mb_release_id'], includes=includes)

        mbr = mb_release['release']

        print '***************************************'

        if 'status' in mbr:
            print mbr['status']

        if 'title' in mbr:
            print mbr['title']

        if 'url-relation-list' in mbr:
            # print mbr['url-relation-list']

            for rel in mbr['url-relation-list']:
                print rel
                if rel['type'] == 'discogs':
                    print 'DISCOGS: %s' % rel['target']

                    try:
                        # pass
                        rel = Relation(content_object=r, url=rel['target'])
                        rel.save()
                    except Exception, e:
                        print 'RELATION EXCEPTION'
                        print e

                    try:
                        discogs_image = discogs_image_by_url(rel['target'])
                        img = filer_extra.url_to_file(discogs_image, r.folder)
                        r.main_image = img
                    except:
                        pass
コード例 #5
0
ファイル: migrator.py プロジェクト: ppp0/openbroadcast
            # r.tags.clear()
            for nt in nts:
                try:
                    t = Ntags.objects.using('legacy').get(id=nt.ntag_id)
                    log.debug('tag for object: %s' % t.name)
                    Tag.objects.add_tag(obj, u'"%s"' % t.name[:30])
                except Exception, e:
                    print e
            """
			Get image
			"""
            try:
                img_url = 'http://openbroadcast.ch/static/images/release/%s/original.jpg' % id_to_location(
                    obj.legacy_id)
                log.debug('download image: %s' % img_url)
                img = filer_extra.url_to_file(img_url, obj.folder)
                obj.main_image = img
            except:
                pass
            """
			Finishing up
			"""
            obj.save()

        return obj, status


class MediaMigrator(Migrator):
    def __init__(self):
        log = logging.getLogger('util.migrator.__init__')
コード例 #6
0
ファイル: migrate_legacy.py プロジェクト: ppp0/openbroadcast
    def import_release(self, lr):

        print 'trying to get related data'

        lms = lr.mediasreleases_set.all()
        las = lr.artistsreleases_set.all()
        lls = lr.labelsreleases_set.all()

        print 'legacy_id: %s' % lr.id

        r, created = Release.objects.get_or_create(legacy_id=lr.id)

        if created:
            print 'Not here yet -> created'
        else:
            print 'found by legacy_id -> use'
        """
        Release creation/update & mapping
        """
        r.slug = slugify(lr.name)
        r.legacy_id = lr.id
        """
        Mapping new <> legacy
        """
        r.name = lr.name
        print u'%s' % r.id

        if lr.catalognumber:
            r.catalognumber = lr.catalognumber

        if lr.releasetype:
            r.releasetype = lr.releasetype

        if lr.releasestatus:
            r.releasestatus = lr.releasestatus

        if lr.published:
            r.publish_date = lr.published

        if lr.notes:
            r.excerpt = lr.notes

        if lr.totaltracks:
            r.totaltracks = lr.totaltracks
            print 'totaltracks: %s' % r.totaltracks

        if lr.releasecountry and len(lr.releasecountry) == 2:
            r.release_country = lr.releasecountry

        # "relation" mapping
        if lr.discogs_releaseid and lr.discogs_releaseid != 'nf':
            url = 'http://www.discogs.com/release/%s' % lr.discogs_releaseid
            print 'discogs_url: %s' % url
            rel = Relation(content_object=r, url=url)
            rel.save()

        if lr.myspace_url:
            print 'myspace_url: %s' % lr.myspace_url
            rel = Relation(content_object=r, url=lr.myspace_url)
            rel.save()

        if lr.wikipedia_url:
            print 'wikipedia_url: %s' % lr.wikipedia_url
            rel = Relation(content_object=r, url=lr.wikipedia_url)
            rel.save()

        if lr.releasedate:
            print 'legacy-date: %s' % lr.releasedate

            seg = lr.releasedate.split('-')

            print seg

            # year only
            if len(seg) == 1:
                r.releasedate = '%s-%s-%s' % (seg[0], '01', '01')

            # year & month only
            if len(seg) == 2:
                if seg[1] in ('00', '0'):
                    seg[1] = '01'
                r.releasedate = '%s-%s-%s' % (seg[0], seg[1], '01')

            # full date
            if len(seg) == 3 and seg[0] != '0000':
                if seg[1] in ('00', '0'):
                    seg[1] = '01'
                if seg[2] in ('00', '0'):
                    seg[2] = '01'
                r.releasedate = '%s-%s-%s' % (seg[0], seg[1], seg[2])

            print 'new-date: %s' % r.releasedate

        #time.sleep(2)

        r.save()

        # id:
        try:
            img_url = 'http://openbroadcast.ch/static/images/release/%s/original.jpg' % id_to_location(
                r.legacy_id)
            print img_url
            img = filer_extra.url_to_file(img_url, r.folder)

            r.main_image = img
            r.save()
        except:
            pass
        """
        Tag Mapping
        """
        ntrs = NtagsReleases.objects.using('legacy').filter(release_id=lr.id)
        # r.tags.clear()
        for ntr in ntrs:
            print 'Tag ID: %s' % ntr.ntag_id
            try:
                nt = Ntags.objects.using('legacy').get(id=ntr.ntag_id)
                print 'Tag Name: %s' % nt.name
                Tag.objects.add_tag(r, u'"%s"' % nt.name)
            except Exception, e:
                print e
                pass
コード例 #7
0
ファイル: migrator.py プロジェクト: ppp0/openbroadcast
			for nt in nts:
				try:
					t = Ntags.objects.using('legacy').get(id=nt.ntag_id)
					log.debug('tag for object: %s' % t.name)
					Tag.objects.add_tag(obj, u'"%s"' % t.name[:30])
				except Exception, e:
					print e
				
				
			"""
			Get image
			"""
			try:
				img_url = 'http://openbroadcast.ch/static/images/release/%s/original.jpg' % id_to_location(obj.legacy_id)
				log.debug('download image: %s' % img_url)
				img = filer_extra.url_to_file(img_url, obj.folder)
				obj.main_image = img
			except:
				pass
				
			"""
			Finishing up
			"""
			obj.save()
		
		return obj, status


class MediaMigrator(Migrator):

コード例 #8
0
    def mb_complete_artist(self, obj, mb_id):
        
        log = logging.getLogger('util.importer.mb_complete_artist')
        log.info('complete artist, a: %s | mb_id: %s' % (obj.name, mb_id))
        
        self.mb_completed.append(mb_id)
        
        inc = ('url-rels', 'tags')
        url = 'http://%s/ws/2/artist/%s/?fmt=json&inc=%s' % (MUSICBRAINZ_HOST, mb_id, "+".join(inc))
        
        r = requests.get(url)
        result = r.json()
        
        
        
        print '#########################################################################'
        self.pp.pprint(result)
        
        
        discogs_url = None
        discogs_image = None
        
        valid_relations = ('wikipedia', 'allmusic', 'BBC Music page', 'social network', 'official homepage', 'youtube', 'myspace',)
        
        relations = result.get('relations', ())

        for relation in relations:
            
            if relation['type'] == 'discogs':
                log.debug('got discogs url for artist: %s' % relation['url'])
                discogs_url = relation['url']
                
            if relation['type'] in valid_relations:
                log.debug('got %s url for artist: %s' % (relation['type'], relation['url']))
                

                try:
                    rel = Relation.objects.get(object_id=obj.pk, url=relation['url'])
                except:
                    rel = Relation(content_object=obj, url=relation['url'])
                    
                    if relation['type'] == 'official homepage':
                        rel.service = 'official'
                    
                    rel.save()

            
            
            
        if discogs_url:
            
            try:
                rel = Relation.objects.get(object_id=obj.pk, url=discogs_url)
            except:
                rel = Relation(content_object=obj, url=discogs_url)
                rel.save()
                
            # try to get image
            try:
                discogs_image = discogs_image_by_url(discogs_url, 'resource_url')
                log.debug('discogs image located at: %s' % discogs_image)
            except:
                pass
            
            
        # try to load & assign image
        if discogs_image:
            try:
                img = filer_extra.url_to_file(discogs_image, obj.folder)
                obj.main_image = img
                obj.save()
            except:
                log.info('unable to assign discogs image')
                
                
        if discogs_url:

            discogs_id = None
            try:
                # TODO: not sure if always working
                discogs_id = discogs_id_by_url(discogs_url)
                log.info('extracted discogs id: %s' % discogs_id)
            except:
                pass
            
            if discogs_id:
                url = 'http://api.discogs.com/artists/%s' % discogs_id
                r = requests.get(url)
                dgs_result = r.json()
                
                self.pp.pprint(dgs_result)
  
                """                  
                styles = dgs_result.get('styles', ())
                for style in styles:
                    log.debug('got style: %s' % (style))
                    Tag.objects.add_tag(obj, '"%s"' % style)
                """ 
                profile = dgs_result.get('profile', None)
                if profile:
                    obj.biography = profile
                    
                realname = dgs_result.get('realname', None)
                if realname:
                    obj.real_name = realname
                    
                """
                verry hackish part here, just as proof-of-concept
                """
                aliases = dgs_result.get('aliases', ())
                for alias in aliases:
                    try:
                        log.debug('got alias: %s' % alias['name'])
                        # TODO: improve! handle duplicates!
                        time.sleep(1.1)
                        r = requests.get(alias['resource_url'])
                        aa_result = r.json()
                        aa_discogs_url = aa_result.get('uri', None)
                        aa_name = aa_result.get('name', None)
                        aa_profile = aa_result.get('profile', None)
                        if aa_discogs_url and aa_name:
                            
                            l_as = lookup.artist_by_relation_url(aa_discogs_url)
                            l_a = None
                                
                            if len(l_as) < 1:
                                l_a = Artist(name=aa_name, biography=aa_profile)
                                l_a.save()
    
                                rel = Relation(content_object=l_a, url=aa_discogs_url)
                                rel.save()
                                
                            if len(l_as) == 1:
                                l_a = l_as[0]
                                print l_as[0]
                                
                            if l_a:
                                obj.aliases.add(l_a)
                    except:
                        pass
                    
                """
                verry hackish part here, just as proof-of-concept
                """
                members = dgs_result.get('members', ())
                for member in members:
                    try:
                        log.debug('got member: %s' % member['name'])
                        # TODO: improve! handle duplicates!
                        time.sleep(1.1)
                        r = requests.get(member['resource_url'])
                        ma_result = r.json()
                        ma_discogs_url = ma_result.get('uri', None)
                        ma_name = ma_result.get('name', None)
                        ma_profile = ma_result.get('profile', None)
                        if ma_discogs_url and ma_name:
                            
                            l_as = lookup.artist_by_relation_url(ma_discogs_url)
                            l_a = None
                                
                            if len(l_as) < 1:
                                l_a = Artist(name=ma_name, biography=ma_profile)
                                l_a.save()
    
                                rel = Relation(content_object=l_a, url=ma_discogs_url)
                                rel.save()
                                
                            if len(l_as) == 1:
                                l_a = l_as[0]
                                print l_as[0]
                                
                            if l_a:                                
                                ma = ArtistMembership.objects.get_or_create(parent=obj, child=l_a)
                                
                    except:
                        pass
                        
                
            
        type = result.get('type', None)
        if type:
            log.debug('got type: %s' % (type))
            obj.type = type
            
        disambiguation = result.get('disambiguation', None)
        if disambiguation:
            log.debug('got disambiguation: %s' % (disambiguation))
            obj.disambiguation = disambiguation
                    
        tags = result.get('tags', ())

        for tag in tags:
            log.debug('got tag: %s' % (tag['name']))
            Tag.objects.add_tag(obj, '"%s"' % tag['name'])
            
                    
        # add mb relation
        mb_url = 'http://musicbrainz.org/artist/%s' % (mb_id)
        try:
            rel = Relation.objects.get(object_id=obj.pk, url=mb_url)
        except:
            log.debug('relation not here yet, add it: %s' % (mb_url))
            rel = Relation(content_object=obj, url=mb_url)
            rel.save()
        
        obj.save()
        
        
        return obj
コード例 #9
0
    def mb_complete_release(self, obj, mb_id):
        
        log = logging.getLogger('util.importer.mb_complete_release')
        log.info('complete release, r: %s | mb_id: %s' % (obj.name, mb_id))
        
        inc = ('artists', 'url-rels', 'aliases', 'tags', 'recording-rels', 'work-rels', 'work-level-rels', 'artist-credits', 'labels', 'label-rels', 'release-groups')
        url = 'http://%s/ws/2/release/%s/?fmt=json&inc=%s' % (MUSICBRAINZ_HOST, mb_id, "+".join(inc))
        
        r = requests.get(url)
        result = r.json()
        
        self.pp.pprint(result)
        
        rg_id = None
        release_group = result.get('release-group', None)
        if release_group:
            rg_id = release_group.get('id', None)
            
        log.debug('release-group id: %s' % rg_id)
        
        discogs_url = None
        discogs_master_url = None
        discogs_image = None
        # try to get relations
        if 'relations' in result:
            for relation in result['relations']:
                
                if relation['type'] == 'discogs':
                    log.debug('got discogs url for release: %s' % relation['url'])
                    discogs_url = relation['url']
                    
                    # obj.save()
                    
                if relation['type'] == 'purchase for download':
                    log.debug('got purchase url for release: %s' % relation['url'])

                    try:
                        rel = Relation.objects.get(object_id=obj.pk, url=relation['url'])
                    except:
                        rel = Relation(content_object=obj, url=relation['url'])
                        rel.save()
                        
        

            
        if rg_id:
            # try to get discogs master url
            inc = ('url-rels',)
            url = 'http://%s/ws/2/release-group/%s/?fmt=json&inc=%s' % (MUSICBRAINZ_HOST, rg_id, "+".join(inc))
            
            r = requests.get(url)
            rg_result = r.json()
            
            print "*******************************************************************"
            self.pp.pprint(rg_result)

            # try to get relations from master
            if 'relations' in rg_result:
                for relation in rg_result['relations']:
                    
                    if relation['type'] == 'discogs':
                        log.debug('got discogs master-url for release: %s' % relation['url'])
                        discogs_master_url = relation['url']

                        
                    if relation['type'] == 'wikipedia':
                        log.debug('got wikipedia url for release: %s' % relation['url'])
    
                        try:
                            rel = Relation.objects.get(object_id=obj.pk, url=relation['url'])
                        except:
                            rel = Relation(content_object=obj, url=relation['url'])
                            rel.save()

                        
                    if relation['type'] == 'lyrics':
                        log.debug('got lyrics url for release: %s' % relation['url'])
    
                        try:
                            rel = Relation.objects.get(object_id=obj.pk, url=relation['url'])
                        except:
                            rel = Relation(content_object=obj, url=relation['url'])
                            rel.save()

                        
                    if relation['type'] == 'allmusic':
                        log.debug('got allmusic url for release: %s' % relation['url'])
    
                        try:
                            rel = Relation.objects.get(object_id=obj.pk, url=relation['url'])
                        except:
                            rel = Relation(content_object=obj, url=relation['url'])
                            rel.save()

                        
                    if relation['type'] == 'review':
                        log.debug('got review url for release: %s' % relation['url'])
    
                        try:
                            rel = Relation.objects.get(object_id=obj.pk, url=relation['url'])
                        except:
                            rel = Relation(content_object=obj, url=relation['url'])
                            rel.save()
        
        
        if discogs_url:
            
            try:
                rel = Relation.objects.get(object_id=obj.pk, url=discogs_url)
            except:
                rel = Relation(content_object=obj, url=discogs_url)
                rel.save()
                
            # try to get image
            try:
                discogs_image = discogs_image_by_url(discogs_url, 'resource_url')
                log.debug('discogs image located at: %s' % discogs_image)
            except:
                pass
            
        if discogs_master_url:
            
            try:
                rel = Relation.objects.get(object_id=obj.pk, url=discogs_master_url)
            except:
                rel = Relation(content_object=obj, url=discogs_master_url)
                rel.save()
                
            # try to get image from master
            if not discogs_image:
                try:
                    discogs_image = discogs_image_by_url(discogs_master_url, 'resource_url')
                    log.debug('discogs image located at: %s' % discogs_master_url)
                except:
                    pass
            
            
            
        # try to load & assign image
        if discogs_image:
            try:
                img = filer_extra.url_to_file(discogs_image, obj.folder)
                obj.main_image = img
                obj.save()
            except:
                log.info('unable to assign discogs image')
                
        else:
            # try at coverartarchive...
            url = 'http://coverartarchive.org/release/%s' % mb_id
            try:    
                r = requests.get(url)
                ca_result = r.json()
                ca_url = ca_result['images'][0]['image']
                img = filer_extra.url_to_file(ca_url, obj.folder)
                obj.main_image = img
                obj.save()
            except:
                pass
            
            
        # try to get some additional information from discogs
        if discogs_url:
            discogs_id = None
            try:
                discogs_id = re.findall(r'\d+', discogs_url)[0]
                log.info('extracted discogs id: %s' % discogs_id)
            except:
                pass
            
            if discogs_id:
                url = 'http://api.discogs.com/releases/%s' % discogs_id
                r = requests.get(url)
                dgs_result = r.json()
                    
                styles = dgs_result.get('styles', None)
                for style in styles:
                    log.debug('got style: %s' % (style))
                    Tag.objects.add_tag(obj, '"%s"' % style)
                    
                genres = dgs_result.get('genres', None)
                for genre in genres:
                    log.debug('got genre: %s' % (genre))
                    Tag.objects.add_tag(obj, '"%s"' % genre)
                    
                notes = dgs_result.get('notes', None)
                if notes:
                    obj.description = notes
                    
        if discogs_master_url:
            discogs_id = None
            try:
                discogs_id = re.findall(r'\d+', discogs_master_url)[0]
                log.info('extracted discogs id: %s' % discogs_id)
            except:
                pass
            
            if discogs_id:
                url = 'http://api.discogs.com/masters/%s' % discogs_id
                r = requests.get(url)
                dgs_result = r.json()
                    
                styles = dgs_result.get('styles', None)
                for style in styles:
                    log.debug('got style: %s' % (style))
                    Tag.objects.add_tag(obj, '"%s"' % style)
                    
                genres = dgs_result.get('genres', None)
                for genre in genres:
                    log.debug('got genre: %s' % (genre))
                    Tag.objects.add_tag(obj, '"%s"' % genre)
                    
                notes = dgs_result.get('notes', None)
                if notes:
                    obj.description = notes
                

                    
        tags = result.get('tags', ())
        for tag in tags:
            log.debug('got tag: %s' % (tag['name']))
            Tag.objects.add_tag(obj, '"%s"' % tag['name'])
            
        status = result.get('status', None)
        if status:
            log.debug('got status: %s' % (status))
            obj.releasestatus = status
            
        country = result.get('country', None)
        if country:
            log.debug('got country: %s' % (country))
            obj.release_country = country
            
        date = result.get('date', None)
        if date:
            log.debug('got date: %s' % (date))
            # TODO: rework field
            if len(date) == 4:
                date = '%s-00-00' % (date)
            elif len(date) == 7:
                date = '%s-00' % (date)
            elif len(date) == 10:
                date = '%s' % (date)
                
            re_date = re.compile('^\d{4}-\d{2}-\d{2}$')
            if re_date.match(date):
                obj.releasedate_approx = '%s' % date
            
            
        asin = result.get('asin', None)
        if asin:
            log.debug('got asin: %s' % (asin))
            obj.asin = asin
            
        barcode = result.get('barcode', None)
        if barcode:
            log.debug('got barcode: %s' % (barcode))
            # obj.barcode = barcode
            
                    
        # add mb relation
        mb_url = 'http://musicbrainz.org/release/%s' % (mb_id)
        try:
            rel = Relation.objects.get(object_id=obj.pk, url=mb_url)
        except:
            log.debug('relation not here yet, add it: %s' % (mb_url))
            rel = Relation(content_object=obj, url=mb_url)
            rel.save()

        obj.save()
        
        
        return obj
コード例 #10
0
ファイル: importer.py プロジェクト: ppp0/openbroadcast
    def mb_complete_artist(self, obj, mb_id):

        log = logging.getLogger('util.importer.mb_complete_artist')
        log.info('complete artist, a: %s | mb_id: %s' % (obj.name, mb_id))

        self.mb_completed.append(mb_id)

        inc = ('url-rels', 'tags')
        url = 'http://%s/ws/2/artist/%s/?fmt=json&inc=%s' % (
            MUSICBRAINZ_HOST, mb_id, "+".join(inc))

        r = requests.get(url)
        result = r.json()

        print '#########################################################################'
        self.pp.pprint(result)

        discogs_url = None
        discogs_image = None

        valid_relations = (
            'wikipedia',
            'allmusic',
            'BBC Music page',
            'social network',
            'official homepage',
            'youtube',
            'myspace',
        )

        relations = result.get('relations', ())

        for relation in relations:

            if relation['type'] == 'discogs':
                log.debug('got discogs url for artist: %s' % relation['url'])
                discogs_url = relation['url']

            if relation['type'] in valid_relations:
                log.debug('got %s url for artist: %s' %
                          (relation['type'], relation['url']))

                try:
                    rel = Relation.objects.get(object_id=obj.pk,
                                               url=relation['url'])
                except:
                    rel = Relation(content_object=obj, url=relation['url'])

                    if relation['type'] == 'official homepage':
                        rel.service = 'official'

                    rel.save()

        if discogs_url:

            try:
                rel = Relation.objects.get(object_id=obj.pk, url=discogs_url)
            except:
                rel = Relation(content_object=obj, url=discogs_url)
                rel.save()

            # try to get image
            try:
                discogs_image = discogs_image_by_url(discogs_url,
                                                     'resource_url')
                log.debug('discogs image located at: %s' % discogs_image)
            except:
                pass

        # try to load & assign image
        if discogs_image:
            try:
                img = filer_extra.url_to_file(discogs_image, obj.folder)
                obj.main_image = img
                obj.save()
            except:
                log.info('unable to assign discogs image')

        if discogs_url:

            discogs_id = None
            try:
                # TODO: not sure if always working
                discogs_id = discogs_id_by_url(discogs_url)
                log.info('extracted discogs id: %s' % discogs_id)
            except:
                pass

            if discogs_id:
                url = 'http://api.discogs.com/artists/%s' % discogs_id
                r = requests.get(url)
                dgs_result = r.json()

                self.pp.pprint(dgs_result)
                """                  
                styles = dgs_result.get('styles', ())
                for style in styles:
                    log.debug('got style: %s' % (style))
                    Tag.objects.add_tag(obj, '"%s"' % style)
                """
                profile = dgs_result.get('profile', None)
                if profile:
                    obj.biography = profile

                realname = dgs_result.get('realname', None)
                if realname:
                    obj.real_name = realname
                """
                verry hackish part here, just as proof-of-concept
                """
                aliases = dgs_result.get('aliases', ())
                for alias in aliases:
                    try:
                        log.debug('got alias: %s' % alias['name'])
                        # TODO: improve! handle duplicates!
                        time.sleep(1.1)
                        r = requests.get(alias['resource_url'])
                        aa_result = r.json()
                        aa_discogs_url = aa_result.get('uri', None)
                        aa_name = aa_result.get('name', None)
                        aa_profile = aa_result.get('profile', None)
                        if aa_discogs_url and aa_name:

                            l_as = lookup.artist_by_relation_url(
                                aa_discogs_url)
                            l_a = None

                            if len(l_as) < 1:
                                l_a = Artist(name=aa_name,
                                             biography=aa_profile)
                                l_a.save()

                                rel = Relation(content_object=l_a,
                                               url=aa_discogs_url)
                                rel.save()

                            if len(l_as) == 1:
                                l_a = l_as[0]
                                print l_as[0]

                            if l_a:
                                obj.aliases.add(l_a)
                    except:
                        pass
                """
                verry hackish part here, just as proof-of-concept
                """
                members = dgs_result.get('members', ())
                for member in members:
                    try:
                        log.debug('got member: %s' % member['name'])
                        # TODO: improve! handle duplicates!
                        time.sleep(1.1)
                        r = requests.get(member['resource_url'])
                        ma_result = r.json()
                        ma_discogs_url = ma_result.get('uri', None)
                        ma_name = ma_result.get('name', None)
                        ma_profile = ma_result.get('profile', None)
                        if ma_discogs_url and ma_name:

                            l_as = lookup.artist_by_relation_url(
                                ma_discogs_url)
                            l_a = None

                            if len(l_as) < 1:
                                l_a = Artist(name=ma_name,
                                             biography=ma_profile)
                                l_a.save()

                                rel = Relation(content_object=l_a,
                                               url=ma_discogs_url)
                                rel.save()

                            if len(l_as) == 1:
                                l_a = l_as[0]
                                print l_as[0]

                            if l_a:
                                ma = ArtistMembership.objects.get_or_create(
                                    parent=obj, child=l_a)

                    except:
                        pass

        type = result.get('type', None)
        if type:
            log.debug('got type: %s' % (type))
            obj.type = type

        disambiguation = result.get('disambiguation', None)
        if disambiguation:
            log.debug('got disambiguation: %s' % (disambiguation))
            obj.disambiguation = disambiguation

        tags = result.get('tags', ())

        for tag in tags:
            log.debug('got tag: %s' % (tag['name']))
            Tag.objects.add_tag(obj, '"%s"' % tag['name'])

        # add mb relation
        mb_url = 'http://musicbrainz.org/artist/%s' % (mb_id)
        try:
            rel = Relation.objects.get(object_id=obj.pk, url=mb_url)
        except:
            log.debug('relation not here yet, add it: %s' % (mb_url))
            rel = Relation(content_object=obj, url=mb_url)
            rel.save()

        obj.save()

        return obj
コード例 #11
0
ファイル: importer.py プロジェクト: ppp0/openbroadcast
    def mb_complete_release(self, obj, mb_id):

        log = logging.getLogger('util.importer.mb_complete_release')
        log.info('complete release, r: %s | mb_id: %s' % (obj.name, mb_id))

        inc = ('artists', 'url-rels', 'aliases', 'tags', 'recording-rels',
               'work-rels', 'work-level-rels', 'artist-credits', 'labels',
               'label-rels', 'release-groups')
        url = 'http://%s/ws/2/release/%s/?fmt=json&inc=%s' % (
            MUSICBRAINZ_HOST, mb_id, "+".join(inc))

        r = requests.get(url)
        result = r.json()

        self.pp.pprint(result)

        rg_id = None
        release_group = result.get('release-group', None)
        if release_group:
            rg_id = release_group.get('id', None)

        log.debug('release-group id: %s' % rg_id)

        discogs_url = None
        discogs_master_url = None
        discogs_image = None
        # try to get relations
        if 'relations' in result:
            for relation in result['relations']:

                if relation['type'] == 'discogs':
                    log.debug('got discogs url for release: %s' %
                              relation['url'])
                    discogs_url = relation['url']

                    # obj.save()

                if relation['type'] == 'purchase for download':
                    log.debug('got purchase url for release: %s' %
                              relation['url'])

                    try:
                        rel = Relation.objects.get(object_id=obj.pk,
                                                   url=relation['url'])
                    except:
                        rel = Relation(content_object=obj, url=relation['url'])
                        rel.save()

        if rg_id:
            # try to get discogs master url
            inc = ('url-rels', )
            url = 'http://%s/ws/2/release-group/%s/?fmt=json&inc=%s' % (
                MUSICBRAINZ_HOST, rg_id, "+".join(inc))

            r = requests.get(url)
            rg_result = r.json()

            print "*******************************************************************"
            self.pp.pprint(rg_result)

            # try to get relations from master
            if 'relations' in rg_result:
                for relation in rg_result['relations']:

                    if relation['type'] == 'discogs':
                        log.debug('got discogs master-url for release: %s' %
                                  relation['url'])
                        discogs_master_url = relation['url']

                    if relation['type'] == 'wikipedia':
                        log.debug('got wikipedia url for release: %s' %
                                  relation['url'])

                        try:
                            rel = Relation.objects.get(object_id=obj.pk,
                                                       url=relation['url'])
                        except:
                            rel = Relation(content_object=obj,
                                           url=relation['url'])
                            rel.save()

                    if relation['type'] == 'lyrics':
                        log.debug('got lyrics url for release: %s' %
                                  relation['url'])

                        try:
                            rel = Relation.objects.get(object_id=obj.pk,
                                                       url=relation['url'])
                        except:
                            rel = Relation(content_object=obj,
                                           url=relation['url'])
                            rel.save()

                    if relation['type'] == 'allmusic':
                        log.debug('got allmusic url for release: %s' %
                                  relation['url'])

                        try:
                            rel = Relation.objects.get(object_id=obj.pk,
                                                       url=relation['url'])
                        except:
                            rel = Relation(content_object=obj,
                                           url=relation['url'])
                            rel.save()

                    if relation['type'] == 'review':
                        log.debug('got review url for release: %s' %
                                  relation['url'])

                        try:
                            rel = Relation.objects.get(object_id=obj.pk,
                                                       url=relation['url'])
                        except:
                            rel = Relation(content_object=obj,
                                           url=relation['url'])
                            rel.save()

        if discogs_url:

            try:
                rel = Relation.objects.get(object_id=obj.pk, url=discogs_url)
            except:
                rel = Relation(content_object=obj, url=discogs_url)
                rel.save()

            # try to get image
            try:
                discogs_image = discogs_image_by_url(discogs_url,
                                                     'resource_url')
                log.debug('discogs image located at: %s' % discogs_image)
            except:
                pass

        if discogs_master_url:

            try:
                rel = Relation.objects.get(object_id=obj.pk,
                                           url=discogs_master_url)
            except:
                rel = Relation(content_object=obj, url=discogs_master_url)
                rel.save()

            # try to get image from master
            if not discogs_image:
                try:
                    discogs_image = discogs_image_by_url(
                        discogs_master_url, 'resource_url')
                    log.debug('discogs image located at: %s' %
                              discogs_master_url)
                except:
                    pass

        # try to load & assign image
        if discogs_image:
            try:
                img = filer_extra.url_to_file(discogs_image, obj.folder)
                obj.main_image = img
                obj.save()
            except:
                log.info('unable to assign discogs image')

        else:
            # try at coverartarchive...
            url = 'http://coverartarchive.org/release/%s' % mb_id
            try:
                r = requests.get(url)
                ca_result = r.json()
                ca_url = ca_result['images'][0]['image']
                img = filer_extra.url_to_file(ca_url, obj.folder)
                obj.main_image = img
                obj.save()
            except:
                pass

        # try to get some additional information from discogs
        if discogs_url:
            discogs_id = None
            try:
                discogs_id = re.findall(r'\d+', discogs_url)[0]
                log.info('extracted discogs id: %s' % discogs_id)
            except:
                pass

            if discogs_id:
                url = 'http://api.discogs.com/releases/%s' % discogs_id
                r = requests.get(url)
                dgs_result = r.json()

                styles = dgs_result.get('styles', None)
                for style in styles:
                    log.debug('got style: %s' % (style))
                    Tag.objects.add_tag(obj, '"%s"' % style)

                genres = dgs_result.get('genres', None)
                for genre in genres:
                    log.debug('got genre: %s' % (genre))
                    Tag.objects.add_tag(obj, '"%s"' % genre)

                notes = dgs_result.get('notes', None)
                if notes:
                    obj.description = notes

        if discogs_master_url:
            discogs_id = None
            try:
                discogs_id = re.findall(r'\d+', discogs_master_url)[0]
                log.info('extracted discogs id: %s' % discogs_id)
            except:
                pass

            if discogs_id:
                url = 'http://api.discogs.com/masters/%s' % discogs_id
                r = requests.get(url)
                dgs_result = r.json()

                styles = dgs_result.get('styles', None)
                for style in styles:
                    log.debug('got style: %s' % (style))
                    Tag.objects.add_tag(obj, '"%s"' % style)

                genres = dgs_result.get('genres', None)
                for genre in genres:
                    log.debug('got genre: %s' % (genre))
                    Tag.objects.add_tag(obj, '"%s"' % genre)

                notes = dgs_result.get('notes', None)
                if notes:
                    obj.description = notes

        tags = result.get('tags', ())
        for tag in tags:
            log.debug('got tag: %s' % (tag['name']))
            Tag.objects.add_tag(obj, '"%s"' % tag['name'])

        status = result.get('status', None)
        if status:
            log.debug('got status: %s' % (status))
            obj.releasestatus = status

        country = result.get('country', None)
        if country:
            log.debug('got country: %s' % (country))
            obj.release_country = country

        date = result.get('date', None)
        if date:
            log.debug('got date: %s' % (date))
            # TODO: rework field
            if len(date) == 4:
                date = '%s-00-00' % (date)
            elif len(date) == 7:
                date = '%s-00' % (date)
            elif len(date) == 10:
                date = '%s' % (date)

            re_date = re.compile('^\d{4}-\d{2}-\d{2}$')
            if re_date.match(date) and date != '0000-00-00':
                obj.releasedate_approx = '%s' % date

        asin = result.get('asin', None)
        if asin:
            log.debug('got asin: %s' % (asin))
            obj.asin = asin

        barcode = result.get('barcode', None)
        if barcode:
            log.debug('got barcode: %s' % (barcode))
            # obj.barcode = barcode

        # add mb relation
        mb_url = 'http://musicbrainz.org/release/%s' % (mb_id)
        try:
            rel = Relation.objects.get(object_id=obj.pk, url=mb_url)
        except:
            log.debug('relation not here yet, add it: %s' % (mb_url))
            rel = Relation(content_object=obj, url=mb_url)
            rel.save()

        obj.save()

        return obj
コード例 #12
0
ファイル: migrate_legacy.py プロジェクト: hzlf/openbroadcast
    def import_release(self, lr):

        print 'trying to get related data'
        
        lms = lr.mediasreleases_set.all()
        las = lr.artistsreleases_set.all()
        lls = lr.labelsreleases_set.all()
        
        
        print 'legacy_id: %s' % lr.id
        
        r, created = Release.objects.get_or_create(legacy_id=lr.id)
        
        if created:
            print 'Not here yet -> created'
        else:
            print 'found by legacy_id -> use'
        
        """
        Release creation/update & mapping
        """
        r.slug = slugify(lr.name)
        r.legacy_id = lr.id
        
        """
        Mapping new <> legacy
        """
        r.name = lr.name
        print u'%s' % r.id
        


        
        if lr.catalognumber:
            r.catalognumber = lr.catalognumber
        
        if lr.releasetype:
            r.releasetype = lr.releasetype
        
        if lr.releasestatus:
            r.releasestatus = lr.releasestatus
        
        if lr.published:
            r.publish_date = lr.published
        
        if lr.notes:
            r.excerpt = lr.notes
            
        if lr.totaltracks:
            r.totaltracks = lr.totaltracks
            print 'totaltracks: %s' % r.totaltracks
        
        if lr.releasecountry and len(lr.releasecountry) == 2:
            r.release_country = lr.releasecountry
            
            
        # "relation" mapping
        if lr.discogs_releaseid and lr.discogs_releaseid != 'nf':
            url = 'http://www.discogs.com/release/%s' % lr.discogs_releaseid
            print 'discogs_url: %s' % url
            rel = Relation(content_object=r, url=url)
            rel.save()
            
        if lr.myspace_url:
            print 'myspace_url: %s' % lr.myspace_url
            rel = Relation(content_object=r, url=lr.myspace_url)
            rel.save()
            
        if lr.wikipedia_url:
            print 'wikipedia_url: %s' % lr.wikipedia_url
            rel = Relation(content_object=r, url=lr.wikipedia_url)
            rel.save()
            
            
        if lr.releasedate:
            print 'legacy-date: %s' % lr.releasedate

            seg = lr.releasedate.split('-')
            
            print seg
            
            # year only
            if len(seg) == 1:
                r.releasedate = '%s-%s-%s' % (seg[0], '01', '01')
            
            # year & month only
            if len(seg) == 2:
                if seg[1] in ('00', '0'):
                    seg[1] = '01'
                r.releasedate = '%s-%s-%s' % (seg[0], seg[1], '01')
            
            # full date
            if len(seg) == 3 and seg[0] != '0000':
                if seg[1] in ('00', '0'):
                    seg[1] = '01'
                if seg[2] in ('00', '0'):
                    seg[2] = '01'
                r.releasedate = '%s-%s-%s' % (seg[0], seg[1], seg[2] )
                
            
            print 'new-date: %s' % r.releasedate
            
        
        
        
        
        #time.sleep(2)
        
        
        r.save()
        
        # id:
        try:
            img_url = 'http://openbroadcast.ch/static/images/release/%s/original.jpg' % id_to_location(r.legacy_id)
            print img_url
            img = filer_extra.url_to_file(img_url, r.folder)
            
            r.main_image = img
            r.save()
        except:
            pass
        
        
        """
        Tag Mapping
        """
        ntrs = NtagsReleases.objects.using('legacy').filter(release_id=lr.id)
        # r.tags.clear()
        for ntr in ntrs:
            print 'Tag ID: %s' % ntr.ntag_id
            try:
                nt = Ntags.objects.using('legacy').get(id=ntr.ntag_id)
                print 'Tag Name: %s' % nt.name
                Tag.objects.add_tag(r, u'"%s"' % nt.name)
            except Exception, e:
                print e
                pass