def find_matching_production_in_dz2_by_title(self, production_info): # get all dz2 prodtype IDs that might conceivably match against any of the prodtypes for this prod dz2_type_ids = [] for dz0_type_id in demozoo0.production_type_ids_for_production(production_info['id']): dz2_type_ids += PRODUCTION_TYPE_SUGGESTIONS[dz0_type_id] dz2_type_ids = tuple(set(dz2_type_ids)) # ditto for platform IDs dz2_platform_ids = [-1] for dz0_platform_id in demozoo0.platform_ids_for_production(production_info['id']): dz2_platform_ids += PLATFORM_SUGGESTIONS[dz0_platform_id] dz2_platform_ids = tuple(set(dz2_platform_ids)) return list( Production.objects.raw(''' SELECT DISTINCT demoscene_production.* FROM demoscene_production INNER JOIN demoscene_production_types ON ( demoscene_production.id = demoscene_production_types.production_id AND demoscene_production_types.productiontype_id IN %s ) LEFT JOIN demoscene_production_platforms ON demoscene_production.id = demoscene_production_platforms.production_id WHERE regexp_replace(LOWER(title), %s, '', 'g') = %s AND (demoscene_production_platforms.platform_id IN %s OR demoscene_production_platforms.platform_id IS NULL) ''', (dz2_type_ids, PUNCTUATION_REGEX, self.depunctuate(production_info['name']), dz2_platform_ids) ) )
def find_matching_production_in_dz2_by_title(self, production_info): # get all dz2 prodtype IDs that might conceivably match against any of the prodtypes for this prod dz2_type_ids = [] for dz0_type_id in demozoo0.production_type_ids_for_production( production_info['id']): dz2_type_ids += PRODUCTION_TYPE_SUGGESTIONS[dz0_type_id] dz2_type_ids = tuple(set(dz2_type_ids)) # ditto for platform IDs dz2_platform_ids = [-1] for dz0_platform_id in demozoo0.platform_ids_for_production( production_info['id']): dz2_platform_ids += PLATFORM_SUGGESTIONS[dz0_platform_id] dz2_platform_ids = tuple(set(dz2_platform_ids)) return list( Production.objects.raw( ''' SELECT DISTINCT demoscene_production.* FROM demoscene_production INNER JOIN demoscene_production_types ON ( demoscene_production.id = demoscene_production_types.production_id AND demoscene_production_types.productiontype_id IN %s ) LEFT JOIN demoscene_production_platforms ON demoscene_production.id = demoscene_production_platforms.production_id WHERE regexp_replace(LOWER(title), %s, '', 'g') = %s AND (demoscene_production_platforms.platform_id IN %s OR demoscene_production_platforms.platform_id IS NULL) ''', (dz2_type_ids, PUNCTUATION_REGEX, self.depunctuate(production_info['name']), dz2_platform_ids)))
def find_matching_production_in_dz2_by_title_and_author_names( self, production_info): # get all names of all releasers of this production author_names = [ self.depunctuate(name) for name in demozoo0.author_and_affiliation_names(production_info['id']) ] if not author_names: return # find IDs of all nicks that match those names in any variant nick_ids = self.find_matching_nicks_in_dz2_by_names( author_names).values_list('id', flat=True) if not nick_ids: return # get all dz2 prodtype IDs that might conceivably match against any of the prodtypes for this prod dz2_type_ids = [] for dz0_type_id in demozoo0.production_type_ids_for_production( production_info['id']): dz2_type_ids += PRODUCTION_TYPE_SUGGESTIONS[dz0_type_id] dz2_type_ids = tuple(set(dz2_type_ids)) # ditto for platform IDs dz2_platform_ids = [] for dz0_platform_id in demozoo0.platform_ids_for_production( production_info['id']): dz2_platform_ids += PLATFORM_SUGGESTIONS[dz0_platform_id] dz2_platform_ids = tuple(set(dz2_platform_ids)) if not dz2_platform_ids: return return list( Production.objects.raw( ''' SELECT DISTINCT demoscene_production.* FROM demoscene_production INNER JOIN demoscene_production_types ON ( demoscene_production.id = demoscene_production_types.production_id AND demoscene_production_types.productiontype_id IN %s ) LEFT JOIN demoscene_production_platforms ON demoscene_production.id = demoscene_production_platforms.production_id INNER JOIN demoscene_production_author_nicks ON (demoscene_production.id = demoscene_production_author_nicks.production_id) INNER JOIN demoscene_production_author_affiliation_nicks ON (demoscene_production.id = demoscene_production_author_affiliation_nicks.production_id) WHERE regexp_replace(LOWER(title), %s, '', 'g') = %s AND (demoscene_production_platforms.platform_id IN %s OR demoscene_production_platforms.platform_id IS NULL) AND ( demoscene_production_author_nicks.nick_id IN %s OR demoscene_production_author_affiliation_nicks.nick_id IN %s ) ''', (dz2_type_ids, PUNCTUATION_REGEX, self.depunctuate(production_info['name']), dz2_platform_ids, tuple(nick_ids), tuple(nick_ids))))
def find_matching_production_in_dz2_by_title_and_author_names(self, production_info): # get all names of all releasers of this production author_names = [ self.depunctuate(name) for name in demozoo0.author_and_affiliation_names(production_info['id']) ] if not author_names: return # find IDs of all nicks that match those names in any variant nick_ids = self.find_matching_nicks_in_dz2_by_names(author_names).values_list('id', flat=True) if not nick_ids: return # get all dz2 prodtype IDs that might conceivably match against any of the prodtypes for this prod dz2_type_ids = [] for dz0_type_id in demozoo0.production_type_ids_for_production(production_info['id']): dz2_type_ids += PRODUCTION_TYPE_SUGGESTIONS[dz0_type_id] dz2_type_ids = tuple(set(dz2_type_ids)) # ditto for platform IDs dz2_platform_ids = [] for dz0_platform_id in demozoo0.platform_ids_for_production(production_info['id']): dz2_platform_ids += PLATFORM_SUGGESTIONS[dz0_platform_id] dz2_platform_ids = tuple(set(dz2_platform_ids)) if not dz2_platform_ids: return return list( Production.objects.raw(''' SELECT DISTINCT demoscene_production.* FROM demoscene_production INNER JOIN demoscene_production_types ON ( demoscene_production.id = demoscene_production_types.production_id AND demoscene_production_types.productiontype_id IN %s ) LEFT JOIN demoscene_production_platforms ON demoscene_production.id = demoscene_production_platforms.production_id INNER JOIN demoscene_production_author_nicks ON (demoscene_production.id = demoscene_production_author_nicks.production_id) INNER JOIN demoscene_production_author_affiliation_nicks ON (demoscene_production.id = demoscene_production_author_affiliation_nicks.production_id) WHERE regexp_replace(LOWER(title), %s, '', 'g') = %s AND (demoscene_production_platforms.platform_id IN %s OR demoscene_production_platforms.platform_id IS NULL) AND ( demoscene_production_author_nicks.nick_id IN %s OR demoscene_production_author_affiliation_nicks.nick_id IN %s ) ''', (dz2_type_ids, PUNCTUATION_REGEX, self.depunctuate(production_info['name']), dz2_platform_ids, tuple(nick_ids), tuple(nick_ids)) ) )
def find_or_create_production(self, production_info): print "finding %s" % production_info['name'] matches = self.find_matching_production_in_dz2(production_info) if matches: production = matches[0] print u'- found %s' % production is_new = False if not production.demozoo0_id: production.demozoo0_id = production_info['id'] if not production.pouet_id: production.pouet_id = production_info['pouet_id'] if not production.csdb_id: production.csdb_id = production_info['csdb_id'] if not production.zxdemo_id: production.zxdemo_id = production_info['zxdemo_id'] if not production.scene_org_id: production.scene_org_id = production_info['scene_org_id'] else: print "- not found" is_new = True dz0_platforms = demozoo0.platform_ids_for_production(production_info['id']) dz2_platforms = [] for dz0_platform in dz0_platforms: dz2_platform = PLATFORM_DZ0_TO_DZ2.get(dz0_platform) if dz2_platform: dz2_platforms.append(dz2_platform) dz0_prod_types = demozoo0.production_type_ids_for_production(production_info['id']) dz2_prod_types = [] tags = [] for dz0_prod_type in dz0_prod_types: (dz2_prod_type, tag) = PRODUCTION_TYPE_DZ0_TO_DZ2.get(dz0_prod_type) if dz0_prod_type == 19: # special case: set dz2 prod type to bbstro if it's on DOS/Windows, intro otherwise if 1 in dz2_platforms or 4 in dz2_platforms: dz2_prod_types.append(41) else: dz2_prod_types.append(4) elif dz2_prod_type: dz2_prod_types.append(dz2_prod_type) if tag: tags.append(tag) if dz2_prod_types == [42] and not demozoo0.production_has_competition_placings(production_info['id']): print "live act, not in a competition - does not belong in dz2" return None if dz2_prod_types == [] and 32 in dz0_prod_types: # report if dz2_platforms == [34]: # platform = video if demozoo0.production_has_competition_placings(production_info['id']): # appeared in a competition dz2_prod_types.append(34) tags.append('report') else: print "video report not in a competition - does not belong in dz2" return None else: # some non-video platform - add as a demo dz2_prod_types.append(1) tags.append('report') if dz2_prod_types == [] and 29 in dz0_prod_types: # invitation dz2_prod_types.append(1) # demo production = Production( title = production_info['name'], demozoo0_id = production_info['id'], pouet_id = production_info['pouet_id'], csdb_id = production_info['csdb_id'], zxdemo_id = production_info['zxdemo_id'], scene_org_id = production_info['scene_org_id'], updated_at = datetime.datetime.now(), data_source = 'dz0' ) if dz2_prod_types: production.supertype = ProductionType.objects.get(id = dz2_prod_types[0]).supertype else: production.supertype = 'production' # check that release dates agree as far as they go if production_info['release_date_precision']: dz0_date = FuzzyDate( production_info['release_date_datestamp'], production_info['release_date_precision']) else: dz0_date = None dz2_date = production.release_date if dz0_date: if not dz0_date.agrees_with(dz2_date): print "WARNING: dates do not agree. dz0 = %s, dz2 = %s" % (dz0_date, dz2_date) elif dz0_date.precision == 'd' and (dz2_date == None or dz2_date.precision in ['m','y']): production.release_date_date = datetime.date(dz0_date.date.year, dz0_date.date.month, dz0_date.date.day) production.release_date_precision = 'd' elif dz0_date.precision == 'm' and (dz2_date == None or dz2_date.precision == 'y'): production.release_date_date = datetime.date(dz0_date.date.year, dz0_date.date.month, 1) production.release_date_precision = 'm' elif dz0_date.precision == 'y' and dz2_date == None: production.release_date_date = datetime.date(dz0_date.date.year, 1, 1) production.release_date_precision = 'y' production.save() if is_new: production.types = ProductionType.objects.filter(id__in = dz2_prod_types) production.platforms = Platform.objects.filter(id__in = dz2_platforms) if tags: production.tags.add(*tags) last_affiliation_list = None for authorship in demozoo0.authorships_for_production(production_info['id']): releaser = self.find_or_create_releaser(authorship['releaser']) nick = self.find_or_create_nick_for_releaser(releaser, authorship['nick']) production.author_nicks.add(nick) affiliations = [affil for affil in demozoo0.affiliations_for_authorship(authorship['id'])] affiliation_list = [affil['nick']['id'] for affil in affiliations] if last_affiliation_list == None: last_affiliation_list = affiliation_list for affiliation in affiliations: affil_releaser = self.find_or_create_releaser(affiliation['releaser']) affil_nick = self.find_or_create_nick_for_releaser(affil_releaser, affiliation['nick']) production.author_affiliation_nicks.add(affil_nick) else: if last_affiliation_list != affiliation_list: print "WARNING: Different authors have different affiliations. Ignoring all but the first" downloads = demozoo0.download_links_for_production(production_info['id']) for download in downloads: if production.download_links.filter(url = download['url']).count(): continue if len(downloads) == 1 or re.search(r'zxdemo\.org', download['url']) or production_info['pouet_id'] == None: pass elif download['description'] == None: pass elif (re.search(r'(fix|version|patched)', download['description']) or re.match(r'disk', download['description']) or download['description'] == 'final') and not re.search(r'(video|youtube)', download['description']): pass else: continue DownloadLink.objects.create( production_id = production.id, demozoo0_id = download['id'], url = download['url'], description = (download['description'] or '') ) return production
def find_or_create_production(self, production_info): print "finding %s" % production_info['name'] matches = self.find_matching_production_in_dz2(production_info) if matches: production = matches[0] print u'- found %s' % production is_new = False if not production.demozoo0_id: production.demozoo0_id = production_info['id'] if not production.pouet_id: production.pouet_id = production_info['pouet_id'] if not production.csdb_id: production.csdb_id = production_info['csdb_id'] if not production.zxdemo_id: production.zxdemo_id = production_info['zxdemo_id'] if not production.scene_org_id: production.scene_org_id = production_info['scene_org_id'] else: print "- not found" is_new = True dz0_platforms = demozoo0.platform_ids_for_production( production_info['id']) dz2_platforms = [] for dz0_platform in dz0_platforms: dz2_platform = PLATFORM_DZ0_TO_DZ2.get(dz0_platform) if dz2_platform: dz2_platforms.append(dz2_platform) dz0_prod_types = demozoo0.production_type_ids_for_production( production_info['id']) dz2_prod_types = [] tags = [] for dz0_prod_type in dz0_prod_types: (dz2_prod_type, tag) = PRODUCTION_TYPE_DZ0_TO_DZ2.get(dz0_prod_type) if dz0_prod_type == 19: # special case: set dz2 prod type to bbstro if it's on DOS/Windows, intro otherwise if 1 in dz2_platforms or 4 in dz2_platforms: dz2_prod_types.append(41) else: dz2_prod_types.append(4) elif dz2_prod_type: dz2_prod_types.append(dz2_prod_type) if tag: tags.append(tag) if dz2_prod_types == [ 42 ] and not demozoo0.production_has_competition_placings( production_info['id']): print "live act, not in a competition - does not belong in dz2" return None if dz2_prod_types == [] and 32 in dz0_prod_types: # report if dz2_platforms == [34]: # platform = video if demozoo0.production_has_competition_placings( production_info['id']): # appeared in a competition dz2_prod_types.append(34) tags.append('report') else: print "video report not in a competition - does not belong in dz2" return None else: # some non-video platform - add as a demo dz2_prod_types.append(1) tags.append('report') if dz2_prod_types == [] and 29 in dz0_prod_types: # invitation dz2_prod_types.append(1) # demo production = Production( title=production_info['name'], demozoo0_id=production_info['id'], pouet_id=production_info['pouet_id'], csdb_id=production_info['csdb_id'], zxdemo_id=production_info['zxdemo_id'], scene_org_id=production_info['scene_org_id'], updated_at=datetime.datetime.now(), data_source='dz0') if dz2_prod_types: production.supertype = ProductionType.objects.get( id=dz2_prod_types[0]).supertype else: production.supertype = 'production' # check that release dates agree as far as they go if production_info['release_date_precision']: dz0_date = FuzzyDate(production_info['release_date_datestamp'], production_info['release_date_precision']) else: dz0_date = None dz2_date = production.release_date if dz0_date: if not dz0_date.agrees_with(dz2_date): print "WARNING: dates do not agree. dz0 = %s, dz2 = %s" % ( dz0_date, dz2_date) elif dz0_date.precision == 'd' and ( dz2_date == None or dz2_date.precision in ['m', 'y']): production.release_date_date = datetime.date( dz0_date.date.year, dz0_date.date.month, dz0_date.date.day) production.release_date_precision = 'd' elif dz0_date.precision == 'm' and (dz2_date == None or dz2_date.precision == 'y'): production.release_date_date = datetime.date( dz0_date.date.year, dz0_date.date.month, 1) production.release_date_precision = 'm' elif dz0_date.precision == 'y' and dz2_date == None: production.release_date_date = datetime.date( dz0_date.date.year, 1, 1) production.release_date_precision = 'y' production.save() if is_new: production.types = ProductionType.objects.filter( id__in=dz2_prod_types) production.platforms = Platform.objects.filter( id__in=dz2_platforms) if tags: production.tags.add(*tags) last_affiliation_list = None for authorship in demozoo0.authorships_for_production( production_info['id']): releaser = self.find_or_create_releaser(authorship['releaser']) nick = self.find_or_create_nick_for_releaser( releaser, authorship['nick']) production.author_nicks.add(nick) affiliations = [ affil for affil in demozoo0.affiliations_for_authorship( authorship['id']) ] affiliation_list = [ affil['nick']['id'] for affil in affiliations ] if last_affiliation_list == None: last_affiliation_list = affiliation_list for affiliation in affiliations: affil_releaser = self.find_or_create_releaser( affiliation['releaser']) affil_nick = self.find_or_create_nick_for_releaser( affil_releaser, affiliation['nick']) production.author_affiliation_nicks.add(affil_nick) else: if last_affiliation_list != affiliation_list: print "WARNING: Different authors have different affiliations. Ignoring all but the first" downloads = demozoo0.download_links_for_production( production_info['id']) for download in downloads: if production.download_links.filter(url=download['url']).count(): continue if len(downloads) == 1 or re.search( r'zxdemo\.org', download['url']) or production_info['pouet_id'] == None: pass elif download['description'] == None: pass elif (re.search(r'(fix|version|patched)', download['description']) or re.match(r'disk', download['description']) or download['description'] == 'final') and not re.search( r'(video|youtube)', download['description']): pass else: continue DownloadLink.objects.create(production_id=production.id, demozoo0_id=download['id'], url=download['url'], description=(download['description'] or '')) return production