def find_matching_production_in_dz2_by_title(self, production_info):
# get all dz2 prodtype IDs that might conceivably match against any of the prodtypes for this prod
dz2_type_ids = []
for dz0_type_id in demozoo0.production_type_ids_for_production(production_info['id']):
dz2_type_ids += PRODUCTION_TYPE_SUGGESTIONS[dz0_type_id]
dz2_type_ids = tuple(set(dz2_type_ids))
# ditto for platform IDs
dz2_platform_ids = [-1]
for dz0_platform_id in demozoo0.platform_ids_for_production(production_info['id']):
dz2_platform_ids += PLATFORM_SUGGESTIONS[dz0_platform_id]
dz2_platform_ids = tuple(set(dz2_platform_ids))
return list(
Production.objects.raw('''
SELECT DISTINCT demoscene_production.* FROM demoscene_production
INNER JOIN demoscene_production_types ON (
demoscene_production.id = demoscene_production_types.production_id
AND demoscene_production_types.productiontype_id IN %s
)
LEFT JOIN demoscene_production_platforms ON demoscene_production.id = demoscene_production_platforms.production_id
WHERE
regexp_replace(LOWER(title), %s, '', 'g') = %s
AND (demoscene_production_platforms.platform_id IN %s OR demoscene_production_platforms.platform_id IS NULL)
''', (dz2_type_ids, PUNCTUATION_REGEX, self.depunctuate(production_info['name']), dz2_platform_ids) )
)
def find_matching_production_in_dz2_by_title(self, production_info):
# get all dz2 prodtype IDs that might conceivably match against any of the prodtypes for this prod
dz2_type_ids = []
for dz0_type_id in demozoo0.production_type_ids_for_production(
production_info['id']):
dz2_type_ids += PRODUCTION_TYPE_SUGGESTIONS[dz0_type_id]
dz2_type_ids = tuple(set(dz2_type_ids))
# ditto for platform IDs
dz2_platform_ids = [-1]
for dz0_platform_id in demozoo0.platform_ids_for_production(
production_info['id']):
dz2_platform_ids += PLATFORM_SUGGESTIONS[dz0_platform_id]
dz2_platform_ids = tuple(set(dz2_platform_ids))
return list(
Production.objects.raw(
'''
SELECT DISTINCT demoscene_production.* FROM demoscene_production
INNER JOIN demoscene_production_types ON (
demoscene_production.id = demoscene_production_types.production_id
AND demoscene_production_types.productiontype_id IN %s
)
LEFT JOIN demoscene_production_platforms ON demoscene_production.id = demoscene_production_platforms.production_id
WHERE
regexp_replace(LOWER(title), %s, '', 'g') = %s
AND (demoscene_production_platforms.platform_id IN %s OR demoscene_production_platforms.platform_id IS NULL)
''', (dz2_type_ids, PUNCTUATION_REGEX,
self.depunctuate(production_info['name']), dz2_platform_ids)))
def find_matching_production_in_dz2_by_title_and_author_names(
self, production_info):
# get all names of all releasers of this production
author_names = [
self.depunctuate(name) for name in
demozoo0.author_and_affiliation_names(production_info['id'])
]
if not author_names:
return
# find IDs of all nicks that match those names in any variant
nick_ids = self.find_matching_nicks_in_dz2_by_names(
author_names).values_list('id', flat=True)
if not nick_ids:
return
# get all dz2 prodtype IDs that might conceivably match against any of the prodtypes for this prod
dz2_type_ids = []
for dz0_type_id in demozoo0.production_type_ids_for_production(
production_info['id']):
dz2_type_ids += PRODUCTION_TYPE_SUGGESTIONS[dz0_type_id]
dz2_type_ids = tuple(set(dz2_type_ids))
# ditto for platform IDs
dz2_platform_ids = []
for dz0_platform_id in demozoo0.platform_ids_for_production(
production_info['id']):
dz2_platform_ids += PLATFORM_SUGGESTIONS[dz0_platform_id]
dz2_platform_ids = tuple(set(dz2_platform_ids))
if not dz2_platform_ids:
return
return list(
Production.objects.raw(
'''
SELECT DISTINCT demoscene_production.* FROM demoscene_production
INNER JOIN demoscene_production_types ON (
demoscene_production.id = demoscene_production_types.production_id
AND demoscene_production_types.productiontype_id IN %s
)
LEFT JOIN demoscene_production_platforms ON demoscene_production.id = demoscene_production_platforms.production_id
INNER JOIN demoscene_production_author_nicks ON (demoscene_production.id = demoscene_production_author_nicks.production_id)
INNER JOIN demoscene_production_author_affiliation_nicks ON (demoscene_production.id = demoscene_production_author_affiliation_nicks.production_id)
WHERE
regexp_replace(LOWER(title), %s, '', 'g') = %s
AND (demoscene_production_platforms.platform_id IN %s OR demoscene_production_platforms.platform_id IS NULL)
AND (
demoscene_production_author_nicks.nick_id IN %s
OR demoscene_production_author_affiliation_nicks.nick_id IN %s
)
''', (dz2_type_ids, PUNCTUATION_REGEX,
self.depunctuate(production_info['name']), dz2_platform_ids,
tuple(nick_ids), tuple(nick_ids))))
def find_matching_production_in_dz2_by_title_and_author_names(self, production_info):
# get all names of all releasers of this production
author_names = [
self.depunctuate(name)
for name in demozoo0.author_and_affiliation_names(production_info['id'])
]
if not author_names:
return
# find IDs of all nicks that match those names in any variant
nick_ids = self.find_matching_nicks_in_dz2_by_names(author_names).values_list('id', flat=True)
if not nick_ids:
return
# get all dz2 prodtype IDs that might conceivably match against any of the prodtypes for this prod
dz2_type_ids = []
for dz0_type_id in demozoo0.production_type_ids_for_production(production_info['id']):
dz2_type_ids += PRODUCTION_TYPE_SUGGESTIONS[dz0_type_id]
dz2_type_ids = tuple(set(dz2_type_ids))
# ditto for platform IDs
dz2_platform_ids = []
for dz0_platform_id in demozoo0.platform_ids_for_production(production_info['id']):
dz2_platform_ids += PLATFORM_SUGGESTIONS[dz0_platform_id]
dz2_platform_ids = tuple(set(dz2_platform_ids))
if not dz2_platform_ids:
return
return list(
Production.objects.raw('''
SELECT DISTINCT demoscene_production.* FROM demoscene_production
INNER JOIN demoscene_production_types ON (
demoscene_production.id = demoscene_production_types.production_id
AND demoscene_production_types.productiontype_id IN %s
)
LEFT JOIN demoscene_production_platforms ON demoscene_production.id = demoscene_production_platforms.production_id
INNER JOIN demoscene_production_author_nicks ON (demoscene_production.id = demoscene_production_author_nicks.production_id)
INNER JOIN demoscene_production_author_affiliation_nicks ON (demoscene_production.id = demoscene_production_author_affiliation_nicks.production_id)
WHERE
regexp_replace(LOWER(title), %s, '', 'g') = %s
AND (demoscene_production_platforms.platform_id IN %s OR demoscene_production_platforms.platform_id IS NULL)
AND (
demoscene_production_author_nicks.nick_id IN %s
OR demoscene_production_author_affiliation_nicks.nick_id IN %s
)
''', (dz2_type_ids, PUNCTUATION_REGEX, self.depunctuate(production_info['name']), dz2_platform_ids, tuple(nick_ids), tuple(nick_ids)) )
)
def find_or_create_production(self, production_info):
print "finding %s" % production_info['name']
matches = self.find_matching_production_in_dz2(production_info)
if matches:
production = matches[0]
print u'- found %s' % production
is_new = False
if not production.demozoo0_id:
production.demozoo0_id = production_info['id']
if not production.pouet_id:
production.pouet_id = production_info['pouet_id']
if not production.csdb_id:
production.csdb_id = production_info['csdb_id']
if not production.zxdemo_id:
production.zxdemo_id = production_info['zxdemo_id']
if not production.scene_org_id:
production.scene_org_id = production_info['scene_org_id']
else:
print "- not found"
is_new = True
dz0_platforms = demozoo0.platform_ids_for_production(production_info['id'])
dz2_platforms = []
for dz0_platform in dz0_platforms:
dz2_platform = PLATFORM_DZ0_TO_DZ2.get(dz0_platform)
if dz2_platform:
dz2_platforms.append(dz2_platform)
dz0_prod_types = demozoo0.production_type_ids_for_production(production_info['id'])
dz2_prod_types = []
tags = []
for dz0_prod_type in dz0_prod_types:
(dz2_prod_type, tag) = PRODUCTION_TYPE_DZ0_TO_DZ2.get(dz0_prod_type)
if dz0_prod_type == 19:
# special case: set dz2 prod type to bbstro if it's on DOS/Windows, intro otherwise
if 1 in dz2_platforms or 4 in dz2_platforms:
dz2_prod_types.append(41)
else:
dz2_prod_types.append(4)
elif dz2_prod_type:
dz2_prod_types.append(dz2_prod_type)
if tag:
tags.append(tag)
if dz2_prod_types == [42] and not demozoo0.production_has_competition_placings(production_info['id']):
print "live act, not in a competition - does not belong in dz2"
return None
if dz2_prod_types == [] and 32 in dz0_prod_types: # report
if dz2_platforms == [34]: # platform = video
if demozoo0.production_has_competition_placings(production_info['id']):
# appeared in a competition
dz2_prod_types.append(34)
tags.append('report')
else:
print "video report not in a competition - does not belong in dz2"
return None
else:
# some non-video platform - add as a demo
dz2_prod_types.append(1)
tags.append('report')
if dz2_prod_types == [] and 29 in dz0_prod_types: # invitation
dz2_prod_types.append(1) # demo
production = Production(
title = production_info['name'],
demozoo0_id = production_info['id'],
pouet_id = production_info['pouet_id'],
csdb_id = production_info['csdb_id'],
zxdemo_id = production_info['zxdemo_id'],
scene_org_id = production_info['scene_org_id'],
updated_at = datetime.datetime.now(),
data_source = 'dz0'
)
if dz2_prod_types:
production.supertype = ProductionType.objects.get(id = dz2_prod_types[0]).supertype
else:
production.supertype = 'production'
# check that release dates agree as far as they go
if production_info['release_date_precision']:
dz0_date = FuzzyDate(
production_info['release_date_datestamp'],
production_info['release_date_precision'])
else:
dz0_date = None
dz2_date = production.release_date
if dz0_date:
if not dz0_date.agrees_with(dz2_date):
print "WARNING: dates do not agree. dz0 = %s, dz2 = %s" % (dz0_date, dz2_date)
elif dz0_date.precision == 'd' and (dz2_date == None or dz2_date.precision in ['m','y']):
production.release_date_date = datetime.date(dz0_date.date.year, dz0_date.date.month, dz0_date.date.day)
production.release_date_precision = 'd'
elif dz0_date.precision == 'm' and (dz2_date == None or dz2_date.precision == 'y'):
production.release_date_date = datetime.date(dz0_date.date.year, dz0_date.date.month, 1)
production.release_date_precision = 'm'
elif dz0_date.precision == 'y' and dz2_date == None:
production.release_date_date = datetime.date(dz0_date.date.year, 1, 1)
production.release_date_precision = 'y'
production.save()
if is_new:
production.types = ProductionType.objects.filter(id__in = dz2_prod_types)
production.platforms = Platform.objects.filter(id__in = dz2_platforms)
if tags:
production.tags.add(*tags)
last_affiliation_list = None
for authorship in demozoo0.authorships_for_production(production_info['id']):
releaser = self.find_or_create_releaser(authorship['releaser'])
nick = self.find_or_create_nick_for_releaser(releaser, authorship['nick'])
production.author_nicks.add(nick)
affiliations = [affil for affil in demozoo0.affiliations_for_authorship(authorship['id'])]
affiliation_list = [affil['nick']['id'] for affil in affiliations]
if last_affiliation_list == None:
last_affiliation_list = affiliation_list
for affiliation in affiliations:
affil_releaser = self.find_or_create_releaser(affiliation['releaser'])
affil_nick = self.find_or_create_nick_for_releaser(affil_releaser, affiliation['nick'])
production.author_affiliation_nicks.add(affil_nick)
else:
if last_affiliation_list != affiliation_list:
print "WARNING: Different authors have different affiliations. Ignoring all but the first"
downloads = demozoo0.download_links_for_production(production_info['id'])
for download in downloads:
if production.download_links.filter(url = download['url']).count():
continue
if len(downloads) == 1 or re.search(r'zxdemo\.org', download['url']) or production_info['pouet_id'] == None:
pass
elif download['description'] == None:
pass
elif (re.search(r'(fix|version|patched)', download['description']) or re.match(r'disk', download['description']) or download['description'] == 'final') and not re.search(r'(video|youtube)', download['description']):
pass
else:
continue
DownloadLink.objects.create(
production_id = production.id,
demozoo0_id = download['id'],
url = download['url'],
description = (download['description'] or '')
)
return production
def find_or_create_production(self, production_info):
print "finding %s" % production_info['name']
matches = self.find_matching_production_in_dz2(production_info)
if matches:
production = matches[0]
print u'- found %s' % production
is_new = False
if not production.demozoo0_id:
production.demozoo0_id = production_info['id']
if not production.pouet_id:
production.pouet_id = production_info['pouet_id']
if not production.csdb_id:
production.csdb_id = production_info['csdb_id']
if not production.zxdemo_id:
production.zxdemo_id = production_info['zxdemo_id']
if not production.scene_org_id:
production.scene_org_id = production_info['scene_org_id']
else:
print "- not found"
is_new = True
dz0_platforms = demozoo0.platform_ids_for_production(
production_info['id'])
dz2_platforms = []
for dz0_platform in dz0_platforms:
dz2_platform = PLATFORM_DZ0_TO_DZ2.get(dz0_platform)
if dz2_platform:
dz2_platforms.append(dz2_platform)
dz0_prod_types = demozoo0.production_type_ids_for_production(
production_info['id'])
dz2_prod_types = []
tags = []
for dz0_prod_type in dz0_prod_types:
(dz2_prod_type,
tag) = PRODUCTION_TYPE_DZ0_TO_DZ2.get(dz0_prod_type)
if dz0_prod_type == 19:
# special case: set dz2 prod type to bbstro if it's on DOS/Windows, intro otherwise
if 1 in dz2_platforms or 4 in dz2_platforms:
dz2_prod_types.append(41)
else:
dz2_prod_types.append(4)
elif dz2_prod_type:
dz2_prod_types.append(dz2_prod_type)
if tag:
tags.append(tag)
if dz2_prod_types == [
42
] and not demozoo0.production_has_competition_placings(
production_info['id']):
print "live act, not in a competition - does not belong in dz2"
return None
if dz2_prod_types == [] and 32 in dz0_prod_types: # report
if dz2_platforms == [34]: # platform = video
if demozoo0.production_has_competition_placings(
production_info['id']):
# appeared in a competition
dz2_prod_types.append(34)
tags.append('report')
else:
print "video report not in a competition - does not belong in dz2"
return None
else:
# some non-video platform - add as a demo
dz2_prod_types.append(1)
tags.append('report')
if dz2_prod_types == [] and 29 in dz0_prod_types: # invitation
dz2_prod_types.append(1) # demo
production = Production(
title=production_info['name'],
demozoo0_id=production_info['id'],
pouet_id=production_info['pouet_id'],
csdb_id=production_info['csdb_id'],
zxdemo_id=production_info['zxdemo_id'],
scene_org_id=production_info['scene_org_id'],
updated_at=datetime.datetime.now(),
data_source='dz0')
if dz2_prod_types:
production.supertype = ProductionType.objects.get(
id=dz2_prod_types[0]).supertype
else:
production.supertype = 'production'
# check that release dates agree as far as they go
if production_info['release_date_precision']:
dz0_date = FuzzyDate(production_info['release_date_datestamp'],
production_info['release_date_precision'])
else:
dz0_date = None
dz2_date = production.release_date
if dz0_date:
if not dz0_date.agrees_with(dz2_date):
print "WARNING: dates do not agree. dz0 = %s, dz2 = %s" % (
dz0_date, dz2_date)
elif dz0_date.precision == 'd' and (
dz2_date == None or dz2_date.precision in ['m', 'y']):
production.release_date_date = datetime.date(
dz0_date.date.year, dz0_date.date.month, dz0_date.date.day)
production.release_date_precision = 'd'
elif dz0_date.precision == 'm' and (dz2_date == None
or dz2_date.precision == 'y'):
production.release_date_date = datetime.date(
dz0_date.date.year, dz0_date.date.month, 1)
production.release_date_precision = 'm'
elif dz0_date.precision == 'y' and dz2_date == None:
production.release_date_date = datetime.date(
dz0_date.date.year, 1, 1)
production.release_date_precision = 'y'
production.save()
if is_new:
production.types = ProductionType.objects.filter(
id__in=dz2_prod_types)
production.platforms = Platform.objects.filter(
id__in=dz2_platforms)
if tags:
production.tags.add(*tags)
last_affiliation_list = None
for authorship in demozoo0.authorships_for_production(
production_info['id']):
releaser = self.find_or_create_releaser(authorship['releaser'])
nick = self.find_or_create_nick_for_releaser(
releaser, authorship['nick'])
production.author_nicks.add(nick)
affiliations = [
affil for affil in demozoo0.affiliations_for_authorship(
authorship['id'])
]
affiliation_list = [
affil['nick']['id'] for affil in affiliations
]
if last_affiliation_list == None:
last_affiliation_list = affiliation_list
for affiliation in affiliations:
affil_releaser = self.find_or_create_releaser(
affiliation['releaser'])
affil_nick = self.find_or_create_nick_for_releaser(
affil_releaser, affiliation['nick'])
production.author_affiliation_nicks.add(affil_nick)
else:
if last_affiliation_list != affiliation_list:
print "WARNING: Different authors have different affiliations. Ignoring all but the first"
downloads = demozoo0.download_links_for_production(
production_info['id'])
for download in downloads:
if production.download_links.filter(url=download['url']).count():
continue
if len(downloads) == 1 or re.search(
r'zxdemo\.org',
download['url']) or production_info['pouet_id'] == None:
pass
elif download['description'] == None:
pass
elif (re.search(r'(fix|version|patched)', download['description'])
or re.match(r'disk', download['description'])
or download['description'] == 'final') and not re.search(
r'(video|youtube)', download['description']):
pass
else:
continue
DownloadLink.objects.create(production_id=production.id,
demozoo0_id=download['id'],
url=download['url'],
description=(download['description']
or ''))
return production
