def get_field_translation_languages(self, field_name, include_english=True, as_choices=False):
return ([('en', LANGUAGES_NAMES['en']) if as_choices else 'en'] if include_english else []) + [
(language, verbose_language) if as_choices else language
for language, verbose_language in LANGUAGES_NAMES.items()
if (language in self._dict_choices(field_name + 's')
or modelHasField(self, u'{}_{}'.format(LANGUAGES_NAMES.get(language, None), field_name))
or modelHasField(self, u'{}_{}'.format(language, field_name)))
]
def save_item(model, unique_data, data):
if (data or unique_data):
for k, v in data.items():
if k.startswith('d_') and isinstance(v, dict):
data[k] = json.dumps(v)
print model.__name__
print '- Unique data:'
pprint(unique_data)
print '- Data:'
pprint(data)
data.update(unique_data)
try:
item = model.objects.get(
reduce(lambda qs, (k, v): qs | Q(**{k: v}),
[(k, v)
for k, v in unique_data.items() if v is not None],
Q()))
model.objects.filter(pk=item.pk).update(**data)
print 'Updated'
except ObjectDoesNotExist:
if modelHasField(
model, 'owner'
) and 'owner' not in data and 'owner_id' not in data:
data['owner_id'] = 1
item = model.objects.create(**data)
print 'Created'
def prepare_data(data, model, unique, download_images):
manytomany = {}
dictionaries = {}
images = {}
for k, v in data.items():
if modelHasField(model, k) and isinstance(model._meta.get_field(k),
ImageField):
if download_images:
images[k] = v
elif k.startswith('d_') and isinstance(v, dict):
if unique:
data[k] = json.dumps(v)
else:
dictionaries[k] = v
elif (k.startswith('i_') and not getattr(
model, u'{}_WITHOUT_I_CHOICES'.format(k[2:].upper()), False)
and not isinstance(v, int)):
data[k] = model.get_i(k[2:], v)
elif (k.startswith('c_') and isinstance(v, list)):
data[k] = join_data(*v)
elif (k.startswith('j_')):
data[k] = json.dumps(v)
elif not unique and isinstance(v, list):
manytomany[k] = v
for k in manytomany.keys():
del (data[k])
for k in dictionaries.keys():
del (data[k])
for k in images.keys():
del (data[k])
if unique:
return data
return data, manytomany, dictionaries, images
def update_image(model, field):
if field.name.startswith('_'):
return False
tinypng_api_key = getattr(django_settings, 'TINYPNG_API_KEY', None)
if tinypng_api_key and modelHasField(model, u'_original_{}'.format(field.name)):
if tinypng_compress(model, field):
return True
if tinypng_api_key and modelHasField(model, u'_tthumbnail_{}'.format(field.name)):
if tinypng_thumbnail(model, field):
return True
if modelHasField(model, u'_thumbnail_{}'.format(field.name)):
if thumbnail(model, field):
return True
return False
if modelHasField(model, u'_2x_{}'.format(field.name)):
if thumbnail(model, field):
return True
print 'todo generate a 2x version with waifux2'
return True
return False
def update_image(model, field):
if field.name.startswith('_'):
return False
tinypng_api_key = getattr(django_settings, 'TINYPNG_API_KEY', None)
if tinypng_api_key and modelHasField(model, u'_original_{}'.format(field.name)):
if tinypng_compress(model, field):
return True
if tinypng_api_key and modelHasField(model, u'_tthumbnail_{}'.format(field.name)):
if tinypng_thumbnail(model, field):
return True
if modelHasField(model, u'_thumbnail_{}'.format(field.name)):
if thumbnail(model, field):
return True
return False
if modelHasField(model, u'_2x_{}'.format(field.name)):
if thumbnail(model, field):
return True
print 'todo generate a 2x version with waifux2'
return True
return False
def update_markdown(model, field):
if modelHasField(model, u'_cache_{}'.format(field.name[2:])):
item = get_next_item(model, field, u'_cache_{}'.format(field.name[2:]))
if not item:
return False
print u'[Info] Updating markdown {} for {} #{}...'.format(field.name, model.__name__, item.pk)
r = requests.post(
u'https://api.github.com/markdown/raw',
data=getattr(item, field.name).encode('utf-8'),
headers={ 'content-type': u'text/plain' },
)
r.raise_for_status()
save_item(model, item, {
u'_cache_{}'.format(field.name[2:]): r.text,
})
print '[Info] Done.'
return True
return False
def update_markdown(model, field):
if modelHasField(model, u'_cache_{}'.format(field.name[2:])):
item = get_next_item(model, field, u'_cache_{}'.format(field.name[2:]))
if not item:
return False
print u'[Info] Updating markdown {} for {} #{}...'.format(
field.name, model.__name__, item.id)
r = requests.post(
u'https://api.github.com/markdown/raw',
data=getattr(item, field.name).encode('utf-8'),
headers={'content-type': u'text/plain'},
)
r.raise_for_status()
save_item(model, item, {
u'_cache_{}'.format(field.name[2:]): r.text,
})
print '[Info] Done.'
return True
return False
def update_markdown(model, field):
if modelHasField(model, u'_cache_{}'.format(field.name[2:])):
item = get_next_item(model, field, u'_cache_{}'.format(field.name[2:]))
if not item:
return False
print u'[Info] Updating markdown {} for {} #{}...'.format(field.name, model.__name__, item.pk)
headers = {
'content-type': u'text/plain',
}
if getattr(django_settings, 'GITHUB_API_TOKEN', None):
headers['Authorization'] = u'token {}'.format(django_settings.GITHUB_API_TOKEN)
r = requests.post(
u'https://api.github.com/markdown/raw',
data=getattr(item, field.name).encode('utf-8'),
headers=headers,
)
r.raise_for_status()
save_item(model, item, {
u'_cache_{}'.format(field.name[2:]): r.text,
})
print '[Info] Done.'
return True
return False
def dump_item(fileds, model, item, unique_fields, fields, foreign_keys,
many_to_many, foreign_keys_unique_fields, m2m_can_be_created):
data = {'owner_id': 1} if modelHasField(model, 'owner') else {}
raw_data = {}
unique_data = {}
m2m_update = {}
for foreign_key, foreign_key_model in foreign_keys.items():
value = getattr(item, foreign_key)
if value is not None:
fk_var_name = dump_dependencies_getters(
fileds, foreign_key_model, value,
foreign_keys_unique_fields[foreign_key])
raw_data[foreign_key] = fk_var_name
for m2m, m2m_model in many_to_many.items():
values = getattr(item, u'all_{}'.format(m2m))
fk_var_names = []
for value in values:
fk_var_names.append(
dump_dependencies_getters(fileds,
m2m_model,
value,
foreign_keys_unique_fields[m2m],
m2m_fields=m2m_can_be_created.get(
m2m, None)))
if fk_var_names:
m2m_update[m2m] = u'[{}]'.format(u', '.join(fk_var_names))
for field in unique_fields:
value = getattr(item, field)
if value is not None:
unique_data[field] = unicode(value)
for field in fields:
value = getattr(item, field)
if value is not None:
data[field] = unicode(value)
fk_var_name = 'fk_{}_{}'.format(model.__name__, item.pk)
print_to_files(
fileds, u'try: {fk_var_name}\nexcept NameError:\n'.format(
fk_var_name=fk_var_name))
lines_to_print_under_except = [
u'data = {data}'.format(data=data),
]
if raw_data:
lines_to_print_under_except.append(
u'data.update({dumped_raw_data})'.format(
dumped_raw_data=u'{open_b}{dumped_raw_data}{close_b}'.format(
open_b=u'{',
dumped_raw_data=u','.join([
u'"{key}": {value}'.format(key=key, value=value)
for key, value in raw_data.items()
]),
close_b=u'}',
), ))
lines_to_print_under_except += [
u'try:',
u' {fk_var_name} = {model_from}.{model_name}.objects.get({unique_data})'
.format(
fk_var_name=fk_var_name,
model_from=model.__module__,
model_name=model.__name__,
unique_data=u' | '.join([
u'Q({key}={value})'.format(key=key, value=repr(value))
for key, value in unique_data.items()
]),
),
u' {model_from}.{model_name}.objects.filter(pk={fk_var_name}.pk).update(**data)'
.format(
fk_var_name=fk_var_name,
model_from=model.__module__,
model_name=model.__name__,
),
u' {fk_var_name} = {model_from}.{model_name}.objects.get(pk={fk_var_name}.pk)'
.format(
fk_var_name=fk_var_name,
model_from=model.__module__,
model_name=model.__name__,
),
u' total_updated += 1',
u'except ObjectDoesNotExist:',
u' data.update({unique_data})'.format(unique_data=unique_data, ),
u' {fk_var_name} = {model_from}.{model_name}.objects.create(**data)'.
format(
fk_var_name=fk_var_name,
model_from=model.__module__,
model_name=model.__name__,
),
u' total_created += 1',
]
if m2m_update:
for key, value in m2m_update.items():
lines_to_print_under_except.append(
u'{fk_var_name}.{key} = {value}'.format(
fk_var_name=fk_var_name,
key=key,
value=value,
))
lines_to_print_under_except.append(
u'{fk_var_name}.save()'.format(fk_var_name=fk_var_name))
print_to_files(
fileds, u'{}\n\n'.format(u'\n'.join(
[u' {}'.format(line) for line in lines_to_print_under_except])))
def dump_item(fileds, model, item, unique_fields, fields, foreign_keys, many_to_many, foreign_keys_unique_fields, m2m_can_be_created):
data = { 'owner_id': 1 } if modelHasField(model, 'owner') else {}
raw_data = {}
unique_data = {}
m2m_update = {}
for foreign_key, foreign_key_model in foreign_keys.items():
value = getattr(item, foreign_key)
if value is not None:
fk_var_name = dump_dependencies_getters(fileds, foreign_key_model, value, foreign_keys_unique_fields[foreign_key])
raw_data[foreign_key] = fk_var_name
for m2m, m2m_model in many_to_many.items():
values = getattr(item, u'all_{}'.format(m2m))
fk_var_names = []
for value in values:
fk_var_names.append(dump_dependencies_getters(fileds, m2m_model, value, foreign_keys_unique_fields[m2m], m2m_fields=m2m_can_be_created.get(m2m, None)))
if fk_var_names:
m2m_update[m2m] = u'[{}]'.format(u', '.join(fk_var_names))
for field in unique_fields:
value = getattr(item, field)
if value is not None:
unique_data[field] = unicode(value)
for field in fields:
value = getattr(item, field)
if value is not None:
data[field] = unicode(value)
fk_var_name = 'fk_{}_{}'.format(model.__name__, item.pk)
print_to_files(fileds, u'try: {fk_var_name}\nexcept NameError:\n'.format(fk_var_name=fk_var_name))
lines_to_print_under_except = [
u'data = {data}'.format(data=data),
]
if raw_data:
lines_to_print_under_except.append(
u'data.update({dumped_raw_data})'.format(
dumped_raw_data=u'{open_b}{dumped_raw_data}{close_b}'.format(
open_b=u'{',
dumped_raw_data=u','.join([
u'"{key}": {value}'.format(key=key, value=value)
for key, value in raw_data.items()
]),
close_b=u'}',
),
)
)
lines_to_print_under_except += [
u'try:',
u' {fk_var_name} = {model_from}.{model_name}.objects.get({unique_data})'.format(
fk_var_name=fk_var_name,
model_from=model.__module__,
model_name=model.__name__,
unique_data=u' | '.join([
u'Q({key}={value})'.format(key=key, value=repr(value))
for key, value in unique_data.items()
]),
),
u' {model_from}.{model_name}.objects.filter(pk={fk_var_name}.pk).update(**data)'.format(
fk_var_name=fk_var_name,
model_from=model.__module__,
model_name=model.__name__,
),
u' {fk_var_name} = {model_from}.{model_name}.objects.get(pk={fk_var_name}.pk)'.format(
fk_var_name=fk_var_name,
model_from=model.__module__,
model_name=model.__name__,
),
u' total_updated += 1',
u'except ObjectDoesNotExist:',
u' data.update({unique_data})'.format(
unique_data=unique_data,
),
u' {fk_var_name} = {model_from}.{model_name}.objects.create(**data)'.format(
fk_var_name=fk_var_name,
model_from=model.__module__,
model_name=model.__name__,
),
u' total_created += 1',
]
if m2m_update:
for key, value in m2m_update.items():
lines_to_print_under_except.append(
u'{fk_var_name}.{key} = {value}'.format(
fk_var_name=fk_var_name,
key=key, value=value,
)
)
lines_to_print_under_except.append(
u'{fk_var_name}.save()'.format(fk_var_name=fk_var_name)
)
print_to_files(fileds, u'{}\n\n'.format(u'\n'.join([
u' {}'.format(line)
for line in lines_to_print_under_except
])))
def magiCirclesGeneratedSettings(existing_values):
now = timezone.now()
one_week_ago = now - datetime.timedelta(days=10)
generated_settings = {}
imports = []
############################################################
# Get settings only when missing:
# Get staff configurations and latest news
staff_configurations = existing_values.get('STAFF_CONFIGURATIONS', None)
latest_news = []
if not staff_configurations:
staff_configurations, more_latest_news = getStaffConfigurations()
latest_news += more_latest_news
# Get favorite characters
if FAVORITE_CHARACTERS_MODEL:
queryset = FAVORITE_CHARACTERS_FILTER(FAVORITE_CHARACTERS_MODEL.objects.all())
# Cache
if not existing_values.get('FAVORITE_CHARACTERS', None):
generateCharactersSettings(
queryset,
generated_settings=generated_settings,
imports=imports,
)
# Get birthday banners
if modelHasField(FAVORITE_CHARACTERS_MODEL, 'birthday'):
if not hasNewsOfCategory(latest_news, 'characters_birthdays'):
latest_news = getCharactersBirthdays(
queryset,
latest_news=latest_news,
)
# Other characters
if OTHER_CHARACTERS_MODELS:
generated_settings['OTHER_CHARACTERS_KEYS'] = OTHER_CHARACTERS_MODELS.keys()
for key, character_details in OTHER_CHARACTERS_MODELS.items():
if not isinstance(character_details, dict):
character_details = { 'model': character_details }
queryset = character_details.get('filter', lambda q: q)(character_details['model'].objects.all())
# Cache
if not existing_values.get(key, None):
generateCharactersSettings(
queryset,
generated_settings=generated_settings,
imports=imports,
for_favorite=False,
base_name=key,
)
# Get birthday banners
if modelHasField(character_details['model'], 'birthday'):
category = u'{}_birthdays'.format(key.lower())
if not hasNewsOfCategory(latest_news, category):
latest_news = getCharactersBirthdays(
queryset,
category=category,
latest_news=latest_news,
)
# Get total donators
if not existing_values.get('TOTAL_DONATORS', None):
generated_settings['TOTAL_DONATORS'] = totalDonatorsThisMonth() or '\'\''
# Get latest donation month
if not existing_values.get('DONATION_MONTH', None):
generated_settings['DONATION_MONTH'] = latestDonationMonth(failsafe=True)
# Get seasonal settings
if not existing_values.get('SEASONAL_SETTINGS', None):
generated_settings['SEASONAL_SETTINGS'] = seasonalGeneratedSettings(staff_configurations)
# Add banners for seasonal hashtags
if not hasNewsOfCategory(latest_news, 'seasonal_activity_tag'):
latest_news = getSeasonalActivityTagBanners(
seasonal_settings=existing_values.get('SEASONAL_SETTINGS', None) or generated_settings['SEASONAL_SETTINGS'],
latest_news=latest_news,
)
# Get users birthdays
if not hasNewsOfCategory(latest_news, 'users_birthdays'):
latest_news = getUsersBirthdaysToday(
latest_news=latest_news,
)
############################################################
# Always get:
# Generate share images once a week
if django_settings.DEBUG:
generated_share_images_last_date = now
generated_share_images = {}
else:
generated_share_images_last_date = getattr(django_settings, 'GENERATED_SHARE_IMAGES_LAST_DATE', None)
if (generated_share_images_last_date
and generated_share_images_last_date.replace(tzinfo=pytz.utc) > one_week_ago):
generated_share_images = getattr(django_settings, 'GENERATED_SHARE_IMAGES', {})
else:
generated_share_images_last_date = now
generated_share_images = {}
print 'Generate auto share images'
for collection_name, collection in getMagiCollections().items():
if collection.auto_share_image:
generated_share_images[collection.name] = generateShareImageForMainCollections(collection)
generated_settings['GENERATED_SHARE_IMAGES_LAST_DATE'] = 'datetime.datetime.fromtimestamp(' + unicode(
time.mktime(generated_share_images_last_date.timetuple())
) + ')'
generated_settings['GENERATED_SHARE_IMAGES'] = generated_share_images
# Get homepage arts
if GET_HOMEPAGE_ARTS:
generated_settings['HOMEPAGE_ARTS'] = GET_HOMEPAGE_ARTS()
# Get backgrounds
if GET_BACKGROUNDS:
generated_settings['BACKGROUNDS'] = GET_BACKGROUNDS()
# Get past tags count
generated_settings['PAST_ACTIVITY_TAGS_COUNT'] = {}
for tag_name, tag in ACTIVITY_TAGS.items():
if getEventStatus(
tag.get('start_date', None),
tag.get('end_date', None),
without_year_return='ended',
) == 'ended':
generated_settings['PAST_ACTIVITY_TAGS_COUNT'][tag_name] = models.Activity.objects.filter(
c_tags__contains='"{}"'.format(tag_name)).count()
############################################################
# Save
generated_settings.update({
'STAFF_CONFIGURATIONS': staff_configurations,
'LATEST_NEWS': latest_news,
})
return generated_settings, imports
def generateCharactersSettings(
queryset, generated_settings, imports=None,
for_favorite=True, base_name=None,
to_image=None,
):
imports.append('from collections import OrderedDict')
if not base_name:
if for_favorite:
base_name = 'FAVORITE_CHARACTERS'
else:
base_name = camelToSnakeCase(queryset.model.__name__, upper=True) + u'S'
print u'Get the {}'.format(snakeCaseToTitle(base_name))
generated_settings[base_name] = []
original_names = {}
for character in queryset:
name = unicode(character)
if to_image:
image = to_image(character)
else:
for image_field in [
'image_for_favorite_character',
'top_image_list', 'top_image',
'image_thumbnail_url', 'image_url',
]:
image = getattr(character, image_field, None)
if image:
break
if image and name:
original_names[character.pk] = name
generated_settings[base_name].append((
character.pk,
name,
image,
))
all_names = OrderedDict()
for language, _verbose in django_settings.LANGUAGES:
for character in queryset:
if character.pk not in original_names:
continue
translation_activate(language)
name = unicode(character)
if name != original_names[character.pk]:
if character.pk not in all_names:
all_names[character.pk] = {}
all_names[character.pk][language] = name
translation_activate('en')
generated_settings[u'{}_NAMES'.format(base_name)] = all_names
if modelHasField(queryset.model, 'birthday'):
generated_settings[u'{}_BIRTHDAYS'.format(base_name)] = OrderedDict([
(character.pk, (character.birthday.month, character.birthday.day))
for character in birthdayOrderingQueryset(queryset.exclude(birthday=None), order_by=True)
if character.pk in original_names and getattr(character, 'birthday', None)
])
generated_settings[u'{}_BIRTHDAY_TODAY'.format(base_name)] = queryset.filter(
birthdays_within(days_after=1, days_before=1)).values_list(
'pk', flat=True)
def __init__(self, *args, **kwargs):
super(BaseEventCollection, self).__init__(*args, **kwargs)
self.with_versions = modelHasField(self.queryset.model, 'c_versions')
self.versions = self.queryset.model.VERSIONS if self.with_versions else {}
def save_item(details,
unique_data,
data,
log_function=print,
json_item=None,
verbose=False,
download_images=False,
force_reload_images=False):
model = details['model']
unique_together = details.get('unique_together', False)
download_images = details.get('download_images', download_images)
find_existing_item = details.get('find_existing_item', None)
dont_erase_existing_value_fields = details.get(
'dont_erase_existing_value_fields', [])
if (data or unique_data):
unique_data = prepare_data(unique_data,
model,
unique=True,
download_images=download_images)
data, manytomany, dictionaries, images = prepare_data(
data,
model,
unique=False,
download_images=download_images,
)
if verbose:
log_function(model.__name__)
log_function('- Unique data:')
log_function(unique_data)
log_function('- Data:')
log_function(data)
data.update(unique_data)
if find_existing_item:
item = find_existing_item(model, unique_data, data, manytomany,
dictionaries)
else:
item = default_find_existing_item(model, unique_together,
unique_data)
if item:
data = {
k: v
for k, v in data.items()
if v or k not in dont_erase_existing_value_fields
}
model.objects.filter(pk=item.pk).update(**data)
item = model.objects.filter(pk=item.pk)[0]
if verbose:
log_function(u'Updated {} #{}'.format(model.__name__, item.pk))
else:
if modelHasField(
model, 'owner'
) and 'owner' not in data and 'owner_id' not in data:
data['owner'] = get_default_owner()
item = model.objects.create(**data)
log_function(u'Created {} #{}'.format(model.__name__, item.pk))
if manytomany:
if verbose:
log_function('- Many to many:')
for field_name, list_of_items in manytomany.items():
getattr(item, field_name).add(*list_of_items)
if verbose:
log_function(' ', field_name)
item.save()
if dictionaries:
if verbose:
log_function('- Updated dictionaries:')
for field_name, dictionary in dictionaries.items():
for k, v in dictionary.items():
item.add_d(field_name[2:], k, v)
if verbose:
log_function(' ', field_name,
getattr(item, field_name[2:]))
item.save()
if images:
saved_images = []
for field_name, url in images.items():
if not getattr(item, field_name, None) or force_reload_images:
image = saveImageURLToModel(item, field_name, url)
saved_images.append(field_name)
if saved_images:
if not verbose:
log_function(u'{} #{}'.format(model.__name__, item.pk))
log_function(u'- Uploaded images: {}'.format(
', '.join(saved_images)))
item.save()
if 'callback_after_save' in details:
details['callback_after_save'](details, item, json_item)
return item
return None