def get_field_translation_languages(self, field_name, include_english=True, as_choices=False): return ([('en', LANGUAGES_NAMES['en']) if as_choices else 'en'] if include_english else []) + [ (language, verbose_language) if as_choices else language for language, verbose_language in LANGUAGES_NAMES.items() if (language in self._dict_choices(field_name + 's') or modelHasField(self, u'{}_{}'.format(LANGUAGES_NAMES.get(language, None), field_name)) or modelHasField(self, u'{}_{}'.format(language, field_name))) ]
def save_item(model, unique_data, data): if (data or unique_data): for k, v in data.items(): if k.startswith('d_') and isinstance(v, dict): data[k] = json.dumps(v) print model.__name__ print '- Unique data:' pprint(unique_data) print '- Data:' pprint(data) data.update(unique_data) try: item = model.objects.get( reduce(lambda qs, (k, v): qs | Q(**{k: v}), [(k, v) for k, v in unique_data.items() if v is not None], Q())) model.objects.filter(pk=item.pk).update(**data) print 'Updated' except ObjectDoesNotExist: if modelHasField( model, 'owner' ) and 'owner' not in data and 'owner_id' not in data: data['owner_id'] = 1 item = model.objects.create(**data) print 'Created'
def prepare_data(data, model, unique, download_images): manytomany = {} dictionaries = {} images = {} for k, v in data.items(): if modelHasField(model, k) and isinstance(model._meta.get_field(k), ImageField): if download_images: images[k] = v elif k.startswith('d_') and isinstance(v, dict): if unique: data[k] = json.dumps(v) else: dictionaries[k] = v elif (k.startswith('i_') and not getattr( model, u'{}_WITHOUT_I_CHOICES'.format(k[2:].upper()), False) and not isinstance(v, int)): data[k] = model.get_i(k[2:], v) elif (k.startswith('c_') and isinstance(v, list)): data[k] = join_data(*v) elif (k.startswith('j_')): data[k] = json.dumps(v) elif not unique and isinstance(v, list): manytomany[k] = v for k in manytomany.keys(): del (data[k]) for k in dictionaries.keys(): del (data[k]) for k in images.keys(): del (data[k]) if unique: return data return data, manytomany, dictionaries, images
def update_image(model, field): if field.name.startswith('_'): return False tinypng_api_key = getattr(django_settings, 'TINYPNG_API_KEY', None) if tinypng_api_key and modelHasField(model, u'_original_{}'.format(field.name)): if tinypng_compress(model, field): return True if tinypng_api_key and modelHasField(model, u'_tthumbnail_{}'.format(field.name)): if tinypng_thumbnail(model, field): return True if modelHasField(model, u'_thumbnail_{}'.format(field.name)): if thumbnail(model, field): return True return False if modelHasField(model, u'_2x_{}'.format(field.name)): if thumbnail(model, field): return True print 'todo generate a 2x version with waifux2' return True return False
def update_markdown(model, field): if modelHasField(model, u'_cache_{}'.format(field.name[2:])): item = get_next_item(model, field, u'_cache_{}'.format(field.name[2:])) if not item: return False print u'[Info] Updating markdown {} for {} #{}...'.format(field.name, model.__name__, item.pk) r = requests.post( u'https://api.github.com/markdown/raw', data=getattr(item, field.name).encode('utf-8'), headers={ 'content-type': u'text/plain' }, ) r.raise_for_status() save_item(model, item, { u'_cache_{}'.format(field.name[2:]): r.text, }) print '[Info] Done.' return True return False
def update_markdown(model, field): if modelHasField(model, u'_cache_{}'.format(field.name[2:])): item = get_next_item(model, field, u'_cache_{}'.format(field.name[2:])) if not item: return False print u'[Info] Updating markdown {} for {} #{}...'.format( field.name, model.__name__, item.id) r = requests.post( u'https://api.github.com/markdown/raw', data=getattr(item, field.name).encode('utf-8'), headers={'content-type': u'text/plain'}, ) r.raise_for_status() save_item(model, item, { u'_cache_{}'.format(field.name[2:]): r.text, }) print '[Info] Done.' return True return False
def update_markdown(model, field): if modelHasField(model, u'_cache_{}'.format(field.name[2:])): item = get_next_item(model, field, u'_cache_{}'.format(field.name[2:])) if not item: return False print u'[Info] Updating markdown {} for {} #{}...'.format(field.name, model.__name__, item.pk) headers = { 'content-type': u'text/plain', } if getattr(django_settings, 'GITHUB_API_TOKEN', None): headers['Authorization'] = u'token {}'.format(django_settings.GITHUB_API_TOKEN) r = requests.post( u'https://api.github.com/markdown/raw', data=getattr(item, field.name).encode('utf-8'), headers=headers, ) r.raise_for_status() save_item(model, item, { u'_cache_{}'.format(field.name[2:]): r.text, }) print '[Info] Done.' return True return False
def dump_item(fileds, model, item, unique_fields, fields, foreign_keys, many_to_many, foreign_keys_unique_fields, m2m_can_be_created): data = {'owner_id': 1} if modelHasField(model, 'owner') else {} raw_data = {} unique_data = {} m2m_update = {} for foreign_key, foreign_key_model in foreign_keys.items(): value = getattr(item, foreign_key) if value is not None: fk_var_name = dump_dependencies_getters( fileds, foreign_key_model, value, foreign_keys_unique_fields[foreign_key]) raw_data[foreign_key] = fk_var_name for m2m, m2m_model in many_to_many.items(): values = getattr(item, u'all_{}'.format(m2m)) fk_var_names = [] for value in values: fk_var_names.append( dump_dependencies_getters(fileds, m2m_model, value, foreign_keys_unique_fields[m2m], m2m_fields=m2m_can_be_created.get( m2m, None))) if fk_var_names: m2m_update[m2m] = u'[{}]'.format(u', '.join(fk_var_names)) for field in unique_fields: value = getattr(item, field) if value is not None: unique_data[field] = unicode(value) for field in fields: value = getattr(item, field) if value is not None: data[field] = unicode(value) fk_var_name = 'fk_{}_{}'.format(model.__name__, item.pk) print_to_files( fileds, u'try: {fk_var_name}\nexcept NameError:\n'.format( fk_var_name=fk_var_name)) lines_to_print_under_except = [ u'data = {data}'.format(data=data), ] if raw_data: lines_to_print_under_except.append( u'data.update({dumped_raw_data})'.format( dumped_raw_data=u'{open_b}{dumped_raw_data}{close_b}'.format( open_b=u'{', dumped_raw_data=u','.join([ u'"{key}": {value}'.format(key=key, value=value) for key, value in raw_data.items() ]), close_b=u'}', ), )) lines_to_print_under_except += [ u'try:', u' {fk_var_name} = {model_from}.{model_name}.objects.get({unique_data})' .format( fk_var_name=fk_var_name, model_from=model.__module__, model_name=model.__name__, unique_data=u' | '.join([ u'Q({key}={value})'.format(key=key, value=repr(value)) for key, value in unique_data.items() ]), ), u' {model_from}.{model_name}.objects.filter(pk={fk_var_name}.pk).update(**data)' .format( fk_var_name=fk_var_name, model_from=model.__module__, model_name=model.__name__, ), u' {fk_var_name} = {model_from}.{model_name}.objects.get(pk={fk_var_name}.pk)' .format( fk_var_name=fk_var_name, model_from=model.__module__, model_name=model.__name__, ), u' total_updated += 1', u'except ObjectDoesNotExist:', u' data.update({unique_data})'.format(unique_data=unique_data, ), u' {fk_var_name} = {model_from}.{model_name}.objects.create(**data)'. format( fk_var_name=fk_var_name, model_from=model.__module__, model_name=model.__name__, ), u' total_created += 1', ] if m2m_update: for key, value in m2m_update.items(): lines_to_print_under_except.append( u'{fk_var_name}.{key} = {value}'.format( fk_var_name=fk_var_name, key=key, value=value, )) lines_to_print_under_except.append( u'{fk_var_name}.save()'.format(fk_var_name=fk_var_name)) print_to_files( fileds, u'{}\n\n'.format(u'\n'.join( [u' {}'.format(line) for line in lines_to_print_under_except])))
def dump_item(fileds, model, item, unique_fields, fields, foreign_keys, many_to_many, foreign_keys_unique_fields, m2m_can_be_created): data = { 'owner_id': 1 } if modelHasField(model, 'owner') else {} raw_data = {} unique_data = {} m2m_update = {} for foreign_key, foreign_key_model in foreign_keys.items(): value = getattr(item, foreign_key) if value is not None: fk_var_name = dump_dependencies_getters(fileds, foreign_key_model, value, foreign_keys_unique_fields[foreign_key]) raw_data[foreign_key] = fk_var_name for m2m, m2m_model in many_to_many.items(): values = getattr(item, u'all_{}'.format(m2m)) fk_var_names = [] for value in values: fk_var_names.append(dump_dependencies_getters(fileds, m2m_model, value, foreign_keys_unique_fields[m2m], m2m_fields=m2m_can_be_created.get(m2m, None))) if fk_var_names: m2m_update[m2m] = u'[{}]'.format(u', '.join(fk_var_names)) for field in unique_fields: value = getattr(item, field) if value is not None: unique_data[field] = unicode(value) for field in fields: value = getattr(item, field) if value is not None: data[field] = unicode(value) fk_var_name = 'fk_{}_{}'.format(model.__name__, item.pk) print_to_files(fileds, u'try: {fk_var_name}\nexcept NameError:\n'.format(fk_var_name=fk_var_name)) lines_to_print_under_except = [ u'data = {data}'.format(data=data), ] if raw_data: lines_to_print_under_except.append( u'data.update({dumped_raw_data})'.format( dumped_raw_data=u'{open_b}{dumped_raw_data}{close_b}'.format( open_b=u'{', dumped_raw_data=u','.join([ u'"{key}": {value}'.format(key=key, value=value) for key, value in raw_data.items() ]), close_b=u'}', ), ) ) lines_to_print_under_except += [ u'try:', u' {fk_var_name} = {model_from}.{model_name}.objects.get({unique_data})'.format( fk_var_name=fk_var_name, model_from=model.__module__, model_name=model.__name__, unique_data=u' | '.join([ u'Q({key}={value})'.format(key=key, value=repr(value)) for key, value in unique_data.items() ]), ), u' {model_from}.{model_name}.objects.filter(pk={fk_var_name}.pk).update(**data)'.format( fk_var_name=fk_var_name, model_from=model.__module__, model_name=model.__name__, ), u' {fk_var_name} = {model_from}.{model_name}.objects.get(pk={fk_var_name}.pk)'.format( fk_var_name=fk_var_name, model_from=model.__module__, model_name=model.__name__, ), u' total_updated += 1', u'except ObjectDoesNotExist:', u' data.update({unique_data})'.format( unique_data=unique_data, ), u' {fk_var_name} = {model_from}.{model_name}.objects.create(**data)'.format( fk_var_name=fk_var_name, model_from=model.__module__, model_name=model.__name__, ), u' total_created += 1', ] if m2m_update: for key, value in m2m_update.items(): lines_to_print_under_except.append( u'{fk_var_name}.{key} = {value}'.format( fk_var_name=fk_var_name, key=key, value=value, ) ) lines_to_print_under_except.append( u'{fk_var_name}.save()'.format(fk_var_name=fk_var_name) ) print_to_files(fileds, u'{}\n\n'.format(u'\n'.join([ u' {}'.format(line) for line in lines_to_print_under_except ])))
def magiCirclesGeneratedSettings(existing_values): now = timezone.now() one_week_ago = now - datetime.timedelta(days=10) generated_settings = {} imports = [] ############################################################ # Get settings only when missing: # Get staff configurations and latest news staff_configurations = existing_values.get('STAFF_CONFIGURATIONS', None) latest_news = [] if not staff_configurations: staff_configurations, more_latest_news = getStaffConfigurations() latest_news += more_latest_news # Get favorite characters if FAVORITE_CHARACTERS_MODEL: queryset = FAVORITE_CHARACTERS_FILTER(FAVORITE_CHARACTERS_MODEL.objects.all()) # Cache if not existing_values.get('FAVORITE_CHARACTERS', None): generateCharactersSettings( queryset, generated_settings=generated_settings, imports=imports, ) # Get birthday banners if modelHasField(FAVORITE_CHARACTERS_MODEL, 'birthday'): if not hasNewsOfCategory(latest_news, 'characters_birthdays'): latest_news = getCharactersBirthdays( queryset, latest_news=latest_news, ) # Other characters if OTHER_CHARACTERS_MODELS: generated_settings['OTHER_CHARACTERS_KEYS'] = OTHER_CHARACTERS_MODELS.keys() for key, character_details in OTHER_CHARACTERS_MODELS.items(): if not isinstance(character_details, dict): character_details = { 'model': character_details } queryset = character_details.get('filter', lambda q: q)(character_details['model'].objects.all()) # Cache if not existing_values.get(key, None): generateCharactersSettings( queryset, generated_settings=generated_settings, imports=imports, for_favorite=False, base_name=key, ) # Get birthday banners if modelHasField(character_details['model'], 'birthday'): category = u'{}_birthdays'.format(key.lower()) if not hasNewsOfCategory(latest_news, category): latest_news = getCharactersBirthdays( queryset, category=category, latest_news=latest_news, ) # Get total donators if not existing_values.get('TOTAL_DONATORS', None): generated_settings['TOTAL_DONATORS'] = totalDonatorsThisMonth() or '\'\'' # Get latest donation month if not existing_values.get('DONATION_MONTH', None): generated_settings['DONATION_MONTH'] = latestDonationMonth(failsafe=True) # Get seasonal settings if not existing_values.get('SEASONAL_SETTINGS', None): generated_settings['SEASONAL_SETTINGS'] = seasonalGeneratedSettings(staff_configurations) # Add banners for seasonal hashtags if not hasNewsOfCategory(latest_news, 'seasonal_activity_tag'): latest_news = getSeasonalActivityTagBanners( seasonal_settings=existing_values.get('SEASONAL_SETTINGS', None) or generated_settings['SEASONAL_SETTINGS'], latest_news=latest_news, ) # Get users birthdays if not hasNewsOfCategory(latest_news, 'users_birthdays'): latest_news = getUsersBirthdaysToday( latest_news=latest_news, ) ############################################################ # Always get: # Generate share images once a week if django_settings.DEBUG: generated_share_images_last_date = now generated_share_images = {} else: generated_share_images_last_date = getattr(django_settings, 'GENERATED_SHARE_IMAGES_LAST_DATE', None) if (generated_share_images_last_date and generated_share_images_last_date.replace(tzinfo=pytz.utc) > one_week_ago): generated_share_images = getattr(django_settings, 'GENERATED_SHARE_IMAGES', {}) else: generated_share_images_last_date = now generated_share_images = {} print 'Generate auto share images' for collection_name, collection in getMagiCollections().items(): if collection.auto_share_image: generated_share_images[collection.name] = generateShareImageForMainCollections(collection) generated_settings['GENERATED_SHARE_IMAGES_LAST_DATE'] = 'datetime.datetime.fromtimestamp(' + unicode( time.mktime(generated_share_images_last_date.timetuple()) ) + ')' generated_settings['GENERATED_SHARE_IMAGES'] = generated_share_images # Get homepage arts if GET_HOMEPAGE_ARTS: generated_settings['HOMEPAGE_ARTS'] = GET_HOMEPAGE_ARTS() # Get backgrounds if GET_BACKGROUNDS: generated_settings['BACKGROUNDS'] = GET_BACKGROUNDS() # Get past tags count generated_settings['PAST_ACTIVITY_TAGS_COUNT'] = {} for tag_name, tag in ACTIVITY_TAGS.items(): if getEventStatus( tag.get('start_date', None), tag.get('end_date', None), without_year_return='ended', ) == 'ended': generated_settings['PAST_ACTIVITY_TAGS_COUNT'][tag_name] = models.Activity.objects.filter( c_tags__contains='"{}"'.format(tag_name)).count() ############################################################ # Save generated_settings.update({ 'STAFF_CONFIGURATIONS': staff_configurations, 'LATEST_NEWS': latest_news, }) return generated_settings, imports
def generateCharactersSettings( queryset, generated_settings, imports=None, for_favorite=True, base_name=None, to_image=None, ): imports.append('from collections import OrderedDict') if not base_name: if for_favorite: base_name = 'FAVORITE_CHARACTERS' else: base_name = camelToSnakeCase(queryset.model.__name__, upper=True) + u'S' print u'Get the {}'.format(snakeCaseToTitle(base_name)) generated_settings[base_name] = [] original_names = {} for character in queryset: name = unicode(character) if to_image: image = to_image(character) else: for image_field in [ 'image_for_favorite_character', 'top_image_list', 'top_image', 'image_thumbnail_url', 'image_url', ]: image = getattr(character, image_field, None) if image: break if image and name: original_names[character.pk] = name generated_settings[base_name].append(( character.pk, name, image, )) all_names = OrderedDict() for language, _verbose in django_settings.LANGUAGES: for character in queryset: if character.pk not in original_names: continue translation_activate(language) name = unicode(character) if name != original_names[character.pk]: if character.pk not in all_names: all_names[character.pk] = {} all_names[character.pk][language] = name translation_activate('en') generated_settings[u'{}_NAMES'.format(base_name)] = all_names if modelHasField(queryset.model, 'birthday'): generated_settings[u'{}_BIRTHDAYS'.format(base_name)] = OrderedDict([ (character.pk, (character.birthday.month, character.birthday.day)) for character in birthdayOrderingQueryset(queryset.exclude(birthday=None), order_by=True) if character.pk in original_names and getattr(character, 'birthday', None) ]) generated_settings[u'{}_BIRTHDAY_TODAY'.format(base_name)] = queryset.filter( birthdays_within(days_after=1, days_before=1)).values_list( 'pk', flat=True)
def __init__(self, *args, **kwargs): super(BaseEventCollection, self).__init__(*args, **kwargs) self.with_versions = modelHasField(self.queryset.model, 'c_versions') self.versions = self.queryset.model.VERSIONS if self.with_versions else {}
def save_item(details, unique_data, data, log_function=print, json_item=None, verbose=False, download_images=False, force_reload_images=False): model = details['model'] unique_together = details.get('unique_together', False) download_images = details.get('download_images', download_images) find_existing_item = details.get('find_existing_item', None) dont_erase_existing_value_fields = details.get( 'dont_erase_existing_value_fields', []) if (data or unique_data): unique_data = prepare_data(unique_data, model, unique=True, download_images=download_images) data, manytomany, dictionaries, images = prepare_data( data, model, unique=False, download_images=download_images, ) if verbose: log_function(model.__name__) log_function('- Unique data:') log_function(unique_data) log_function('- Data:') log_function(data) data.update(unique_data) if find_existing_item: item = find_existing_item(model, unique_data, data, manytomany, dictionaries) else: item = default_find_existing_item(model, unique_together, unique_data) if item: data = { k: v for k, v in data.items() if v or k not in dont_erase_existing_value_fields } model.objects.filter(pk=item.pk).update(**data) item = model.objects.filter(pk=item.pk)[0] if verbose: log_function(u'Updated {} #{}'.format(model.__name__, item.pk)) else: if modelHasField( model, 'owner' ) and 'owner' not in data and 'owner_id' not in data: data['owner'] = get_default_owner() item = model.objects.create(**data) log_function(u'Created {} #{}'.format(model.__name__, item.pk)) if manytomany: if verbose: log_function('- Many to many:') for field_name, list_of_items in manytomany.items(): getattr(item, field_name).add(*list_of_items) if verbose: log_function(' ', field_name) item.save() if dictionaries: if verbose: log_function('- Updated dictionaries:') for field_name, dictionary in dictionaries.items(): for k, v in dictionary.items(): item.add_d(field_name[2:], k, v) if verbose: log_function(' ', field_name, getattr(item, field_name[2:])) item.save() if images: saved_images = [] for field_name, url in images.items(): if not getattr(item, field_name, None) or force_reload_images: image = saveImageURLToModel(item, field_name, url) saved_images.append(field_name) if saved_images: if not verbose: log_function(u'{} #{}'.format(model.__name__, item.pk)) log_function(u'- Uploaded images: {}'.format( ', '.join(saved_images))) item.save() if 'callback_after_save' in details: details['callback_after_save'](details, item, json_item) return item return None