def translit(new):
lang = transliterate.detect_language(new)
if lang:
new = transliterate.translit(new, reversed = True)
new = cleaned_up_filename = re.sub(r"[\/\\\:\*\?\"\<\>\|]", "", new)
new = new.strip()
return new
def test_25_false_language_detection_cyrillic(self):
"""
Testing language detection. Detecting is not Russian (Cyrillic).
"""
res = detect_language(self.latin_text)
self.assertNotEqual(res, 'ru')
return res
async def handle_message(self, message):
# ignore private messages
if message.channel.is_private:
return
# do not check bot's own messages
if message.author.id == self.bot.user.id:
return
script = transliterate.detect_language(message.clean_content)
if script not in self.settings['scripts'].keys():
return
action = await self.get_action(script, message.channel.name)
if action == 'keep':
return
elif action == 'warn':
await self.bot.send_message(
message.channel, self.settings['warn'].format(script))
elif action == 'delete':
await self.bot.delete_message(message)
else:
raise ValueError
def build_post(self, user: User):
# Поиск хештегов в тексте.
raw_tags = re.findall("[^\\\]#[\w-]+", self.text)
clear_tags = []
for raw_tag in raw_tags:
tag = raw_tag[raw_tag.find("#") + 1:].lower()
alias = transliterate.slugify(tag) if transliterate.detect_language(tag) else tag
document_tag = domains.PostTag(title=tag, alias=alias)
clear_tags.append(document_tag)
meta_info = domains.PostMetaInfo(
user=user,
datetime_create=datetime.utcnow(),
datetime_update=datetime.utcnow(),
)
post = domains.Post(
text=self.text,
title=self.title,
alias=self.alias,
tags=clear_tags,
meta_info=meta_info
)
return post
def test_25_language_detection_cyrillic(self):
"""
Testing language detection. Detecting Russian (Cyrillic).
"""
res = detect_language(self.cyrillic_text)
self.assertEqual(res, 'ru')
return res
def pre_save_post(sender, instance, *args, **kwargs):
if instance.slug:
slug_init = instance.slug
else:
if detect_language(instance.title) == "ru":
slug_init = slugify(translit(instance.title, reversed=True))
else:
slug_init = slugify(instance.title)
slug = slug_init
exists = sender.objects.filter(slug=slug).exclude(id=instance.id).exists()
i = 0
while exists:
i += 1
slug = "%s-%s" % (slug_init, i)
exists = sender.objects.filter(slug=slug).exists()
instance.slug = slug
def save(self, *args, **kwargs):
a = '{} {} {}'.format(self.last_name, self.first_name, self.patronymic)
if detect_language(self.last_name) is not None or detect_language(
self.first_name) is not None or detect_language(
self.patronymic) is not None:
self.slug = slugify(translit(a, reversed=True), allow_unicode=True)
else:
self.slug = slugify(a)
super(Participant, self).save(*args, **kwargs)
def save(self, *args, **kwargs):
a = '{} {}'.format(str(self.dance), self.name)
if detect_language(self.name) is not None and detect_language(
str(self.dance)) is not None:
self.slug = slugify(translit(a, reversed=True), allow_unicode=True)
else:
self.slug = slugify(a)
super(Costume, self).save(*args, **kwargs)
def save(self, *args, **kwargs):
# if not self.id:
a = '{} {} {}'.format(self.last_name, self.first_name, self.patronymic)
if detect_language(self.last_name) is not None or detect_language(
self.first_name) is not None or detect_language(
self.patronymic) is not None:
self.slug = slugify(translit(a, reversed=True), allow_unicode=True)
else:
self.slug = slugify(a, allow_unicode=True)
super(Choreographer, self).save(*args, **kwargs)
def save(self, *args, **kwargs):
a = '{} {} {}'.format(str(self.participant), str(self.month),
str(self.participant.group.payment))
if detect_language(str(
self.participant)) is not None or detect_language(
str(self.month)) is not None:
self.slug = slugify(translit(a, reversed=True), allow_unicode=True)
else:
self.slug = slugify(a)
super(Payment, self).save(*args, **kwargs)
def zip(exp):
zname = path + '/f.zip'
newzip = zipfile.ZipFile(zname, 'w') #создаем архив
for file in os.listdir(path):
if file.endswith("." + exp):
fname = file.split('.')[0]
tr = transliterate.detect_language(fname)
if (tr == 'ru'):
new = transliterate.translit(fname, reversed=True)
os.rename(file, new + '.' + exp)
for file in os.listdir(path):
if file.endswith("." + exp):
print(file)
newzip.write(file)
if (os.stat("f.zip").st_size == 0):
print(
"В данной папке не существует такого расширения!\nПопробуйте еще раз!"
)
main()
else:
print("Архив успешно создан!")
newzip.close()
def __test_25_language_detection_mongolian_cyrillic(self):
"""
Testing language detection. Detecting Mongolian (Cyrillic).
"""
res = detect_language(self.mongolian_cyrillic_text)
self.assertEqual(res, 'mn')
return res
def save(self, *args, **kwargs):
if self.date_until and self.participant:
a = '{} {} {} {}'.format(self.date, self.date_until,
self.participant, self.title)
elif self.date_until:
a = '{} {} {}'.format(self.date, self.date_until, self.title)
elif self.participant:
a = '{} {} {}'.format(self.date, self.participant, self.title)
else:
a = '{} {}'.format(self.date, self.title)
if detect_language(str(self.title)) is not None or detect_language(
str(self.participant)) is not None:
self.slug = slugify(translit(a, reversed=True), allow_unicode=True)
else:
self.slug = slugify(a)
super(Announcement, self).save(*args, **kwargs)
def __test_25_language_detection_ukrainian_cyrillic(self):
"""
Testing language detection. Detecting Ukrainian (Cyrillic).
"""
res = detect_language(self.ukrainian_cyrillic_text)
self.assertEqual(res, 'uk')
return res
def test_21_language_detection_armenian(self):
"""
Testing language detection. Detecting Amenian.
"""
res = detect_language(self.armenian_text)
self.assertEqual(res, 'hy')
return res
def test_22_language_detection_georgian(self):
"""
Testing language detection. Detecting Georgian.
"""
res = detect_language(self.georgian_text)
self.assertEqual(res, 'ka')
return res
def test_22_language_detection_georgian(self):
"""
Testing language detection. Detecting Georgian.
"""
res = detect_language(self.georgian_text)
self.assertEqual(res, 'ka')
return res
def __test_24_language_detection_hebrew(self):
"""
Testing language detection. Detecting Hebrew.
"""
res = detect_language(self.hebrew_text)
self.assertEqual(res, 'he')
return res
def test_21_language_detection_armenian(self):
"""
Testing language detection. Detecting Amenian.
"""
res = detect_language(self.armenian_text)
self.assertEqual(res, 'hy')
return res
def test_25_false_language_detection_cyrillic(self):
"""
Testing language detection. Detecting is not Russian (Cyrillic).
"""
res = detect_language(self.latin_text)
self.assertNotEqual(res, 'ru')
return res
def __test_25_language_detection_bulgarian_cyrillic(self):
"""
Testing language detection. Detecting Bulgarian (Cyrillic).
"""
res = detect_language(self.bulgarian_cyrillic_text)
self.assertEqual(res, 'bg')
return res
def __test_25_language_detection_ukrainian_cyrillic(self):
"""
Testing language detection. Detecting Ukrainian (Cyrillic).
"""
res = detect_language(self.ukrainian_cyrillic_text)
self.assertEqual(res, 'uk')
return res
def create_page():
title = request.form['title']
description = request.form['description']
tags = request.form['tags']
content = request.form['content']
article_name = translit(title,
reversed=True) if detect_language(title) else title
hashed_dirname = f'{article_name.lower()}-{uuid.uuid4().hex[:8]}'
dirname = hashed_dirname.replace(' ', '-')
article_dir = path.join(articles_dir, dirname)
os.mkdir(article_dir)
with open(path.join(article_dir, 'content.md'), 'w') as file:
file.write(content)
with open(path.join(article_dir, 'settings.yml'), 'w') as file:
settings = {
'title': title,
'description': description,
'created_date':
os.path.getctime(path.join(article_dir, 'content.md')),
'tags': list(map(lambda tag: tag.strip(), tags.split(',')))
}
yaml.dump(settings,
stream=file,
Dumper=yaml.RoundTripDumper,
allow_unicode=True)
# git_push(branch_name=dirname, commit_message=article_name)
# rm_tree(article_dir)
return redirect(f'/articles/{dirname}')
def document_file_directory_path(instance, filename):
if transliterate.detect_language(filename)=='ru':
name, expansion = filename.split('.')
file_name = transliterate.translit(name, reversed=True) + "." + expansion
else:
file_name = filename
if instance.category=="1":
path = 'files/bs_docs/' + str(datetime.datetime.now().year) + '/' + str('{:02d}'.format(datetime.datetime.now().month)) + '/' + '%s' % file_name
return path
elif instance.category=="2":
path = 'files/comit_docs/' + '%s' % file_name
return path
elif instance.category=="3":
path = 'files/comitet/audit/' + '%s' % file_name
return path
elif instance.category=="4":
path = 'files/comitet/strategy/' + '%s' % file_name
return path
elif instance.category=="5":
path = 'files/comitet/ethics/' + '%s' % file_name
return path
elif instance.category=="9":
path = 'files/bs_docs/' + str(datetime.datetime.now().year) + '/' + str('{:02d}'.format(datetime.datetime.now().month)) + '/' + '%s' % file_name
return path
elif instance.category=="11":
path = 'files/listing_docs/' + '%s' % file_name
return path
else:
path = 'files/' + '%s' % file_name
return path
def check(level, answer):
global answers
answer = answer.lower()
if detect_language(answer) == 'ru':
answer = translit(answer, 'ru', reversed=True)
return answer in answers[level]
def save(self, *args, **kwargs):
if detect_language(self.dance_style) is not None:
self.slug = slugify(translit(self.dance_style, reversed=True),
allow_unicode=True)
else:
self.slug = slugify(self.dance_style)
super(DanceStyle, self).save(*args, **kwargs)
def save(self, *args, **kwargs):
a = '{} {}'.format(self.name, self.sum)
if detect_language(str(self.name)) is not None:
self.slug = slugify(translit(a, reversed=True), allow_unicode=True)
else:
self.slug = slugify(a)
super(OtherSourceOfFinances, self).save(*args, **kwargs)
def __test_25_language_detection_bulgarian_cyrillic(self):
"""
Testing language detection. Detecting Bulgarian (Cyrillic).
"""
res = detect_language(self.bulgarian_cyrillic_text)
self.assertEqual(res, 'bg')
return res
def __test_24_language_detection_hebrew(self):
"""
Testing language detection. Detecting Hebrew.
"""
res = detect_language(self.hebrew_text)
self.assertEqual(res, 'he')
return res
def test_25_language_detection_cyrillic(self):
"""
Testing language detection. Detecting Russian (Cyrillic).
"""
res = detect_language(self.cyrillic_text)
self.assertEqual(res, 'ru')
return res
def __test_25_language_detection_mongolian_cyrillic(self):
"""
Testing language detection. Detecting Mongolian (Cyrillic).
"""
res = detect_language(self.mongolian_cyrillic_text)
self.assertEqual(res, 'mn')
return res
def product_task(self, product_ids):
"""Product task"""
try:
products = Product.objects.filter(id__in=product_ids)
products.update(status=Product.STATUS_CHOICE_PROGRESS)
products_items = [{
'id': product.id,
'link': product.link
} for product in products]
product_parser = ProductParser()
items = product_parser.get_products(products_items=products_items)
# items: [{'id':1 , 'price':23412, ..}, {'id':2 , 'price':2342, ..}, ..]
for item in items:
product = Product.objects.filter(id=item.get('id')).first()
status = item.get('status')
if product and status == Product.STATUS_CHOICE_DONE:
name_url = item.get('name_url', '')[:255]
attributes = item.get('attributes')
product.name = item.get('name', '')[:255]
product.manufacturer = item.get('manufacturer', '')[:255]
product.name_url = name_url
product.price = item.get('price')
product.front_picture = item.get('front_picture')
product.back_picture = item.get('back_picture')
product.description_text = item.get('description_text')
product.description_html = item.get('description_html')
product.attributes = attributes
name_url_cleaned = re.sub(r'(\-\d{4})$', '', name_url)
color_value = attributes.get('color', '')
color_value_cleaned = re.sub(r'[\-\/\s]', '-',
color_value.lower())
lang = detect_language(color_value_cleaned)
color_value_translated = translit(color_value_cleaned,
'ru',
reversed=lang)
if color_value_translated:
name_url_color = f'{name_url_cleaned}-{color_value_translated}'
else:
name_url_color = f'{name_url_cleaned}'
product.name_url_color = name_url_color
product.status = Product.STATUS_CHOICE_DONE
product.save()
elif status == Product.STATUS_CHOICE_ERROR:
product.name = item.get('name', '')[:255]
product.status = Product.STATUS_CHOICE_ERROR
product.save()
return len(
list(
filter(lambda i: i.get('status') == Product.STATUS_CHOICE_DONE,
items)))
except (ConnectionError, WebDriverException):
self.retry(countdown=10)
def save(self, *args, **kwargs):
if detect_language(self.name) is not None:
self.slug = slugify(translit(self.name, reversed=True),
allow_unicode=True)
else:
self.slug = slugify(self.name)
super(PhotoGallery, self).save(*args, **kwargs)
def save(self, *args, **kwargs):
a = '{} {}'.format(str(self.participant), self.sum)
if detect_language(str(self.participant)) is not None:
self.slug = slugify(translit(a, reversed=True), allow_unicode=True)
else:
self.slug = slugify(a)
super(Fond, self).save(*args, **kwargs)
def test_23_language_detection_greek(self):
"""Test language detection.
Detecting Greek.
"""
res = detect_language(self.greek_text)
self.assertEqual(res, 'el')
return res
def save(self, *args, **kwargs):
a = '{} {} {}'.format(self.name, self.begin_date, self.begin_time)
if detect_language(self.name) is not None:
self.slug = slugify(translit(a, reversed=True), allow_unicode=True)
else:
self.slug = slugify(a)
super(Concert, self).save(*args, **kwargs)
def save(self, *args, **kwargs):
a = '{}{}'.format(self.title, str(self.date))
if detect_language(self.title) is not None:
self.slug = slugify(translit(a, reversed=True), allow_unicode=True)
else:
self.slug = slugify(a)
super(News, self).save(*args, **kwargs)
def test_23_language_detection_greek(self):
"""
Testing language detection. Detecting Greek.
"""
#set_setting('DEBUG', True)
res = detect_language(self.greek_text)
#reset_to_defaults_settings()
self.assertEqual(res, 'el')
return res
def test_23_language_detection_greek(self):
"""
Testing language detection. Detecting Greek.
"""
#set_setting('DEBUG', True)
res = detect_language(self.greek_text)
#reset_to_defaults_settings()
self.assertEqual(res, 'el')
return res
def form_valid(self, form):
edited_event = form.save(commit=False)
edited_event.save()
data = dict()
data['event_id'] = edited_event.pk # Инициализируем pk события
# Загрузить изображения в указанную директорию
directory = EVENT_IMAGE_STORAGE + 'events/' + str(
edited_event.event_date.year) + '-' + edited_event.event_shortname
images = self.request.FILES.getlist('image_field')
for image in images:
if not os.path.exists(directory):
os.makedirs(directory)
filename, expansion = image.name.split('.')
if transliterate.detect_language(filename) == 'ru':
if os.path.exists(
directory + '/' +
transliterate.translit(filename, reversed=True) + '.' +
expansion):
with open(
directory + '/' +
transliterate.translit(filename, reversed=True) +
str(uuid.uuid4().hex) + '.' + expansion,
'wb+') as destination:
for chunk in image.chunks():
destination.write(chunk)
else:
with open(
directory + '/' +
transliterate.translit(filename, reversed=True) +
'.' + expansion, 'wb+') as destination:
for chunk in image.chunks():
destination.write(chunk)
else:
if os.path.exists(directory + '/' + image.name):
with open(
directory + '/' + filename +
str(uuid.uuid4().hex) + '.' + expansion,
'wb+') as destination:
for chunk in image.chunks():
destination.write(chunk)
else:
with open(directory + '/' + image.name,
'wb+') as destination:
for chunk in image.chunks():
destination.write(chunk)
# HTML редактируемого события
data['html_event'] = render_to_string(
'events/event.html', {
'event': edited_event,
'perms': PermWrapper(self.request.user),
'current_language': self.kwargs.get('language'),
'current_year': self.kwargs.get('year'),
},
request=self.request)
data['form_is_valid'] = True
return JsonResponse(data)
def transliterateLatToCyr(text):
# if text is already cyrillic return original text else return transliterated latin to cyrilic script
scriptLanguage = detect_language(text)
if(scriptLanguage in ['ru', 'sr', 'uk']):
return text
else:
transliteratedText = translit(text, 'sr')
return transliteratedText
def save(self, *args, **kwargs):
a = '{} {} {} {} {} {}'.format(str(self.group), str(self.begin_time),
str(self.end_time),
str(self.day_of_the_week),
str(self.dance_style),
str(self.choreographer))
if detect_language(a) is not None:
self.slug = slugify(translit(a, reversed=True), allow_unicode=True)
else:
self.slug = slugify(a)
super(GroupChoreographerSchedule, self).save(*args, **kwargs)
def transliterate_name(self):
"""Transliterate this problem's name to English"""
try:
from transliterate import detect_language, translit
import unicodedata
import re
except ImportError:
return 'problem' + str(self.id)
language_code = detect_language(self.name)
trans = self.name
if language_code:
trans = translit(self.name, language_code, reversed=True)
slug = unicodedata.normalize('NFKD', trans) \
.encode('ascii', 'ignore') \
.decode('ascii')
slug = re.sub(r'[^\w\s-]', '', slug).strip().lower()
return re.sub(r'[-\s]+', '-', slug) or ('problem' + str(self.id))
