def build_post(self, user: User): # Поиск хештегов в тексте. raw_tags = re.findall("[^\\\]#[\w-]+", self.text) clear_tags = [] for raw_tag in raw_tags: tag = raw_tag[raw_tag.find("#") + 1:].lower() alias = transliterate.slugify(tag) if transliterate.detect_language(tag) else tag document_tag = domains.PostTag(title=tag, alias=alias) clear_tags.append(document_tag) meta_info = domains.PostMetaInfo( user=user, datetime_create=datetime.utcnow(), datetime_update=datetime.utcnow(), ) post = domains.Post( text=self.text, title=self.title, alias=self.alias, tags=clear_tags, meta_info=meta_info ) return post
def startSearchingEngine(): searchRequest = ui.lineEdit.text() data["searchRequest"] = searchRequest minPrice = ui.lineEdit_2.text() data["minPrice"] = minPrice maxPrice = ui.lineEdit_3.text() data["maxPrice"] = maxPrice searchRadius = ui.lineEdit_4.text() data["searchRadius"] = searchRadius numPages = ui.lineEdit_5.text() data["numPages"] = numPages registeredBeforeYear = ui.lineEdit_6.text() data["registeredBeforeYear"] = registeredBeforeYear registeredAfterYear = ui.lineEdit_7.text() data["registeredAfterYear"] = registeredAfterYear activeGoodMax = ui.lineEdit_8.text() data["activeGoodMax"] = activeGoodMax activeGoodsMin = ui.lineEdit_9.text() data["activeGoodsMin"] = activeGoodsMin viewsMax = ui.lineEdit_10.text() data["viewsMax"] = viewsMax viewsMin = ui.lineEdit_11.text() data["viewsMin"] = viewsMin adsPostedInThirtyDays = ui.lineEdit_12.text() data["adsPostedInThirtyDays"] = adsPostedInThirtyDays reserved = ui.lineEdit_13.text() data["reserved"] = reserved reserved_2 = ui.lineEdit_14.text() data["reserved_2"] = reserved_2 reserved_3 = ui.lineEdit_15.text() data["reserved_3"] = reserved_3 reserved_4 = ui.lineEdit_16.text() data["reserved_4"] = reserved_4 city = ui.comboBox.currentText() data["city"] = slugify(city).replace('j', 'y') category = ui.comboBox_2.currentText() data["category"] = slugify(category).replace("j", "y") print(data)
def save(self, *args, **kwargs): # print(bool(re.search('[а-яА-Я]', self.url_name))) # if bool(re.search('[а-яА-Я]', self.url_name)): self.url_name = transliterate.slugify(str(self.name)) # else: # self.url_name = self.url_name.replace(" ", "_") super(Tournament, self).save(*args, **kwargs)
def save_prom_image(src): src = src image = slugify(src.replace('https://ckl.com.ua/', '').replace( 'https://images.ua.prom.st/', ''), language_code='uk') + ".jpg" path = MEDIA_ROOT + '/%s/%s/' db_path = '%s/%s/%s' % (image[0], image[1], image) path = path % (image[0], image[1]) print(src) if not os.path.isdir(path): os.makedirs(path) path = path + image if not os.path.isfile(path): r = get(src) print(r.status_code) if r.status_code == 200: with open(path, 'wb') as f: for chunk in r.iter_content(): f.write(chunk) return db_path
def __test_29_slugify_hebrew(self): """ Testing slugify from Hebrew. """ res = slugify(self.hebrew_text) self.assertEqual(res, 'lorem-ipsum-dolor-sit-amet') return res
def test_30_slugify_bulgarian_cyrillic(self): """ Testing slugify from Bulgarian Cyrillic. """ res = slugify(self.bulgarian_cyrillic_text, language_code='bg') self.assertEqual(res, 'lorem-ipsum-dolor-sit-amet') return res
def test_26_slugify_armenian(self): """ Testing slugify from Armenian. """ res = slugify(self.armenian_text) self.assertEqual(res, 'lorem-ipsum-dolor-sit-amet') return res
def test_28_slugify_greek(self): """ Testing slugify from Greek. """ res = slugify(self.greek_text) self.assertEqual(res, 'lorem-ipsum-dolor-sit-amet') return res
def test_27_slugify_georgian(self): """ Testing slugify from Georgian. """ res = slugify(self.georgian_text) self.assertEqual(res, 'lorem-ipsum-dolor-sit-amet') return res
def create_cat(self, r): for attr in ProductAttribute.objects.all(): self.ATT_SLUGS[attr.slug] = attr print('create_cat', r) # id, image, parent_id, name_en, name_ru, descr_en, descr_ru if r[2] != r[0]: if int(r[2]) not in self.ICECAT_CATEGORIES: self.create_cat(self.ICECAT_CATEGORIES_SRC[int(r[2])]) parent = self.ICECAT_CATEGORIES[int(r[2])] else: parent = None slug = slug_base = slugify(r[3] or r[4], language_code='ru') counter = 1 while slug in self.CAT_SLUGS: slug = '{}-{}'.format(slug_base, counter) counter += 1 self.CAT_SLUGS[slug] = self.ICECAT_CATEGORIES[int( r[0])] = Category.objects.create(icecat_id=r[0], slug=slug, image_url=r[1], name_en=r[3], name=r[4] or r[3], descr_en=r[5], descr=r[6] or r[5], parent=parent) print('>', self.ICECAT_CATEGORIES[int(r[0])])
def get_upload_path(folder: str, filename: str) -> str: ''' Возвращает путь для сохранения объекта (файла, изображения) ''' file_name, file_extension = os.path.splitext(filename) return os.path.join('media', folder, strftime("%Y/%m/%d/"), slugify(file_name, 'ru') + file_extension)
def test_30_slugify_cyrillic(self): """ Testing slugify from Cyrillic. """ res = slugify(self.cyrillic_text) self.assertEqual(res, 'lorem-ipsum-dolor-sit-amet') return res
def test_30_slugify_ukrainian_cyrillic(self): """ Testing slugify from Ukrainian Cyrillic. """ res = slugify(self.ukrainian_cyrillic_text, language_code='uk') self.assertEqual(res, 'lorem-ipsum-dolor-sit-amet') return res
def handle_uploaded_file(cls, file, file_name: str): with open(cls.PATH_FOR_SAVING + slugify(file_name, 'uk') + '.xlsx', 'wb+') as destination: for chunk in file.chunks(): destination.write(chunk) file_path = cls.PATH_FOR_SAVING + file_name + '.xlsx' return file_path
def save_image(image): """Receives image, saves it and returns its url""" filename = image.name if _settings.TRANSLITERATE_FILENAME: try: from transliterate import slugify root, ext = os.path.splitext(filename) filename = '{}{}'.format(slugify(root), ext) except ImportError as e: logger.error(e) path = os.path.join(_settings.UPLOAD_PATH, filename) if _settings.THUMBNAIL_SIZE: im = Image.open(image) initial_size = im.size im.thumbnail(_settings.THUMBNAIL_SIZE) if im.size != initial_size: # if original is larger than thumbnail real_path = os.path.join(_settings.UPLOAD_PATH, 'thumb_%s' % filename) try: im.save(os.path.join(settings.MEDIA_ROOT, real_path), "JPEG") return default_storage.url(real_path) except IOError: print("cannot create thumbnail for", image) return real_path = default_storage.save(path, image) return default_storage.url(real_path)
def parse(folder: str): _files = [] for dir, _, files in os.walk(folder): _files.extend([os.path.join(dir, x) for x in files if x and x.endswith(".md")]) categories = [] for x in _files: with open(x, "r") as fio: head = [next(fio) for _ in range(4)] space = next(fio).strip() while not space: space = next(fio) categories.append(head[3].replace("Category: ", "").replace("-", " ").replace(" ", " ").strip()) categories = list(set(categories)) for x in categories: try: if not x: continue slug = translit(x.strip(), reversed=True).replace(" ", "-").lower() BlogCategory.objects.get_or_create(title=x, slug=slug) except LanguageDetectionError as e: print(e, x) for x in _files: with open(x, "r") as fio: head = [next(fio) for _ in range(4)] space = next(fio).strip() while not space: space = next(fio) raw_title = head[0].replace("Title: ", "").strip() slug = slugify(raw_title) date = datetime.datetime.strptime(head[1].replace("Date: ", "").strip(), "%Y-%m-%d %H:%M") tags = [x.strip() for x in head[2].replace("Tags: ", "").strip().split(",") if x and x.strip()] category = head[3].replace("Category: ", "").replace("-", " ").replace(" ", " ").strip() text = fio.read().replace("!embedlycard", "!embed") post, created = BlogPost.objects.get_or_create( title=raw_title, slug=slug, publish_date=date, status=CONTENT_STATUS_PUBLISHED, user=User.objects.get(username="******"), content=text, ) for tag in tags: keyword, _ = Keyword.objects.get_or_create(title=tag) post.keywords.add(AssignedKeyword(keyword=keyword)) post.categories.add(BlogCategory.objects.get_or_create(title=category)[0])
def parse_products(cursor_in, cursor_out): sql_select = ("select * from tmp_products_with_map") sql_insert = ("insert into products (product, manufacturer_id, " "category_id, product_dlink) values " "(%(product)s, %(manufacturer_id)s, %(category_id)s, " "%(product_dlink)s)") sql_insert_map = ("insert into iav_products_map(new_map_id, " "middle_map_id, old_map_id) values (%(new_map_id)s, " "%(middle_map_id)s, %(old_map_id)s)") try: unique = {} cursor_out.execute(sql_select) request = cursor_out.fetchall() for row in request: mid_id_, id_, pname_, mf_id_, cat_id_ = row insert_pattern = { "product": pname_, "manufacturer_id": mf_id_, "category_id": cat_id_, "product_dlink": slugify(pname_) } dlink = insert_pattern["product_dlink"] if unique.get(dlink, False): insert_pattern["product_dlink"] = f"{dlink}_{unique[dlink]}" unique[dlink] += 1 else: unique.setdefault(dlink, 1) try: cursor_out.execute(sql_insert, insert_pattern) except connector.IntegrityError as inerr: if op.eq(inerr.errno, 1062): insert_pattern["product"] = f"{pname_}_{id_}" cursor_out.execute(sql_insert, insert_pattern) else: print(inerr) except connector.Error as err: print(err) lri = cursor_out.lastrowid insert_map = { "new_map_id": lri, "middle_map_id": mid_id_, "old_map_id": id_ } try: cursor_out.execute(sql_insert_map, insert_map) except connector.Error as err: print("map", lri, err) except Exception as e: raise e
def get_upload_path(folder: str, filename: str) -> str: ''' **DEPRECATED** надо исползовать functions.get_upload_path ''' file_name, file_extension = os.path.splitext(filename) return os.path.join('media', folder, strftime("%Y/%m/%d/"), slugify(file_name, 'ru') + file_extension)
def export_files(is_transliterate=True): si = pd.read_csv('si_rec.csv', sep='\t') si = dispersing_clusters(si, 'display_lon', 'display_lat') nap = pd.read_csv('nap_rec.csv', sep='\t') nap = dispersing_clusters(nap, 'lon', 'lat') si = si[[ 'area', 'place', 'court_short', 'price', 'type', 'display_lon', 'display_lat', 'is_appr_coor', 'period_start', 'period_end', 'link', 'ideal_parts' ]] si.rename(columns={ 'place': 'address', 'display_lon': 'lon', 'display_lat': 'lat', 'court_short': 'region' }, inplace=True) si['source'] = 'https://sales.bcpea.org/' nap = nap[[ 'area', 'address', 'region', 'price', 'type', 'lon', 'lat', 'is_appr_coor', 'period_start', 'period_end', 'link', 'ideal_parts' ]] nap['source'] = 'https://sales.nra.bg' export = pd.concat([si, nap]) export['price'] = export['price'].astype(int) export = export[~pd.isnull(export['lon'])] print("len before date filtering is {}".format(export.shape[0])) date_filter = str(datetime.datetime.now().date()) export = export[export['period_end'].apply(lambda x: x > date_filter)] print("len before date filtering is {}".format(export.shape[0])) if is_transliterate: export['address'] = export['address'].apply( lambda x: slugify(x) if str(x) not in ['nan', ''] else x) export['region'] = export['region'].apply( lambda x: slugify(x) if str(x) not in ['nan', ''] else x) export['type'] = export['type'].apply( lambda x: slugify(x) if str(x) not in ['nan', ''] else x) export.to_csv('export_tr.tsv', sep='\t', index=False) else: export.to_csv('export.tsv', sep='\t', index=False)
def from_json(self, json_data): for key, value in json_data.items(): try: setattr(self, key, value) if key == 'title': value = slugify(u'{}'.format(value)) or value setattr(self, 'key', value) except AttributeError: continue return self
def list_files(self): device_category = Category.objects.filter(name='Устройства').first() for product in Product.objects.exclude(part_num=None).all(): brand = str(product.brand).upper() partno = str(product.part_num).upper() filename = slugify(brand + '__' + partno, 'ru') + '.xml' if not os.path.exists( os.path.join('icecat_download/by_partno/', filename)): continue print(1, product.id, filename, sep=';') PARTS = set() for product in Product.objects.filter(part_num=None).all(): brand = str(product.brand).upper() for name_part in product.name.split(): partno = name_part.upper() if partno in PARTS: continue PARTS.add(partno) filename = slugify(brand + '__' + partno, 'ru') + '.xml' if not os.path.exists( os.path.join('icecat_download/by_name/', filename)): continue print(2, product.id, filename, sep=';') PARTS = set() for product in Product.objects.exclude(part_num=None).\ filter(category__in=device_category.get_descendants(include_self=True)).all(): brand = str(product.brand).upper() partno = str(product.part_num).upper() filename = slugify(brand + '__' + partno, 'ru') + '.xml' for name_part in product.name.replace(',', ' ').split(): partno = name_part.upper() if partno in PARTS: continue PARTS.add(partno) filename = slugify(brand + '__' + partno, 'ru') + '.xml' if not os.path.exists( os.path.join('icecat_download/by_name2/', filename)): continue print(3, product.id, filename, sep=';')
def create_brand(self, r): if r[2]: self.ICECAT_BRANDS[r[0]] = Brand.objects.filter(name=r[2]).first() assert self.ICECAT_BRANDS[r[0]] self.ICECAT_BRANDS[r[0]].icecat_id = r[0] self.ICECAT_BRANDS[r[0]].save() else: self.ICECAT_BRANDS[r[0]] = Brand.objects.create( name=r[1], icecat_id=r[0], slug=slugify(r[1], language_code='ru'))
def save_file(dest_path, f, filename=False): original_name, file_extension = os.path.splitext(f.name) filename_postfix_inc = 1 def_filename = False while True: if filename is not False: new_filename = u'%s' % filename else: new_filename = u'%s' % original_name if def_filename is False: def_filename = new_filename if dest_path is False: try: url = slugify(new_filename) + file_extension except: url = new_filename + file_extension path = os.path.join(settings.MEDIA_ROOT, url) else: url = dest_path + '/' + slugify(new_filename) + file_extension path = os.path.join(settings.MEDIA_ROOT, url) if os.path.isfile(path) == False: break if os.path.isfile(path): filename_postfix_inc += 1 filename = '%s-%s' % (def_filename, str(filename_postfix_inc)) if dest_path == False: mkdir_recursive(settings.MEDIA_ROOT) else: mkdir_recursive(os.path.join(settings.MEDIA_ROOT, dest_path)) destination = open(path, 'wb+') for chunk in f.chunks(): destination.write(chunk) destination.close() return url
def textprepairer(text): #func that prepare text from db to format we need try: if '«' in text: temp = re.findall(r'«(.*)»', text) a1 = temp[0].lower() print(a1) if slugify(a1) == None: a2=a1 else: a2=slugify(a1).replace('-', ' ') return [a1, a2] else: a1 = text.lower() if slugify(a1) == None: a2=a1 else: a2=slugify(a1).replace('-', ' ') return [a1, a2] except AttributeError: print("Can't handle " + text) return ["undefined undefined", "undefined-undefined"]
def save(self, *args, **kwargs): if not self.slug: slug = slugify(self.name, 'ru') self.slug = slug while True: qs = self.__class__.objects.filter(slug=self.slug) if self.id: qs.exclude(id=self.id) if qs.first(): self.slug = slug + str(randint(0, 1000000000)) continue break super(AbstractTag, self).save(*args, **kwargs)
def transliterate(text): # pieces = str(re.sub('[\W]+]', ' ', text)).lower().split(' ') # result = [] # for piece in pieces: # try: # result.append(translit(piece, reversed=True)) # except LanguageDetectionError: # return my_str # return '-'.join([r for r in result if r]) my_str = slugify(text) return my_str
def create_or_update_company( cid: int, name: str, is_ad_agency: bool = False, is_agency: bool = False, is_builder: bool = False) -> None: defaults = {'name': name, 'is_ad_agency': is_ad_agency, 'is_agency': is_agency, 'is_builder': is_builder} c, created = Company.objects.update_or_create(id=cid, defaults=defaults) if created: slugged_name = slugify(name, 'ru') user = get_user_model().objects.create_user(slugged_name, None, get_random_string(0)) user.companies.add(c) create_user_profile(user, name)
def send_mail(fromaddr, toaddr, password, files): msg = MIMEMultipart() msg['From'] = fromaddr msg['To'] = toaddr msg['Subject'] = "FILES" body = "time {}".format(datetime.now()) msg.attach(MIMEText(body, 'plain')) print(files) # cycle adds attachment to mail for file in files: attachment = open(file, "rb") part = MIMEBase('application', 'octet-stream') part['Content-Type'] = "text/plain; charset=utf-8" part.set_payload((attachment).read()) encoders.encode_base64(part) if slugify(file) != None: file = slugify(file) part.add_header('Content-Disposition', "attachment; filename={}".format(file)) msg.attach(part) server = smtplib.SMTP('smtp.gmail.com', 587) server.starttls() server.login(fromaddr, password) text = msg.as_string() server.sendmail(fromaddr, toaddr, text) server.quit() print("Files were sent")
def save(self, *args, **kwargs): if not self.slug: if self and self.slug_base: slug = slugify(getattr(self, self.slug_base), 'ru')[:190] self.slug = slug while True: qs = self.__class__.objects.filter(slug=self.slug) if self.id: qs.exclude(id=self.id) if qs.exists(): self.slug = slug + '_' + str(randint(0, 999999999)) continue break else: self.slug = str(time()) + '_' + str(randint(0, 999999999)) super().save(*args, **kwargs)
def add_post(self, title, content, slug=None, labels=None): self.check_blogid() if slug is None: slug = transliterate.slugify(title) req = self._client.posts().insert( blogId=self._blogid, body={ "title": slug, "content": content, "labels": list(labels) if labels else []} ) rep = req.execute() if slug != title: req = self._client.posts().patch(blogId=self._blogid, postId=rep['id'], body={"title": title}) rep = req.execute() return Post(rep)
def create_place(json_url): json = get_json(json_url) image_urls = json['imgs'] place_object, is_created = Place.objects.get_or_create( title=json['title'], slug=slugify(json['title']), description_short=json['description_short'], description_long=json['description_long'], latitude=json['coordinates']['lat'], longitude=json['coordinates']['lng'], ) if not is_created: return for image in get_raw_image(image_urls): name, content = image image_object = Image.objects.create(place=place_object) image_object.image.save(name, ContentFile(content), save=True)
def save_image(self,src,type): href = src if type == 1: src = src.replace('https://pp.userapi.com/','').replace('/','').replace('-','') elif type == 2: src = src.replace('https://platform-lookaside.fbsbx.com/','').replace('/','').replace('-','') path = self.path % (MEDIA_ROOT,src[0:2]) if not os.path.isdir(path): os.mkdir(path) os.chmod(path,0o777) image = slugify(src,language_code='uk') + ".jpg" path = path + image if not os.path.isfile(path): r = get(href) image = "users/image/%s/%s" % (src[0:2],image) with open(path, 'wb') as f: for chunk in r.iter_content(): f.write(chunk) return image
def create_attrubute(self, r): assert r[2] in self.ICECAT_MEASURES assert r[3] in self.ICECAT_CAT_FEGR_RELATIONS slug = slug_base = slugify(r[1], language_code='ru') counter = 1 while slug in self.ATT_SLUGS: slug = '{}-{}'.format(slug_base, counter) counter += 1 print('create_attrubute: ', slug) self.ATT_SLUGS[slug] = self.ICECAT_FEATURES[ r[0]] = ProductAttribute.objects.create( name=r[1], icecat_id=r[0], unit=self.ICECAT_MEASURES[r[2]], type=Attribute.TYPE_STRING, slug=slug) self.ICECAT_FEATURES[r[0]]._relation = r[2] icecat_rel = self.ICECAT_CAT_FEGR_RELATIONS[r[3]] icecat_fegrid = icecat_rel[2] #print(self.ICECAT_FEGR_CAT[icecat_fegrid]) relations = [] for icecat_rel in self.ICECAT_FEGR_CAT[icecat_fegrid]: relation = CategoryToProductAttributeRelation( category=self.ICECAT_CATEGORIES[icecat_rel[1]], attribute=self.ICECAT_FEATURES[r[0]], group=self.ICECAT_FEATURE_GROUPS[icecat_rel[2]], use_in_filters=False, include_in_full_text_index=True, show_at_product_page=True, main_attribute=False, main_relation_attribute=False, ) relations.append(relation) #print('>', relation) CategoryToProductAttributeRelation.objects.bulk_create(relations) print('>', self.ICECAT_FEATURES[r[0]])
def download_file(request): try: filename = request.GET['filename'] with open(StudentUploader.PATH_FOR_SAVING + filename, "rb") as f: current_file = f.read() response = HttpResponse( content=current_file, content_type= 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet' ) response['Content-Disposition'] = 'attachment; filename={}'.format( slugify(filename, )) return response except Exception as e: print('[ERROR]{}'.format(e)) form = UploadFileForm() return render( request, 'student_uploader/index.html', { 'form': form, 'message': '', 'directory': os.listdir(StudentUploader.PATH_FOR_SAVING) })
def handle(self, *args, **options): url = options["url"] response = requests.get(url) place_raw = response.json() response.raise_for_status() place_, created = Place.objects.get_or_create( title=place_raw["title"], defaults={ 'placeid': slugify(place_raw["title"]).replace("-", "_"), 'description_long': place_raw["description_long"], 'description_short': place_raw["description_short"], } ) Coordinates.objects.get_or_create( lng=place_raw["coordinates"]["lng"], lat=place_raw["coordinates"]["lat"], place=place_, ) for index, img_ in enumerate(place_raw["imgs"], 1): filename = Path(urlparse(img_).path).name filepath = path.join(settings.MEDIA_ROOT, filename) response = requests.get(img_, timeout=2.5) response.raise_for_status() with open(filepath, 'wb') as f: f.write(response.content) Image.objects.get_or_create( post=place_, position=index, defaults={ 'img': filename, } )
def perform_create(self, serializer): """Adding some extra data to created object.""" _lang = get_language()[:2] slug = slugify(self.request.data.get('title'), _lang) serializer.save(publisher=self.request.user, slug=slug)