def test_safe_chars(self):
slugify = Slugify()
slugify.safe_chars = '_'
self.assertEqual(slugify('test_sanitize'), 'test_sanitize')
slugify.safe_chars = "'"
self.assertEqual(slugify('Конь-Огонь'), "Kon'-Ogon'")
def slugify_name(name):
"""Get a slugifier used to ensure asset names are safe"""
# Create the slugifier
slugifier = Slugify()
# Configure the slugifier
slugifier.to_lower = True
slugifier.safe_chars = '-/'
slugifier.max_length = 200
# Names cannot start or end with forward slashes '/'
return slugifier(name).strip('/')
def test_stop_words(self):
slugify = Slugify(stop_words=['a', 'the'])
self.assertEqual(slugify('A red apple'), 'red-apple')
self.assertEqual(slugify('The4 red apple'), 'The4-red-apple')
self.assertEqual(slugify('_The_red_the-apple'), 'red-apple')
self.assertEqual(slugify('The__red_apple'), 'red-apple')
slugify.safe_chars = '*'
self.assertEqual(slugify('*The*red*apple'), '*-*red*apple')
self.assertEqual(slugify('The**red*apple'), '**red*apple')
slugify.stop_words = ['x', 'y']
self.assertEqual(slugify('x y n'), 'n')
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import os
from urllib.parse import urlparse, parse_qs
from itertools import product
from slugify import Slugify
custom_slugify = Slugify(to_lower=True)
custom_slugify.safe_chars = '_'
HEADERS = {
'user-agent': ('Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) '
'AppleWebKit/537.36 (KHTML, like Gecko) '
'Chrome/45.0.2454.101 Safari/537.36'),
}
def _state_enrollment_url_list(output_dir):
"""One off method for dealing with non-standard format of state-level enrollment data"""
var_map = {
'1': 'grade-by-gender',
'2': 'race_ethnicity-by-gender',
'3': 'race_ethnicity-by-special-education',
'4': 'race_ethnicity-by-ell',
import time
import urllib
import traceback
import threading
import json
import shutil
import errno
import requests
from requests.adapters import HTTPAdapter
from requests.packages.urllib3.util.retry import Retry
try:
from slugify import Slugify
except ImportError:
from octoprint.vendor.awesome_slugify import Slugify
_SLUGIFY = Slugify()
_SLUGIFY.safe_chars = "-_.()[] "
# create the module level logger
from octoprint_octolapse.log import LoggingConfigurator
logging_configurator = LoggingConfigurator()
logger = logging_configurator.get_logger(__name__)
from threading import Timer
FLOAT_MATH_EQUALITY_RANGE = 0.0000001
def get_float(value, default):
if value is None:
return default
try:
return float(value)
except ValueError:
import re
import math
def getSoupFromUrl(url):
result = requests.get(url)
if result.status_code == 200:
print 'Request successful'
return BeautifulSoup(result.text, "html.parser")
else:
print 'Request failed', url
return "Failed"
slugify_normalize = Slugify()
slugify_normalize.safe_chars = ',./'
def normalize(s):
return slugify_normalize(s, separator=' ')
url = 'http://courses.monoprix.fr'
result = pd.DataFrame(columns=[
"enseigne", "url", "nom_produit", "marque", "quantite", "poids_volume",
"poids_volume_total", "unite", "descriptif", "ingredients", "conservation",
"valeur_energetique", "origine", "prix", "prix_au_poids"
])
result = result.fillna(0)
# file to write url that did'nt work properly
def model_slug(self):
custom_slugify = Slugify(to_lower=True)
custom_slugify.safe_chars = '+.ii'
return custom_slugify(self.model)
def name_slug(self):
custom_slugify = Slugify(to_lower=True)
custom_slugify.safe_chars = '+.ii'
return custom_slugify(self.name)
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import models, migrations
from slugify import Slugify
my_slugify = Slugify()
my_slugify.separator = '-'
my_slugify.pretranslate = {'&': 'and'}
my_slugify.to_lower = True
my_slugify.max_length = None
my_slugify.capitalize = False
my_slugify.safe_chars = ''
def add_slug(CategoryModel, category_name, category_group, category_code,
category_description):
category = CategoryModel.objects.get(code=category_code)
category.slug = my_slugify(category_name)
category.save()
def create_category_slugs(apps, schema_editor):
CategoryModel = apps.get_model("core", "ChCategory")
print("CategoryModel: ", CategoryModel)
# Art & cultural events // Arte y eventos culturales
add_slug(CategoryModel, 'Art & Cultural events', 'Art & Cultural events',
'01.01', 'Dummy description')
# Books & Comics // Libros y cómics
if metadata.has_key(TITLE_META_NAME) and \
metadata.has_key(AUTHOR_META_NAME) and metadata.has_key(ISBN_META_NAME):
add_metadata(file_name, metadata, directory_name)
else:
os.rename(file_name, get_new_file_name_from_old_path(file_name))
except Exception as err:
os.rename(file_name, get_new_file_name_from_old_path(file_name))
print "ERROR: File with errors ", file_name, err.__str__()
TITLE_PROPERTY_NAME = "Title"
TITLE_META_NAME = "/Title"
AUTHOR_PROPERTY_NAME = "Authors"
AUTHOR_META_NAME = "/Author"
ISBN_PROPERTY_NAME = "ISBN"
ISBN_META_NAME = "/ISBN"
FILE_DEFAULT_EXTENSION = ".pdf"
DASH_SEPARATOR = " -- "
DEFAULT_SEPARATOR = " "
DEFAULT_OLD_FOLDER = "old/"
DEFAULT_PENDING_FOLDER = "Waiting4ManualCheck/"
my_slugify = Slugify()
my_slugify.separator = DEFAULT_SEPARATOR
my_slugify.safe_chars = '-.'
set_ebook_metadata()
import inspect
import os
import os.path
try:
# Python 3.x
from urllib.parse import urlsplit
except ImportError:
# Python 2.x
from urlparse import urlsplit
from slugify import Slugify
slugify_filename = Slugify(to_lower=True)
slugify_filename.separator = '_'
slugify_filename.safe_chars = '-.'
slugify_filename.max_length = 255
def get_safe_path_name(filename):
"""
:type filename: unicode
:rtype: unicode
"""
safe_filename = slugify_filename(filename)
return safe_filename.lower()
def get_filename_from_url(url):
"""
from shortuuidfield import ShortUUIDField
from vote.models import VoteModel
from profiles.models import Profile
from djeddit.models import Thread
from .validators import validate_pdf_extension
# from moderation.db import ModeratedModel
# from django.contrib.contenttypes.fields import GenericForeignKey
from django.contrib.contenttypes.models import ContentType
slugify = Slugify()
slugify.safe_chars = '.'
slugify.to_lower = True
# class Resource(ModeratedModel):
class Resource(models.Model):
TEXTBOOK = 'TB'
ONLINE = 'OL'
TEST = 'TS'
COURSE = 'CR'
RESOURCE_TYPE_CHOICES = (
(TEXTBOOK, 'textbook'),
# (ONLINE, 'online learning resource'),
# (TEST, 'standardized test'),
# (COURSE, 'course')
import inspect
import os
import os.path
try:
# Python 3.x
from urllib.parse import urlsplit
except ImportError:
# Python 2.x
from urlparse import urlsplit
from slugify import Slugify
slugify_filename = Slugify(to_lower=True)
slugify_filename.separator = '_'
slugify_filename.safe_chars = '-.'
slugify_filename.max_length = 255
def get_safe_path_name(filename):
"""
:type filename: unicode
:rtype: unicode
"""
safe_filename = slugify_filename(filename)
return safe_filename.lower()
def get_filename_from_url(url):
"""
import numpy as np
from slugify import slugify, Slugify
import re
import math
def getSoupFromUrl(url):
result = requests.get(url)
if result.status_code == 200:
print 'Request successful'
return BeautifulSoup(result.text, "html.parser")
else:
print 'Request failed', url
return "Failed"
slugify_normalize = Slugify()
slugify_normalize.safe_chars = ',./'
def normalize(s):
return slugify_normalize(s, separator=' ')
url = 'http://courses.monoprix.fr'
result = pd.DataFrame(columns=["enseigne", "url", "nom_produit", "marque", "quantite", "poids_volume", "poids_volume_total", "unite", "descriptif", "ingredients", "conservation","valeur_energetique", "origine", "prix", "prix_au_poids"])
result = result.fillna(0)
# file to write url that did'nt work properly
with open("Produits_Problem_5.txt", "a") as f_w_pb:
# file with urls to scrap
n = 0
for line in open('Produits_Problem_4.txt', 'r'):
res = pd.DataFrame(columns=["enseigne", "url", "nom_produit", "marque", "quantite", "poids_volume", "poids_volume_total", "unite", "descriptif", "ingredients", "conservation","valeur_energetique", "origine", "prix", "prix_au_poids"])
from slugify import Slugify
slugify_filename = Slugify()
slugify_filename.separator = ' '
slugify_filename.safe_chars = '~-._!'
slugify_filename.max_length = 255
def safe(name):
return slugify_filename(name)
# coding=utf-8 """**Utilities functions** """ __author__ = 'Ismail Sunni <*****@*****.**>' __revision__ = '$Format:%H$' __date__ = '23/04/2014' __license__ = '' __copyright__ = '' from slugify import Slugify version_slugify = Slugify() version_slugify.safe_chars = '.'
# coding=utf-8 """**Utilities functions** """ __author__ = 'Ismail Sunni <*****@*****.**>' __revision__ = '$Format:%H$' __date__ = '23/04/2014' __license__ = '' __copyright__ = '' from slugify import Slugify version_slugify = Slugify() version_slugify.safe_chars = '.'