def test_images(client, user, image):
# check "get" not works for anonymous user
response = client.get('/api/ckeditor/images/')
assert response.status_code == 403
# check works for registered user
response = user.get('/api/ckeditor/images/')
assert response.status_code == 200
data = response.json()
assert data == {}
# Prepare image
upload = uploadedfile.SimpleUploadedFile(
"file.jpeg", image, content_type="image/jpeg")
dj_client = test_client.Client()
# check anonymous user cant upload files
response = dj_client.post('/api/ckeditor/images/', {'upload': upload})
assert response.status_code == 403
# login & upload
dj_client.login(username=user.user.username, password=user.user.username)
upload = uploadedfile.SimpleUploadedFile(
"file.jpeg", image, content_type="image/jpeg")
response = dj_client.post('/api/ckeditor/images/', {'upload': upload})
assert response.status_code == 200
data = response.json()
# check we can load it by url
response = client.get(data['url'])
assert response.status_code == 200
def get_image_file(url):
r = requests.get(url)
return uploadedfile.SimpleUploadedFile(
urlparse(url).path.split('/')[-1],
r.content,
r.headers['Content-Type'],
)
def _fetch_youtube_image(video_id):
url = IMG_URL.format(vid=video_id)
req = http.urlopen('GET', url, preload_content=False)
ct = req.getheader('Content-Type')
f = uploadedfile.SimpleUploadedFile('youtube-{}'.format(video_id),
req.read(), ct)
return f
def setUp(self):
self.user1 = dcam.User.objects.create_user(username='******',
password='******')
upload_file = open(
('data/dixon2012-h1hesc-hindiii-allreps-filtered.1000kb'
'.multires.cool'), 'rb')
tm.Tileset.objects.create(datafile=dcfu.SimpleUploadedFile(
upload_file.name, upload_file.read()),
filetype='cooler',
uuid='cool-v1',
owner=self.user1)
upload_file = open(
'data/dixon2012-h1hesc-hindiii-allreps-filtered.1000kb.mcoolv2',
'rb')
tm.Tileset.objects.create(datafile=dcfu.SimpleUploadedFile(
upload_file.name, upload_file.read()),
filetype='cooler',
uuid='cool-v2',
owner=self.user1)
def test_add_article_with_media(self):
file1 = uploadedfile.SimpleUploadedFile('file.jpg', b'content', 'image/jpeg')
self.client.credentials(HTTP_AUTHORIZATION=f'Bearer {self.author.token.encrypted}')
response = self.client.post(reverse('article-list'), data={
'media': file1.file,
'article': {'headline': 'h1', 'content': 'c1', },
}, format='multipart')
self.assertEqual(response.status_code, status.HTTP_201_CREATED)
def story_illustration_fixture(story, image):
obj = models.Illustration(
story=story, name='Map of location', admins_only=False)
obj.save()
obj.image.save(
'image.jpg',
uploadedfile.SimpleUploadedFile(
"image.jpeg", image, content_type="image/jpeg"))
obj.save()
return obj
def _product_crawler(self):
from products.models import Product, Picture, Price
def _to_decimal(s):
from decimal import Decimal as D
return D(s.strip().replace(u'€', '').replace(',',
'.').replace(' ', ''))
while True:
try:
product_url, category = self.product_links.popitem()
except KeyError:
break
product_url = quote(product_url)
try:
with closing(urlopen(self.base_url +
product_url)) as product_page:
product_soup = BeautifulSoup(product_page, 'html.parser')
except HTTPError:
print 'error loading page for object at url', self.base_url + product_url
image_url = product_soup.find('div', id='photo').a.get('href')
image_url = quote(image_url)
infosProduits = product_soup.find('div', id='infosProduits')
price = infosProduits.find('div', id='prix').text
price = _to_decimal(price)
deposit_amount = infosProduits.find(
'div', id='composition').find('strong').text
deposit_amount = _to_decimal(deposit_amount)
description = infosProduits.find('p', class_='expandable').text
composition = infosProduits.find('div', id='composition')
description += composition.h2.text
description += '\n'
description += composition.p.text
summary = infosProduits.h1.text
from products.models import Category, Price
from products.choices import UNIT
product = Product.objects.create(
summary=summary,
description=description,
deposit_amount=deposit_amount,
address=self.address,
owner=self.patron,
category=Category.objects.get(slug=category_mapping[category]))
try:
with closing(urlopen(self.base_url + image_url)) as image:
product.pictures.add(
Picture.objects.create(
image=uploadedfile.SimpleUploadedFile(
name='img', content=image.read())))
except HTTPError as e:
print '\nerror loading image for object at url:', self.base_url + product_url
product.prices.add(Price(amount=price, unit=UNIT.DAY))
sys.stdout.write('.')
sys.stdout.flush()
def test_create_with_anonymous_user(self):
"""
Don't allow the creation of datasets by anonymouse users.
"""
with self.assertRaises(ValueError):
upload_file = open(
'data/wgEncodeCaltechRnaSeqHuvecR1x75dTh1014IlnaPlusSignalRep2.hitile',
'r')
tm.Tileset.objects.create(datafile=dcfu.SimpleUploadedFile(
upload_file.name, upload_file.read()),
filetype='hitile',
owner=dcam.AnonymousUser())
def setUp(self):
self.user1 = dcam.User.objects.create_user(username='******',
password='******')
upload_file = open('data/gene_annotations.short.db', 'r')
self.tileset = tm.Tileset.objects.create(
datafile=dcfu.SimpleUploadedFile(upload_file.name,
upload_file.read()),
filetype='beddb',
datatype='gene-annotations',
owner=self.user1,
uuid='bdb')
def setUp(self):
self.user1 = dcam.User.objects.create_user(username='******',
password='******')
upload_file = open('data/cnv_short.hibed', 'r')
#x = upload_file.read()
self.tileset = tm.Tileset.objects.create(
datafile=dcfu.SimpleUploadedFile(upload_file.name,
upload_file.read()),
filetype='hibed',
datatype='stacked-interval',
owner=self.user1,
uuid='hbt')
def setUp(self):
self.user1 = dcam.User.objects.create_user(username='******',
password='******')
upload_file = open(
'data/Dixon2012-J1-NcoI-R1-filtered.100kb.multires.cool', 'r')
self.tileset = tm.Tileset.objects.create(
datafile=dcfu.SimpleUploadedFile(upload_file.name,
upload_file.read()),
filetype='cooler',
datatype='matrix',
owner=self.user1,
uuid='md')
def story_detective_fixture(story, image):
avatar = models.Avatar(story=story, name='Detective Smith')
avatar.save()
avatar.image.save(
'detective.jpg',
uploadedfile.SimpleUploadedFile(
"detective.jpeg", image, content_type="image/jpeg"))
avatar.save()
obj = models.Character(
story=story, name='Detective Smith', description='Public character',
show_in_character_list=True, avatar=avatar)
obj.save()
return obj
def story_fixture(image, admin):
obj = models.Story(
name='Murder in tests',
short_comment='Story for tests',
creation_year=2020,
)
obj.save()
obj.card_image.save(
'card.jpg',
uploadedfile.SimpleUploadedFile(
"card.jpeg", image, content_type="image/jpeg"))
obj.top_banner.save(
'top_banner.jpg',
uploadedfile.SimpleUploadedFile(
"top_banner.jpeg", image, content_type="image/jpeg"))
obj.bottom_banner.save(
'bottom_banner.jpg',
uploadedfile.SimpleUploadedFile(
"bottom_banner.jpeg", image, content_type="image/jpeg"))
obj.save()
models.StoryAdmin(story=obj, user=admin.user).save()
return obj
def setUp(self):
self.user1 = dcam.User.objects.create_user(username='******',
password='******')
upload_file = open('data/arrowhead_domains_short.txt.multires.db', 'r')
self.tileset = tm.Tileset.objects.create(
datafile=dcfu.SimpleUploadedFile(upload_file.name,
upload_file.read()),
filetype='bed2ddb',
datatype='arrowhead-domains',
owner=self.user1,
uuid='ahd')
def variation_hidden_illustration_fixture(story, variation, image):
obj = models.Illustration(story=story,
variation=variation,
name='Map of treasures',
admins_only=True)
obj.save()
obj.image.save(
'image.jpg',
uploadedfile.SimpleUploadedFile("image.jpeg",
image,
content_type="image/jpeg"))
obj.save()
return obj
def handle(self, *args, **options):
from products.models import Picture, Price, Category, Product
from products.choices import UNIT
from accounts.models import Patron
try:
patron = Patron.objects.get(pk=22784)
address = patron.addresses.all()[0]
except Patron.DoesNotExist:
print "Can't find the user"
return
if len(args) != 1:
print "I need exactly one argument"
return
with open(args[0]) as xlsx:
sheet = xlrd.open_workbook(file_contents=xlsx.read()).sheets()[0]
rows = iter(xrange(sheet.nrows))
header = tuple(next_row(sheet, next(rows))) # the header line
next_row(sheet, next(rows)) # the emtpy line
for row in iter(rows):
while True:
try:
product_row = dict(zip(header, next_row(sheet, row)))
image_url = product_row["photo"]
summary = product_row["titre"]
description = product_row["description"]
category = Category.objects.get(
slug=product_row["categorie"])
product = Product.objects.create(
summary=summary,
description=description,
deposit_amount=0,
category=category,
address=address,
owner=patron)
product.prices.add(Price(amount=1, unit=UNIT.DAY))
try:
with closing(urlopen(image_url)) as image:
picture = Picture.objects.create(
image=uploadedfile.SimpleUploadedFile(
name='img', content=image.read()))
product.pictures.add(picture)
except HTTPError as e:
print "error"
product.save()
except:
break
else:
break
def setUp(self):
self.user1 = dcam.User.objects.create_user(username='******',
password='******')
self.user2 = dcam.User.objects.create_user(username='******',
password='******')
upload_file = open(
'data/dixon2012-h1hesc-hindiii-allreps-filtered.1000kb.multires.cool',
'r')
self.cooler = tm.Tileset.objects.create(
datafile=dcfu.SimpleUploadedFile(upload_file.name,
upload_file.read()),
filetype='cooler',
owner=self.user1)
upload_file = open(
'data/wgEncodeCaltechRnaSeqHuvecR1x75dTh1014IlnaPlusSignalRep2.hitile',
'r')
self.hitile = tm.Tileset.objects.create(
datafile=dcfu.SimpleUploadedFile(upload_file.name,
upload_file.read()),
filetype='hitile',
owner=self.user1)
def test_smiles(client, image):
# make a smile
smile = models.Smile(name='lol', text=':lol:')
smile.save()
smile.image.save(
'smile.jpg',
uploadedfile.SimpleUploadedFile(
"file.jpeg", image, content_type="image/jpeg"))
smile.save()
# get it
response = client.get('/api/ckeditor/smiles/')
assert response.status_code == 200
data = response.json()
assert data['smiles'][0]['text'] == ':lol:'
url = data['base_url'] + data['smiles'][0]['file_name']
response = client.get(url)
assert response.status_code == 200
def split_text(self, textfile: uploadedfile.InMemoryUploadedFile,
max_lines):
"""
Splits the textfile into smaller files with at most max_lines sentences.
A list of SimpleUploadedFile objects is returned.
"""
filename = textfile.name
textfiles = []
# get encoding
textfile.open(mode='rb')
encoding = chardet.detect(textfile.read())['encoding']
# put all sentences in a list
filecontent = [] # list of all sentences in the textfile
sentence = '' # one sentence in the textfile that is resetted after every \n\n and added to filecontent
# the open method simply does seek(0). This needs to be done, because the file was already opened to find the encoding
textfile.open()
for line in textfile:
line = line.decode(encoding=encoding)
# this will not work if the newline character is just '\r'
line = line.replace('\r', '')
if line == '\n':
if sentence != '':
filecontent.append(sentence)
sentence = ''
else:
sentence += line.replace('\n', '')
if sentence != '':
filecontent.append(sentence)
# end of gathering filecontent
# validate max_lines
self.check_max_lines(max_lines, len(filecontent))
# create SimpleUploadedFiles with max_lines of content from the textfile
for i in range(math.ceil(len(filecontent) / max_lines)):
filesentences, filecontent = filecontent[:max_lines], filecontent[
max_lines:]
content = ''
for sentence in filesentences:
content += sentence + '\n\n'
new_filename = f'{filename[:-4]}_{i + 1}{filename[-4:]}'
textfiles.append(
uploadedfile.SimpleUploadedFile(new_filename,
content.encode('utf-8-sig')))
return textfiles
def test_create_private_tileset(self):
"""Test to make sure that when we create a private dataset, we can only
access it if we're logged in as the proper user
"""
upload_file = open(
'data/wgEncodeCaltechRnaSeqHuvecR1x75dTh1014IlnaPlusSignalRep2.hitile',
'r')
private_obj = tm.Tileset.objects.create(
datafile=dcfu.SimpleUploadedFile(upload_file.name,
upload_file.read()),
filetype='hitile',
private=True,
owner=self.user1)
c = dt.Client()
c.login(username='******', password='******')
returned = json.loads(
self.client.get('/api/v1/tileset_info/?d={uuid}'.format(
uuid=private_obj.uuid)).content)
def test_tile_symmetry(self):
'''
Make sure that tiles are symmetric
'''
upload_file = open(
'data/Dixon2012-J1-NcoI-R1-filtered.100kb.multires.cool', 'r')
tileset = tm.Tileset.objects.create(datafile=dcfu.SimpleUploadedFile(
upload_file.name, upload_file.read()),
filetype='cooler',
datatype='matrix',
owner=self.user1,
uuid='aa')
ret = self.client.get('/api/v1/tiles/?d=aa.0.0.0')
contents = json.loads(ret.content)
import base64
r = base64.decodestring(contents['aa.0.0.0']['dense'])
q = np.frombuffer(r, dtype=np.float32)
q = q.reshape((256, 256))
def uploaded_audio_file(audio_file):
yield uploadedfile.SimpleUploadedFile(name=audio_file.name,
content=audio_file.read())
def _new_with_transaction(f, transaction):
# f is a Django UploadedFile
image_type = 'image/'
if f.content_type[:len(image_type)] == image_type:
try:
img = Image.open(f)
w, h = img.size
max_size = 1600
if max(w, h) > max_size:
img.thumbnail((max_size, max_size))
tmp = io.BytesIO()
img.save(tmp, img.format)
tmp.seek(0)
f = uploadedfile.SimpleUploadedFile(
name=f.name,
content=tmp.read(),
content_type=f.content_type
)
tmp.close()
except:
pass
finally:
if img:
img.close()
hasher = hashlib.sha256()
for chunk in f.chunks():
hasher.update(chunk)
h = hasher.hexdigest()
test1 = File.objects.filter(hash=h)
test2 = test1.filter(transaction=transaction)
if test2.count() > 0:
# This transaction already has the given file attached.
return test2.get()
if test1.count() > 0:
# Another transaction already has the given file attached.
other_file = test1[0]
this_file = File(
hash=other_file.hash,
filename=other_file.filename,
transaction=transaction
)
this_file.save()
return this_file
# Save this file to the filesystem and the database.
this_file = File(
hash=h,
filename=f.name,
transaction=transaction
)
try:
os.makedirs(os.path.dirname(this_file.abs_path))
except OSError as e:
if e.errno != errno.EEXIST:
raise
with open(this_file.abs_path, 'wb') as w:
for chunk in f.chunks():
w.write(chunk)
this_file.save()
return this_file
def _get_products(self):
from products.models import Product, Picture, Price
for x in xrange(len(self.products['data'])):
# Get the product title
summary = self.products['data'][x]['title_article']
# Get the product price
price = self.products['data'][x]['prix']
# Get the product description
nb_player = 'Nombre de joueurs: ' + self.products['data'][x][
'nbjoueursmin'] + '-' + self.products['data'][x][
'nbjoueursmax'] + '\n'
game_time = 'Temps(minutes): ' + self.products['data'][x][
'temps_minutes'] + '\n'
age_min = 'Age minimum: ' + self.products['data'][x][
'agemin'] + '\n'
game_genre = 'Genre: ' + self.products['data'][x][
'title_genre'] + '\n'
description = nb_player + game_time + age_min + game_genre + self.products[
'data'][x]['descr']
#print description
# Get the product image
image_url = self.image_med_base_url + self.products['data'][x][
'imgsrc']
deposit_amount = 0.0
from products.models import Category
from products.choices import UNIT
try:
product = Product.objects.create(
summary=summary,
description=description,
deposit_amount=deposit_amount,
address=self.address,
owner=self.patron,
category=Category.objects.get(slug=self.category))
try:
with closing(urlopen(image_url)) as image:
product.pictures.add(
Picture.objects.create(
image=uploadedfile.SimpleUploadedFile(
name='img', content=image.read())))
except HTTPError as e:
print '\nerror loading image for object at url:', image_url
print summary
try:
product.prices.add(Price(amount=price, unit=UNIT.DAY))
except Exception, e:
print 'PRICE ERROR'
pass
except Exception, e:
print 'CANNOT CREATE THE PRODUCT %s \n' % (summary)
print 'error: %s' % str(e)
break
def _product_crawler(self):
from products.models import Product, Picture, Price
# Return the price in the right format
def _to_decimal(s):
from decimal import Decimal as D
return D(s.strip().replace(u'€', '').replace(',',
'.').replace(' ', ''))
while True:
try:
product_url, category = self.product_links.popitem()
# print product_url
except KeyError:
break
try:
with closing(urlopen(product_url)) as product_page:
product_soup = BeautifulSoup(product_page, 'html.parser')
except HTTPError:
print 'error loading page for object at url', self.base_url + product_url
family = product_url.split('/')[3]
# Get the image
try:
image_url = product_soup.find(
'a', href=re.compile('.+\.jpg')).get('href')
image_url = product_url.rsplit('/', 1)[0] + '/' + image_url
except:
pass
# Get the title
try:
infosProduits = product_soup.find('h2').text
except:
infosProduits = ''
# print infosProduits
# Get the price
# price_soup = product_soup.find('span', attrs={'class': 'wg-price'})
# print price_soup
# print price
# Get the description
try:
description = product_soup.find(
'td', style='text-align:justify').text
except:
print 'no description'
pass
# Format the title
summary = infosProduits
deposit_amount = 0.0
# print 'summary : %s\n description : %s\n' % (summary, description)
# Create the product
try:
from products.models import Category, Price
from products.choices import UNIT
product = Product.objects.create(
summary=summary,
description=description,
deposit_amount=deposit_amount,
address=self.address,
owner=self.patron,
category=Category.objects.get(
slug=category_mapping[category]))
try:
with closing(urlopen(image_url)) as image:
product.pictures.add(
Picture.objects.create(
image=uploadedfile.SimpleUploadedFile(
name='img', content=image.read())))
except HTTPError as e:
print '\nerror loading image for object at url:', self.base_url + product_url
except:
print 'CANNOT CREATE PRODUCT : %s' % summary
pass
def _product_crawler(self):
from products.models import Product, Picture, Price
# Return the price in the right format
def _to_decimal(s):
from decimal import Decimal as D
return D(s.strip().replace(u'€', '').replace(',', '.').replace(' ', ''))
while True:
try:
product_url, category = self.product_links.popitem()
except KeyError:
break
try:
with closing(urlopen(product_url)) as product_page:
product_soup = BeautifulSoup(product_page, 'html.parser')
except HTTPError:
print 'error loading page for object at url', self.base_url + product_url
#no need to parse all the page
block = product_soup.find('div', id="content")
# Get the title
infosProduits = block.find('h1', class_="product_title entry-title").text
# Get the description
try:
description = block.find('div', id='tab-description').find('p').text
except:
description = 'odessance'
# Format the title
summary = infosProduits
deposit_amount = _to_decimal('0.0')
details = block.find('div', class_="short-description").find('div', class_='std').text
# description += '\n %s' % details
# Create the product
from products.models import Category, Price
from products.choices import UNIT
try:
product = Product.objects.create(
summary=summary, description=description,
deposit_amount=deposit_amount, address=self.address, owner=self.patron,
category=Category.objects.get(slug=category_mapping[category]))
try:
images = block.find('ul', class_="product_thumbnails").find_all('li') #this way we get all the four images
main_img = images[0].find('a').get('href')
with closing(urlopen(main_img)) as image:
product.pictures.add(Picture.objects.create(
image=uploadedfile.SimpleUploadedFile(
name='img', content=image.read())
)
)
except HTTPError as e:
print '\nerror loading image for object at url:', self.base_url + product_url
# Add the price to the product
try:
regex = re.compile(r"(?<=Tarif location\W:\W)\d+")
price = regex.search(details)
price = _to_decimal(price.group(0))
product.prices.add(Price(amount=price, unit=UNIT.DAY))
sys.stdout.flush()
except:
print 'ERROR PRICE'
pass
print 'PRODUCT SUCCESSFULY CREATED'
except:
print 'PRODUCT CANNOT BE CREATED'
pass
def _product_crawler(self):
from products.models import Product, Picture, Price
# Return the price in the right format
def _to_decimal(s):
from decimal import Decimal as D
return D(s.strip().replace(u'€', '').replace(',', '.').replace(' ', ''))
#print self.product_links
while True:
try:
product_url, category = self.product_links.popitem()
except KeyError:
break
#print product_url
#product_url = quote(product_url)
try:
with closing(urlopen(product_url)) as product_page:
product_soup = BeautifulSoup(product_page, 'html.parser')
except HTTPError:
print 'error loading page for object at url', product_url
#Get the image
try:
image_url = product_soup.find('a', class_="MagicZoomPlus").find('img').get('src')
#print "image_url : %s" % image_url
except:
print "pass image"
pass
#Get the title
try:
summary = product_soup.find('h1').text
#print "summary : %s" % summary
except:
print "pass title"
pass
# Get the description
try:
description1 = product_soup.find('div', id='short_description_content').find('p').text
description2 = product_soup.find('div', id='more_info_sheets').find('p').text
description3 = product_soup.find('div', id='more_info_sheets').find_all('span', style="box-sizing: border-box; font-size: 12.222222328186px;")
description4 = "\n".join([description.text for description in description3])
description = "%s\n%s\n%s" % (description1, description2, description4)
#print "description : %s" % description
except:
description = " "
print 'pass description'
pass
# Get the price
try:
price1 = product_soup.find('span', id='our_price_display').text
price2 = (re.findall('\d+', price1 ))
price = "%s.%s" % (int(price2[0]), int(price2[1]))
#print "price : %s" % price
except:
price = "10.00"
print 'pass price'
pass
# Create deposit
deposit_amount = 0.0
# Create the product
from products.models import Category, Price
from products.choices import UNIT
try:
#print "try create"
product = Product.objects.create(
summary=summary, description=description,
deposit_amount=deposit_amount, address=self.address, owner=self.patron,
category=Category.objects.get(slug=category_mapping[category])
)
#print "product_id : %s" % product.pk
try:
#print "try upload image"
with closing(urlopen(image_url)) as image:
product.pictures.add(Picture.objects.create(
image=uploadedfile.SimpleUploadedFile(
name='img', content=image.read())
)
)
#print "picture : %s" % product.pictures.all()[0]
except HTTPError as e:
print '\nerror loading image for object at url:', self.base_url + product_url
# Add the price to the product
try:
product.prices.add(Price(amount=price, unit=UNIT.DAY))
#print "price : %s" % product.prices.all()[0]
except:
print 'PRICE ERROR'
pass
# sys.stdout.write('.')
# sys.stdout.flush()
except:
print 'CANNOT CREATE THE PRODUCT %s \n %s' % (summary, product_url)
pass
print "\n %s products created" % self.patron.products.all().count()
def _product_crawler(self):
from products.models import Product, Picture, Price
# Return the price in the right format
def _to_decimal(s):
from decimal import Decimal as D
return D(s.strip().replace(u'€', '').replace(',',
'.').replace(' ', ''))
while True:
try:
product_url, category = self.product_links.popitem()
except KeyError:
break
try:
with closing(urlopen(self.base_url +
product_url)) as product_page:
product_soup = BeautifulSoup(product_page, 'html.parser')
except HTTPError:
print 'error loading page for object at url', self.base_url + product_url
#no need to parse all the page
block = product_soup.find('div', id="catalogue_pwb")
# Get the image
image_url = block.find('a',
id='zoomlightbox').find('img').get('src')
image_url = self.base_url + '/' + image_url
# Get the title
infosProduits = block.find('h1', class_="h1_pwb").find(
'span', id='grand_titre_nom_produit_fiche_produit').text
# Get the description
description = ''
if block.find('div', id="zoneAttributsSimplesContenu"):
description = block.find('div',
id="zoneAttributsSimplesContenu").text
description = description.replace(u'\n', ' ')
# Format the title
summary = infosProduits
deposit_amount = 0.0
# Create the product
from products.models import Category, Price
from products.choices import UNIT
try:
product = Product.objects.create(
summary=summary,
description=description,
deposit_amount=deposit_amount,
address=self.address,
owner=self.patron,
category=Category.objects.get(
slug=category_mapping[category]))
try:
with closing(urlopen(image_url)) as image:
product.pictures.add(
Picture.objects.create(
image=uploadedfile.SimpleUploadedFile(
name='img', content=image.read())))
except HTTPError as e:
print '\nerror loading image for object at url:', self.base_url + product_url
# Add the price to the product
try:
price = block.find('span', id="zone_prix").find(
'span',
id="prix_pas_promotion_euro_fiche_produit").text
price = _to_decimal(price)
print price
product.prices.add(Price(amount=price, unit=UNIT.DAY))
sys.stdout.flush()
except:
'PRICE ERROR'
pass
except:
'ERROR : CANNOT create PRODUCT %s' % summary
pass
def _product_crawler(self):
from products.models import Product, Picture, Price
# Return the price in the right format
def _to_decimal(s):
from decimal import Decimal as D
s2 = s.strip().replace(u'Location à partir de',
'').replace(',', '.').replace(' ', '')
#print s2
return s2
while True:
try:
product_url, category = self.product_links.popitem()
except KeyError:
#print KeyError
break
try:
with closing(urlopen(product_url)) as product_page:
product_soup = BeautifulSoup(product_page, 'html.parser')
#print product_soup
except HTTPError:
print 'error loading page for object at url', product_url
#Get the image
try:
image_href = product_soup.find(
'a', class_="elevatezoom-gallery").find('img').get('src')
#print image_href
image_url = " %s/boutique/%s" % (self.base_url, image_href)
#print image_url
except:
print "pass image"
pass
#Get the title
try:
summary = product_soup.find('h1').text
#print summary
except:
print "pass title"
pass
# Get the description
try:
description = product_soup.find(
'span', id='texte_description_fiche_produit').text
#print description
except:
print 'pass description'
pass
# Get the price
try:
price1 = product_soup.find(
'span', id='texte_prix_si_prix_desactive').text
price2 = re.findall('\d+', price1)
price = "%s.%s" % (int(price2[0]), int(price2[1]))
#print price
except:
print 'pass price'
pass
# Create deposit
deposit_amount = 0.0
# Create the product
from products.models import Category, Price
from products.choices import UNIT
try:
#print "try create"
product = Product.objects.create(
summary=summary,
description=description,
deposit_amount=deposit_amount,
address=self.address,
owner=self.patron,
category=Category.objects.get(
slug=category_mapping[category]))
#print product.address
try:
#print "try upload image"
with closing(urlopen(image_url)) as image:
product.pictures.add(
Picture.objects.create(
image=uploadedfile.SimpleUploadedFile(
name='img', content=image.read())))
#print image
except HTTPError as e:
print '\nerror loading image for object at url:', family
# Add the price to the product
try:
product.prices.add(Price(amount=price, unit=UNIT.DAY))
except:
print 'PRICE ERROR'
pass
except:
print 'CANNOT CREATE THE PRODUCT %s \n %s' % (summary,
product_url)
pass
print "\n %s products created" % self.patron.products.all().count()
# Create the product
from products.models import Category, Price
from products.choices import UNIT
try:
product = Product.objects.create(
summary=summary, description=description,
deposit_amount=deposit_amount, address=self.address, owner=self.patron,
category=Category.objects.get(slug=category_mapping[category])
)
#print "product_id : %s" % product.pk
try:
#print "try upload image"
with closing(urlopen(image_url)) as image:
product.pictures.add(Picture.objects.create(
image=uploadedfile.SimpleUploadedFile(
name='img', content=image.read())
)
)
#print "picture : %s" % product.pictures.all()[0]
except HTTPError as e:
print '\nerror loading image for object at url:', self.base_url + product_url
# Add the price to the product
if self.price_tag:
try:
if price:
product.prices.add(Price(amount=price, unit=UNIT.DAY))
#print "price : %s" % product.prices.all()[0]
except Exception, e:
print 'PRICE ERROR'
pass
