def save(self, *args, **kwargs):
if self.image:
#open image
pil_image_obj = Image.open(self.image)
img_size = pil_image_obj.size
if float(img_size[0]) > 500:
new_image = resizeimage.resize_width(pil_image_obj, 500)
new_image_io = BytesIO()
new_image.save(new_image_io, format='JPEG')
temp_name = self.image.name
self.image.delete(save=False)
self.image.save(temp_name, content=ContentFile(new_image_io.getvalue()), save=False)
if self.image_slurp:
print("image slurp")
imgRequest = request.urlopen(self.image_slurp)
if imgRequest.status == 200:
file_name = self.image_slurp.split('/')[-1]
img_temp = NamedTemporaryFile()
img_temp.write(imgRequest.read())
img_temp.flush()
img_file = File(img_temp)
pil_image_obj = Image.open(img_temp)
img_size = pil_image_obj.size
if float(img_size[0]) > 500:
new_image = resizeimage.resize_width(pil_image_obj, 500)
else:
new_image = pil_image_obj
new_image_io = BytesIO()
new_image.save(new_image_io, format='JPEG')
temp_name = file_name
self.image_slurp = None
self.image.delete(save=False)
self.image.save(temp_name, content=ContentFile(new_image_io.getvalue()), save=False)
super(Article, self).save(*args, **kwargs)
def download_media(self, media_url):
"""
Fetches the recording and stores it with the provided recording_id
:param media_url: the url where the media lives
:return: the url for our downloaded media with full content type prefix
"""
response = requests.get(media_url, stream=True, auth=self.auth)
disposition = response.headers.get('Content-Disposition', None)
content_type = response.headers.get('Content-Type', None)
if content_type:
extension = None
if disposition:
filename = re.findall("filename=\"(.+)\"", disposition)[0]
extension = filename.rpartition('.')[2]
elif content_type == 'audio/x-wav':
extension = 'wav'
temp = NamedTemporaryFile(delete=True)
temp.write(response.content)
temp.flush()
return '%s:%s' % (content_type, self.org.save_media(File(temp), extension))
return None
def upload_image_url(request):
if request.method != 'POST':
return HttpResponse(status=403)
image_url = request.POST.get('image_url', None)
source_domain = request.POST.get('source_domain', None)
headers = {
'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11',
# 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
# 'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.3',
# 'Accept-Encoding': 'none',
# 'Accept-Language': 'en-US,en;q=0.8',
# 'Connection': 'keep-alive',
'referer': source_domain,
}
ext = mimetypes.guess_extension(mimetypes.guess_type(image_url)[0])
req = urllib2.Request(image_url, headers=headers)
img_temp = NamedTemporaryFile(delete=True)
img_temp.write(urllib2.urlopen(req).read())
img_temp.flush()
post_photo = Post_photo()
post_photo.photo.save('%s%s'%(uuid.uuid4(), ext), File(img_temp))
post_photo.save()
res = {
'link': post_photo.photo.url,
}
return JsonResponse(res, safe=False)
def create_user_profile(sender,instance,created,**kwargs):
"""
Creates a Profile model for each User that is created.
This function is called on the post_save signal from User.
"""
u = instance
if created:
p = Profile.objects.create(user=u)
if not u.last_name:
names = u.first_name.split(" ")
if len(names)>1:
u.last_name = names[1]
u.save()
if not u.get_profile().image:
# get gravatar
user_sa = UserSocialAuth.objects.filter(user=u)
if user_sa:
user_sa = user_sa[0]
token = user_sa.extra_data['access_token']
g = Github(token).get_user()
gravatar_id = g.gravatar_id
url = "https://secure.gravatar.com/avatar/" + gravatar_id + "?s=500"
img_temp = NamedTemporaryFile(delete=True)
img_temp.write(urllib2.urlopen(url).read())
img_temp.flush()
u.get_profile().image.save("gravatar_image_" + str(u.pk), File(img_temp))
def process(filename):
data = json.loads( open(filename, 'r').read() )
# pprint.pprint( data )
print "%s (%s) - %s" % (data['name'], data['slug'], filename)
slug = data['slug']
try:
person = models.Person.objects.get(slug=slug)
return # don't try to update the person
except models.Person.DoesNotExist:
person = models.Person(slug=slug)
person.legal_name = data['name']
person.summary = data['summary']
person.date_of_birth = data['date_of_birth']
person.save()
content_type = ContentType.objects.get_for_model(person)
if data.get('profile_url'):
models.Contact.objects.get_or_create(
content_type = content_type,
object_id = person.id,
value = re.sub('\s', '%20', data['profile_url'] ),
kind = profile_url_kind,
)
if data.get('email'):
models.Contact.objects.get_or_create(
content_type = content_type,
object_id = person.id,
value = data['email'],
kind = email_kind,
)
# import image
if data.get('image') and 'img_not_found' not in data['image']:
image_url = re.sub('\s', '%20', data['image'] );
photo, created = Image.objects.get_or_create(
content_type = content_type,
object_id = person.id,
source = image_url,
)
if created:
print " Fetching " + image_url
try:
img_temp = NamedTemporaryFile(delete=True)
img_temp.write( urllib2.urlopen(image_url).read() )
img_temp.flush()
photo.image.save( person.slug, File(img_temp) )
photo.save()
except urllib2.HTTPError:
print " ...failed!"
def import_images(logo_dir):
report_data = dict(failed=0, succeeded=0)
for logo_name in os.listdir(logo_dir):
error_message = ""
identifier, extension = splitext(logo_name)
if extension.lower() in (".png", ".jpg", ".jpeg", ".gif"):
try:
internal_org_id = InternalOrganisationID.objects.get(
recording_org=Organisation.objects.get(id=CORDAID_ORG_ID), identifier=identifier
)
org = internal_org_id.referenced_org
filename = model_and_instance_based_filename("Organisation", org.pk, "logo", logo_name)
with open(os.path.join(logo_dir, logo_name), "rb") as f:
logo_data = f.read()
logo_tmp = NamedTemporaryFile(delete=True)
logo_tmp.write(logo_data)
logo_tmp.flush()
org.logo.save(filename, File(logo_tmp), save=True)
action = "succeeded"
except Exception, e:
action = "failed"
error_message = "with the following error message: {error_message}".format(error_message=e.message)
report_data[action] += 1
log_and_alert(
u"Upload of image to organisation {org_id} {action} {error_message}".format(
org_id=org.id, action=action, error_message=error_message
)
)
def _update_image(facebook_id, image_url):
'''
Updates the user profile's image to the given image url
Unfortunately this is quite a pain to get right with Django
Suggestions to improve this are welcome
'''
image_name = 'fb_image_%s.jpg' % facebook_id
image_temp = NamedTemporaryFile()
try:
image_response = urllib2.urlopen(image_url)
except AttributeError:
image_response = urllib.request.urlopen(image_url)
image_content = image_response.read()
image_temp.write(image_content)
http_message = image_response.info()
image_size = len(image_content)
try:
content_type = http_message.type
except AttributeError:
content_type = http_message.get_content_type()
image_file = InMemoryUploadedFile(
file=image_temp, name=image_name, field_name='image',
content_type=content_type, size=image_size, charset=None
)
image_file.seek(0)
image_temp.flush()
return image_name, image_file
def saveProject(self, args):
print u'%s' % args
try:
obj = Project.objects.get(project_name = args['name'],url = args['url'])
return None
except Project.DoesNotExist:
obj = Project(
submitted_event = self.event,
project_name = args['name'],
short_description = args['desc'],
url = args['url'],
project_type = args['type']
)
img_temp = NamedTemporaryFile(delete=True)
if args['image']:
img_temp.write(urllib2.urlopen(args['image']).read())
img_temp.flush()
img_temp.seek(0)
img_filepath = urlparse(args['image']).path.split('/')[-1]
obj.image.save(img_filepath, File(img_temp))
obj.save()
else :
obj.save()
return obj
def save(self, *args, **kwargs):
hash_name = os.urandom(32).encode('hex')
if not self.image:
temp_img = NamedTemporaryFile()
temp_img.write(urllib2.urlopen(self.url).read())
temp_img.flush()
image = Image.open(temp_img.name)
image.save(temp_img.name, 'JPEG')
self.image.save(''.join([hash_name, '.jpg']), File(temp_img))
if not self.thumbnail:
if not self.image:
image = Image.open(temp_img.name)
else:
super(Pin, self).save()
image = Image.open(self.image.path)
size = image.size
prop = 200.0 / float(image.size[0])
size = (int(prop*float(image.size[0])), int(prop*float(image.size[1])))
image.thumbnail(size, Image.ANTIALIAS)
temp_thumb = NamedTemporaryFile()
image.save(temp_thumb.name, 'JPEG')
self.thumbnail.save(''.join([hash_name, '.jpg']), File(temp_thumb))
super(Pin, self).save(*args, **kwargs)
def save_url_image(field, url, name):
r = requests.get(url)
img_temp = NamedTemporaryFile(delete=True)
img_temp.write(r.content)
img_temp.flush()
field.save(name, File(img_temp), save=True)
def fetch_photos_from_msg(self, album, msg=None):
u = album.user
token = get_access_token(u)
graph = facebook.GraphAPI(token)
if msg.status == 'awaiting':
parts = urlparse.urlparse(msg.next_page)
qs = urlparse.parse_qs(parts.query)
after = qs.get('after')[0]
photos = graph.get_object(album.fb_album_id + "/photos", fields='id,source', limit=2, after=after)
new_next_page = photos.get('paging').get('next')
new_msg = Message.objects.create(next_page=new_next_page, user=u, status='awaiting')
for photo in photos.get('data'):
img_temp = NamedTemporaryFile(delete=True)
img_temp.write(urlopen(photo.get('source')).read())
img_temp.flush()
photo_object = Photo.objects.create(title=photo.get('id'),
description=photo.get('created_time'),
album=album,
file=File(img_temp))
pprint(photo_object.filename)
self.stdout.write('Successfully fetched photo for source "%s"\n' % photo.get('source'))
msg.status = 'done'
msg.save()
self.stdout.write('Finished this queue "%s"\n' % new_msg.next_page)
def post(self, request, *args, **kwargs):
# Set Project
project = request.POST.get('project', 'default')
open_db(project)
url = request.POST.get('url')
tags = request.POST.get('tag_list')
tags = "url," + tags
if request.POST.get('tor'):
downloaded_file = network.download(url, tor=True)
else:
downloaded_file = network.download(url, tor=False)
if downloaded_file is None:
messages.error(request, "server can't download from URL")
return redirect(reverse("main-page-project", kwargs={"project": project}))
tf = NamedTemporaryFile()
tf.write(downloaded_file)
if not tf:
messages.error(request, "server can't download from URL")
return redirect(reverse("main-page-project", kwargs={"project": project}))
tf.flush()
sha_256 = add_file(tf.name, name=url.split('/')[-1], tags=tags)
if sha_256:
messages.success(request, "stored file in database: {}".format(tf.name))
return redirect(reverse('main-page-project', kwargs={'project': project}))
else:
messages.error(request, "Unable to Store The File, already in database")
return redirect(reverse("main-page-project", kwargs={"project": project}))
def upload_fb(request):
"""Uploads the user's picture from Facebook."""
if request.method == "POST":
form = FacebookPictureForm(request.POST)
if form.is_valid():
# Need to download the image from the url and save it.
photo_temp = NamedTemporaryFile(delete=True)
fb_url = form.cleaned_data["facebook_photo"]
photo_temp.write(urllib2.urlopen(fb_url).read())
photo_temp.flush()
photo_temp.seek(0)
# Delete old avatars if they exist
avatars = Avatar.objects.filter(user=request.user)
for avatar in avatars:
avatar.avatar.storage.delete(avatar.avatar.name)
avatar.avatar.delete()
avatar.delete()
path = avatar_file_path(user=request.user,
filename="fb_photo.jpg")
avatar = Avatar(
user=request.user,
primary=True,
avatar=path,
)
# print "saving facebook photo to " + path
avatar.avatar.storage.save(path, File(photo_temp))
avatar.save()
return HttpResponseRedirect(
reverse("profile_index") + "?changed_avatar=True")
raise Http404
def save_to_model(file_field, file_name):
img_temp = NamedTemporaryFile(delete=True)
img_temp.write(open(os.path.join(settings.MEDIA_ROOT, file_name), 'r').read())
img_temp.flush()
file_field.save(os.path.basename(file_name), File(img_temp), save=False)
# delete files after saving in models
delete_file(file_name)
def save_image(profile, url):
img = NamedTemporaryFile(delete=True)
img.write(urllib.request.urlopen(url).read())
img.flush()
profile.avatar_image.save(str(profile.id), File(img))
def import_entries(self, feed_entries):
"""Import entries"""
for feed_entry in feed_entries:
self.write_out("> %s... " % feed_entry.title)
creation_date = datetime(*feed_entry.date_parsed[:6])
slug = slugify(feed_entry.title)[:255]
if Entry.objects.filter(
creation_date__year=creation_date.year,
creation_date__month=creation_date.month,
creation_date__day=creation_date.day,
slug=slug,
):
self.write_out(self.style.NOTICE("SKIPPED (already imported)\n"))
continue
categories = self.import_categories(feed_entry)
entry_dict = {
"title": feed_entry.title[:255],
"content": feed_entry.description,
"excerpt": feed_entry.get("summary"),
"status": PUBLISHED,
"creation_date": creation_date,
"start_publication": creation_date,
"last_update": datetime.now(),
"slug": slug,
}
if not entry_dict["excerpt"] and self.auto_excerpt:
entry_dict["excerpt"] = Truncator("...").words(50, strip_tags(feed_entry.description))
if self.tags:
entry_dict["tags"] = self.import_tags(categories)
entry = Entry(**entry_dict)
entry.save()
entry.categories.add(*categories)
entry.sites.add(self.SITE)
if self.image_enclosure:
for enclosure in feed_entry.enclosures:
if "image" in enclosure.get("type") and enclosure.get("href"):
img_tmp = NamedTemporaryFile(delete=True)
img_tmp.write(urlopen(enclosure["href"]).read())
img_tmp.flush()
entry.image.save(slug, File(img_tmp))
break
if self.default_author:
entry.authors.add(self.default_author)
elif feed_entry.get("author_detail"):
try:
user = User.objects.create_user(
slugify(feed_entry.author_detail.get("name")), feed_entry.author_detail.get("email", "")
)
except IntegrityError:
user = User.objects.get(username=slugify(feed_entry.author_detail.get("name")))
entry.authors.add(user)
self.write_out(self.style.ITEM("OK\n"))
def get_image_cover(self, path):
r = requests.get(u"http://stratege.ru{}".format(path))
img_temp = NamedTemporaryFile(delete=True)
img_temp.write(r.content)
img_temp.flush()
name_file = path.split("/")[-1]
return name_file, img_temp
def image(self):
try:
r = requests.get(self.remote_path)
except Exception as e:
raise Exception('Exception %s raised '
'during loading image %s' % (e, self.remote_path))
if self.storage.exists(self.full_name) and self.if_cache:
im = Image.open(self.storage.path(self.full_name))
im = processors.save_image(im, format=self.file_extension)
else:
img_temp = NamedTemporaryFile(delete=True)
img_temp.write(r.content)
img_temp.flush()
im = Image.open(img_temp.name)
if not self.final_size is None:
im = processors.scale_and_crop(
im,
self.final_size,
self.method)
im = processors.colorspace(im)
im = processors.save_image(im, format=self.file_extension)
self.storage.save(self.full_name, im)
return im
def get_image_from_url(url):
""" Get and save images from urls """
r = requests.get(url)
img_temp = NamedTemporaryFile(delete=True)
img_temp.write(r.content)
img_temp.flush()
return File(img_temp)
def user_join(request):
login_from = int(request.POST.get("login_from", 0))
user_id = request.POST.get("user_id")
password = request.POST.get("password")
user_name = request.POST.get("user_name")
user_email = request.POST.get("user_email")
phone = request.POST.get("user_phone")
gender = int(request.POST.get("user_gender", 0))
img_url = request.POST.get("img_url", "")
profile_img = None
if login_from != 0:
password = "******" % (login_from, password)
if img_url:
profile_img = NamedTemporaryFile(delete=True)
profile_img.write(urllib2.urlopen(img_url).read())
profile_img.flush()
try:
user = User.objects.create_user(username=user_id, password=password, email=user_email, first_name=user_name)
if user:
profile = UserProfile.objects.create(user=user, gender=gender, login_from=login_from, phone=phone)
if profile_img:
profile.src.save("%s.jpg" % user_id, File(profile_img))
user = authenticate(username=user_id, password=password)
if user:
login(request, user)
request.session.set_expiry(31536000)
return HttpResponseRedirect("/")
except Exception as e:
pass
return HttpResponseRedirect("/home/")
def shrinkImageFromData(data):
api_key = settings.TINYPNG_API_KEY
info, new_data = shrink_data(data, api_key)
img_shrunked = NamedTemporaryFile(delete=False)
img_shrunked.write(new_data)
img_shrunked.flush()
return ImageFile(img_shrunked)
def _download_file(url, obj, obj_fieldname='executed_file', filename=None, update_obj=True):
"""
Task to download a file form a url and save it to a model field
"""
if url not in [None, '']:
# Validate the url
URLValidator(url)
if filename is None:
filename = urlparse.urlparse(url).path
filename_no_ext, ext = os.path.splitext(filename.split('/')[-1])
#
# @BUSINESSRULE must have a file .suffix
#
if ext is None:
raise Exception('Cannot download a file with no filename.extension: %s' % url)
filename = '%s%s' % (filename_no_ext, ext)
#try:
img_temp = NamedTemporaryFile(delete=True, suffix=ext)
img_temp.write(urllib2.urlopen(url).read())
img_temp.flush()
#
# SAVE THE FILE LOCALLY
#
# use the upload_to function to name and place the file appropriately
filename = obj._meta.get_field(obj_fieldname).upload_to(instance=obj, filename=filename)
file_object = File(img_temp)
# return both the filename and the file_object for saving to the model
return (default_storage.save(filename, file_object), file_object,)
def handle(self, *args, **options):
rss_url = 'http://blog.djangogirls.org/rss'
response = requests.get(rss_url)
rss = ElementTree.fromstring(response.content)
for post in rss.iter('item'):
title = post.find('title').text
if 'Your Django Story: Meet' in title:
name = title.replace('Your Django Story: Meet ', '')
is_story = True
else:
name = title
is_story = False
if not Story.objects.filter(name=name).exists():
post_url = post.find('link').text
post = pq(post.find('description').text)
image_url = post('img').attr.src
story = Story(name=name, post_url=post_url, content=post, is_story=is_story)
if image_url:
img = NamedTemporaryFile(delete=True)
img.write(urlopen(image_url).read())
img.flush()
story.image.save(image_url.split('/')[-1], File(img))
story.save()
if is_story:
print('Story of %s has been fetched' % name)
else:
print('Blogpost "%s" has been fetched' % name)
def _save_image(self, url):
img = NamedTemporaryFile(delete=True)
img.write(urllib.request.urlopen(url).read())
img.flush()
self.user.avatar_image.save(str(self.user.id), File(img))
def create_thumbnail(model_instance):
# CREATING IMAGE FROM THUMBNAIL
backend = detect_backend(model_instance.url)
thumbnail_url = backend.get_thumbnail_url()
if backend.__class__.__name__ == 'YoutubeBackend':
if thumbnail_url.endswith('hqdefault.jpg'):
for resolution in YOUTUBE_RESOLUTIONS:
temp_thumbnail_url = thumbnail_url.replace(
'hqdefault.jpg', resolution)
if checkUrl(temp_thumbnail_url):
thumbnail_url = temp_thumbnail_url
break
img_temp = NamedTemporaryFile(delete=True)
try:
img_temp.write(urllib2.urlopen(thumbnail_url).read())
except:
http = urllib3.PoolManager()
img_temp.write(http.request('GET', thumbnail_url).data)
img_temp.flush()
image = WagtailImage(title=model_instance.title)
image.file.save(model_instance.title + '.jpg', File(img_temp))
model_instance.thumbnail = image
model_instance.thumbnail.tags.add('video-thumbnail')
model_instance.save()
def multiple_dna(*args):
"""
List of tuples: (seq_name, seq_frame, seq)
"""
seq_name_lengths = []
input_file = NamedTemporaryFile(prefix="mafft_")
for arg in args:
seq_name, seq_frame, seq = arg
if seq_frame < 0:
seq_name = "%s(%s)" % (seq_name, "-")
seq = Seq(seq).reverse_complement().tostring()
elif seq_frame > 0:
seq_name = "%s(%s)" % (seq_name, "+")
input_file.write(">%s\n%s\n" % (seq_name, seq.upper()))
seq_name_lengths.append(len(seq_name))
input_file.flush()
namelength = max(seq_name_lengths) + 4
mafft_cmd = (
"mafft --genafpair --maxiterate 1000 --preservecase --clustalout --namelength "
+ str(namelength)
+ " "
+ input_file.name
)
mafft_proc = Popen(mafft_cmd, stdout=PIPE, stderr=PIPE, shell=True)
stdout, stderr = mafft_proc.communicate()
input_file.close()
return stdout
def post(self, request, *args, **kwargs):
parent = self.get_object()
files = json.loads(request.POST.get('files[]'))
need_upload = request.POST.get('need_upload') == 'true'
for f in files:
obj = self.model(parent=parent)
if need_upload:
r = requests.get(f.get('link'), stream=True)
img_temp = NamedTemporaryFile(delete=True)
for chunk in r.iter_content(8192):
img_temp.write(chunk)
obj.file.save(f.get('name'), File(img_temp))
img_temp.flush()
link = f.get('thumbnailLink').replace('bounding_box=75', 'bounding_box=256')
r = requests.get(link, stream=True)
if f.get('thumbnailLink'):
img_temp = NamedTemporaryFile(delete=True)
for chunk in r.iter_content(8192):
img_temp.write(chunk)
obj.thumbnail.save(f.get('name'), File(img_temp))
img_temp.flush()
if f.get('thumbnailLink'):
obj.outer_thumbnail_url = f.get('thumbnailLink')
obj.content_type = mimetypes.guess_type(f.get('name'))[0] # or 'image/png',
obj.outer_url = f.get('link')
obj.save()
response = JSONResponse({'status': 'ok'}, mimetype=response_mimetype(request))
response['Content-Disposition'] = 'inline; filename=files.json'
return response
def pairwise_protein(query_name, query_seq, query_frame, subject_name, subject_seq, subject_frame):
if query_frame < 0:
query_name = query_name + "(" + str(query_frame) + ")"
query_seq = Seq(query_seq).reverse_complement()[-query_frame - 1 :].translate().tostring()
elif query_frame > 0:
query_name = query_name + "(" + str(query_frame) + ")"
query_seq = Seq(query_seq)[query_frame - 1 :].translate().tostring()
if subject_frame < 0:
subject_name = subject_name + "(" + str(subject_frame) + ")"
subject_seq = Seq(subject_seq).reverse_complement()[-subject_frame - 1 :].translate().tostring()
elif subject_frame > 0:
subject_name = subject_name + "(" + str(subject_frame) + ")"
subject_seq = Seq(subject_seq)[subject_frame - 1 :].translate().tostring()
input_file = NamedTemporaryFile(prefix="mafft_")
input_file.write("\n".join([">" + query_name, query_seq.upper(), ">" + subject_name, subject_seq.upper()]))
input_file.flush()
namelength = max([len(query_name), len(subject_name)]) + 4
mafft_cmd = "mafft --preservecase --clustalout --namelength " + str(namelength) + " " + input_file.name
mafft_proc = Popen(mafft_cmd, stdout=PIPE, stderr=PIPE, shell=True)
stdout, stderr = mafft_proc.communicate()
return stdout
def get_cached(cls, key):
"""
Busca imagem no cache, se não encontrada tenta buscar no banco de dados.
* key: chave da imagem
"""
if not cls._cache.has_key(key):
obj = ImageConstant.objects.with_id(key)
if not obj:
return
f = NamedTemporaryFile(
delete=False,
suffix='.%s' % obj.image.format)
buf = obj.image.read()
if not buf:
return
f.write(buf)
cls._open_images[key] = f
f.flush()
cls._cache[key] = f.name
return cls._cache[key]
def scrap_items():
for itemlist in ITEMLIST:
soup = BS(urllib2.urlopen(''.join([LOLWIKI, itemlist])).read())
item_table = soup.find('table', class_='stdt sortable')
for tr in item_table.find_all('tr'):
tds = tr.find_all('td')
if len(tds) < 1:
continue
if tr.find('p') == None:
continue
item_name = tr.find('p').text.strip()
item_url = tr.find('img')['src']
if item_url.split(':')[0] == 'data':
item_url = tr.find('img')['data-src']
if not HOOKED:
continue
#store item in database
d_item = Item()
d_item.name = item_name
t_img = NamedTemporaryFile(delete=True)
t_img.write(urllib2.urlopen(item_url).read())
t_img.flush()
t_img.name = '.'.join([item_name, 'jpg'])
d_item.picture = File(t_img)
d_item.save()
def generate_kml_export(export_type, username, id_string, export_id=None,
options=None, xform=None):
"""
Generates kml export for geographical data
param: export_type
params: username: logged in username
params: id_string: xform id_string
params: export_id: ID of export object associated with the request
param: options: additional parameters required for the lookup.
ext: File extension of the generated export
"""
extension = options.get("extension", export_type)
user = User.objects.get(username=username)
if xform is None:
xform = XForm.objects.get(user__username=username, id_string=id_string)
response = render_to_response(
'survey.kml', {'data': kml_export_data(id_string, user, xform=xform)})
basename = "%s_%s" % (id_string,
datetime.now().strftime("%Y_%m_%d_%H_%M_%S"))
filename = basename + "." + extension
file_path = os.path.join(
username,
'exports',
id_string,
export_type,
filename)
storage = get_storage_class()()
temp_file = NamedTemporaryFile(suffix=extension)
temp_file.write(response.content)
temp_file.seek(0)
export_filename = storage.save(
file_path,
File(temp_file, file_path))
temp_file.close()
dir_name, basename = os.path.split(export_filename)
# get or create export object
if export_id and Export.objects.filter(pk=export_id).exists():
export = Export.objects.get(id=export_id)
else:
export_options = get_export_options(options)
export = Export.objects.create(xform=xform,
export_type=export_type,
options=export_options)
export.filedir = dir_name
export.filename = basename
export.internal_status = Export.SUCCESSFUL
export.save()
return export
def generate_osm_export(export_type, username, id_string, export_id=None,
options=None, xform=None):
"""
Generates osm export for OpenStreetMap data
param: export_type
params: username: logged in username
params: id_string: xform id_string
params: export_id: ID of export object associated with the request
param: options: additional parameters required for the lookup.
ext: File extension of the generated export
"""
extension = options.get("extension", export_type)
if xform is None:
xform = XForm.objects.get(user__username=username, id_string=id_string)
osm_list = OsmData.objects.filter(instance__xform=xform)
content = get_combined_osm(osm_list)
basename = "%s_%s" % (id_string,
datetime.now().strftime("%Y_%m_%d_%H_%M_%S"))
filename = basename + "." + extension
file_path = os.path.join(
username,
'exports',
id_string,
export_type,
filename)
storage = get_storage_class()()
temp_file = NamedTemporaryFile(suffix=extension)
temp_file.write(content)
temp_file.seek(0)
export_filename = storage.save(
file_path,
File(temp_file, file_path))
temp_file.close()
dir_name, basename = os.path.split(export_filename)
# get or create export object
if export_id and Export.objects.filter(pk=export_id).exists():
export = Export.objects.get(id=export_id)
else:
export_options = get_export_options(options)
export = Export.objects.create(xform=xform,
export_type=export_type,
options=export_options)
export.filedir = dir_name
export.filename = basename
export.internal_status = Export.SUCCESSFUL
export.save()
return export
def get_file_from_url(url):
tmp = NamedTemporaryFile(delete=True)
req = requests.get(url)
if not req.ok:
logging.error('Failed to import image from {}'.format(url))
return None
tmp.write(req.content)
tmp.flush()
name = urlparse(req.url).path.split('/')[-1]
return name, File(tmp)
def download_avatar(self, url):
"""
"""
r = requests.get(url)
img_temp = NamedTemporaryFile(delete=True)
img_temp.write(r.content)
img_temp.flush()
img_temp.seek(0)
return File(img_temp)
def save(self, *args, **kwargs):
try:
self.image = compress_image(self.image)
except AttributeError:
img_temp = NamedTemporaryFile(delete=True)
img_temp.write(urlopen(self.image.url).read())
img_temp.flush()
self.image = compress_image(File(img_temp))
super(Publication, self).save(*args, **kwargs)
def saveInnImage(model, url):
try:
img_tmp = NamedTemporaryFile(delete = True)
img_tmp.write(urllib.request.urlopen(url))
img_tmp.flush()
model.inn_photo.save("inn_image_"+str(model.inn_id)+".jpg", File(img_temp), save=True)
except urllib.error.HTTPError as error:
pass
except Exception as e:
print("Error(store_inn_data.py, saveInnImage) URL:" + url)
def getUserPictures(obj, url_photo, url_cover):
img_temp = NamedTemporaryFile()
img_temp.write(urllib2.urlopen(url_photo).read())
img_temp.flush()
obj.photo.save((('%s%s') % (obj.id, img_temp.name)), File(img_temp))
img_temp = NamedTemporaryFile()
img_temp.write(urllib2.urlopen(url_cover).read())
img_temp.flush()
obj.cover.save((('%s%s') % (obj.id, img_temp.name)), File(img_temp))
return obj
def _comic_book_crawle_cover(comic_book):
headers = {'refer': comic_book.original_link}
r = requests.get(comic_book.original_cover_url, headers=headers)
img_temp = NamedTemporaryFile(delete=True)
img_temp.write(r.content)
img_temp.flush()
comic_book.cover.save("%s.jpg" % comic_book.name,
File(img_temp),
save=True)
def handle_upload_url_file(url):
img_temp = NamedTemporaryFile()
opener = urllib2.build_opener()
opener.addheaders = [(
'User-agent',
'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:15.0) Gecko/20120427 Firefox/15.0a1'
)]
img_temp.write(opener.open(url).read())
img_temp.flush()
return img_temp
def save_image_from_url(field, url, image, i, save):
"""This function download an image from a given url"""
r = requests.get(url)
img_temp = NamedTemporaryFile(delete=True)
img_temp.write(r.content)
img_temp.flush()
# field.save("zef", File(img_temp), save=True)
field.save(image + str(i) + url[-4:], File(img_temp), save=save)
def saveImageUrl(model, url):
r = requests.get(url)
if r.status_code == requests.codes.ok:
img_temp = NamedTemporaryFile(delete=True)
img_temp.write(r.content)
img_temp.flush()
img_filename = img_temp.name + splitext(urlparse(url).path)[1]
model.avatar.save(img_filename, FileDjango(img_temp), save=True)
return True
return False
def make_from_url(self, url, product):
extension = url.split('.')[-1]
img_temp = NamedTemporaryFile(delete=True)
img_temp.write(urlopen(url).read())
img_temp.flush()
obj = ProductPhoto(product=product)
obj.photo.save('product_%d.%s' % (product.id, extension),
content=File(img_temp))
obj.save()
return obj
def save(self, *args, **kwargs):
if self.pk is None or Film.objects.get(pk=self.pk).imdb_poster_url != self.imdb_poster_url:
img = requests.get(self.imdb_poster_url)
if img.status_code == requests.codes.ok:
img_temp = NamedTemporaryFile(delete=True)
img_temp.write(img.content)
img_temp.flush()
self.imdb_poster.save(self.slug, File(img_temp), save=False)
img_temp.close()
super().save(*args, **kwargs)
def image_load(request):
if request.method == 'POST':
image_url = request.POST.get('image_url')
pos_img_format = image_url.rfind('.')
img_format = image_url[pos_img_format:]
img_temp = NamedTemporaryFile(delete=True)
try:
img_temp.write(urllib.request.urlopen(image_url).read())
except ValueError:
return HttpResponseBadRequest()
img_temp.flush()
# new_image = Image.open(img_temp).resize((470, 245))
# new_image = resize_and_crop(Image.open(img_temp), (570, 345))
new_image = resize(Image.open(img_temp), (380, 480))
if img_format == ".jpg":
img_format = ".JPEG"
image_io = io.BytesIO()
new_image.save(image_io, format=img_format[1:])
image_file = ContentFile(image_io.getvalue())
if request.user.is_authenticated():
profile = request.user.profile
image_palette = ImagePalette.objects.create(profile=profile)
else:
image_palette = ImagePalette.objects.create()
image_palette.original_url = image_url
image_palette.save()
image_palette.image.save(image_url, image_file, save=True)
# File(img_temp)
meta_img = Image.open(image_palette.image)
meta_img = meta_img.resize((50, 50))
result = meta_img.convert('P', palette=Image.ADAPTIVE, colors=5)
result.putalpha(0)
colors = result.getcolors(50*50)
for count, color_rgb in colors:
color_hex = '%02x%02x%02x' % (color_rgb[0], color_rgb[1], color_rgb[2])
color, created = Color.objects.get_or_create(color=color_hex)
image_palette.colors.add(color)
return HttpResponse(image_palette.id)
else:
raise Http404
def save_tack(request):
"""
This method is used to save a new tack.
"""
img_url = request.POST["img_url"]
bookmark_url = request.POST["tack_url"]
file_input = request.FILES.get('file')
if request.POST["new_board"] != "":
boardName = request.POST["new_board"]
else:
boardName = request.POST["ex_board"]
enteredTags = request.POST["tags"]
if not enteredTags:
tagArray = []
else:
#Get tags for the tack
tagArray = []
tagsSplit = enteredTags.split()
print tagsSplit
for ts in tagsSplit:
print ts
tagArray.append(str(ts))
print tagArray
print str(tagArray)
if file_input:
try:
im = Image.open(request.FILES["file"])
tackFileType = "image"
except IOError:
tackFileType = "not image"
tack_file = file_input
elif img_url:
r = requests.get(img_url)
img_temp = NamedTemporaryFile(delete=True)
img_temp.write(r.content)
img_temp.flush()
parsed_uri = urlparse(img_url)
domain = '{uri.scheme}://{uri.netloc}/'.format(uri=parsed_uri)
print domain
tack_file = File(img_temp)
tackFileType = "fromurl"
#Save tack details
TackImages(file_name=request.POST["tack_name"],
tack_file=tack_file,
file_type=tackFileType,
tags=tagArray,
username=get_user(request),
bookmark=request.POST["tack_url"],
board=boardName).save()
#Add tack to board
board = Boards.objects.get(name=boardName)
tack_name = request.POST["tack_name"]
board.tacks.append(tack_name)
board.save()
return redirect("/board?boardName=" + boardName)
def save_image_from_url(model, url):
try:
r = requests.get(url)
from urllib import parse
filename = parse.urlparse(url).path.split('/')[-1]
img_temp = NamedTemporaryFile(delete=True)
img_temp.write(r.content)
img_temp.flush()
model.image.save("image.jpg", File(img_temp), save=True)
except:
pprint('sorry')
def get_image(self, image_url):
resp = requests.get(image_url)
if resp.status_code != 200:
print('Warning: %s does not exist' % image_url)
return None
img_tmp = NamedTemporaryFile(delete=False)
img_tmp.write(resp.content)
img_tmp.flush()
img_tmp.close()
filename = os.path.basename(image_url)
return File(open(img_tmp.name, 'rb'), name=filename)
def save_image_from_url(field, id):
r = requests.get(IMAGE_URL + id + '.png', stream=True)
if r.status_code == requests.codes.ok:
img_temp = NamedTemporaryFile(delete=True)
img_temp.write(r.content)
img_temp.flush()
field.save(id + '.png', File(img_temp), save=True)
return True
return False
def get_image_data(image_url):
""" 抓取图片 """
content = requests.get(image_url, stream=True)
if content.status_code == 200:
img_name = image_url.split('/')[-1]
img_temp = NamedTemporaryFile()
img_temp.write(content.content)
img_temp.flush()
return img_name, File(img_temp)
else:
return None, None
def save_photo(self, photo_url):
"""Save initiator's photo."""
r = self.download_file(photo_url)
if r and r.status_code == 200:
img_temp = NamedTemporaryFile(delete=True)
img_temp.write(r.content)
img_temp.flush()
filename = photo_url.split('/')[-1]
self.photo.save(filename, File(img_temp))
return None
def fetch_file(self):
"""
Download the file from iati_import.url and store it in iati_import.local_file.
"""
tmp_file = NamedTemporaryFile(delete=True)
tmp_file.write(urllib.request.urlopen(self.iati_import.url, timeout=100).read())
tmp_file.flush()
filename = 'iati_import_%s.xml' % str(self.pk)
self.iati_xml_file.save(filename, File(tmp_file))
self.add_log('Downloaded file from %s' % str(self.iati_import.url),
LOG_ENTRY_TYPE.INFORMATIONAL)
def download_from_url_to_temp_file(url):
success = False
lf = NamedTemporaryFile(delete=True)
request = requests.get(url, stream=True)
if request.status_code == requests.codes.ok:
for block in request.iter_content(1024 * 8):
if not block:
break
lf.write(block)
success = True
return success, lf
def download_image(self, link):
if link:
temp_bg = NamedTemporaryFile()
bg_file = requests.get(link, stream=True)
temp_bg.write(bg_file.content)
return Image.open(temp_bg)
else:
print('new img created')
return Image.new('RGBA',
(YOUTUBE_WIDTH, YOUTUBE_HEIGHT), (0, 0, 0, 0)
)
def extract_instagram(item):
location = None
if item.get("lat") is not None and item.get("lng") is not None:
location = {
"lat": item.get("lat"),
"lng": item.get("lng")
}
caption = None if item.get("caption") is None else item.get("caption").get("text")
user_raw = item.get("user")
images = []
if item.get("image_versions2") or item.get("image"):
images = [extract_ig_media(item)]
if item.get("carousel_media"):
x = item.get("carousel_media")
images = list(map(extract_ig_media, x))
user = {
"instagram_id": user_raw.get("pk"),
"full_name": user_raw.get("full_name"),
"username": user_raw.get("username"),
}
ig_user = IGUser.objects.filter(instagram_id=user.get("instagram_id")).first()
if ig_user is None:
ig_user = IGUser.objects.create(
instagram_id=user_raw.get("pk"),
full_name=user_raw.get("full_name"),
username=user_raw.get("username"),
)
tags = [word.replace('#', '') for word in caption.split() if word.startswith('#')]
ig_post = IGPost.objects.filter(instagram_id=item.get("pk")).first()
if ig_post is None:
ig_post = IGPost.objects.create(
instagram_id=item.get("pk"),
tags=tags,
user=ig_user,
caption=caption,
coordinate=location,
time_posted=datetime.fromtimestamp(item.get("taken_at"))
)
for url in images:
temp = NamedTemporaryFile(delete=True)
temp.write(urlopen(url).read())
name = urlparse(url).path.split('/')[-1]
ext = name.split('.')[-1]
if ext in ['jpg', 'jpeg', 'png']:
temp.flush()
img = Media(title=name)
img.path.save(name, File(temp))
ig_post.photos.add(img)
ig_post.save()
return ig_post
def _generate(self, width, height, theme):
url = 'http://lorempixel.com/%s/%s/' % (width, height)
if theme:
url += '%s/' % theme
response = requests.get(url)
img_temp = NamedTemporaryFile(delete=True)
img_temp.write(response.content)
img_temp.flush()
return 'lorem_ipsum.jpg', File(img_temp)
def get_image(self, url):
response = requests.get(url)
data = response.content
img_temp = NamedTemporaryFile(delete=True)
img_temp.write(data)
img_temp.flush()
image_file = File(img_temp)
file_name = ntpath.basename(url)
return (file_name, image_file)
def downloadFile(url):
img_temp = NamedTemporaryFile(delete=True)
req = urllib2.Request(
url,
headers={
'User-Agent':
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.94 Safari/537.36'
})
img_temp.write(urllib2.urlopen(req).read())
img_temp.flush()
return ImageFile(img_temp)
def init_db(self):
files = DumpFile.get_last_of_type(DumpFile.TYPE_SQL, node=None)
self.assertTrue(files)
sql_dump_file = files[0]
f = NamedTemporaryFile(delete=False)
f.write(sql_dump_file.file.read())
f.seek(0)
f.close()
proxy.initialize(peewee.SqliteDatabase(f.name))
proxy.create_tables([Metadatos], safe=True)
def save(self, *args, **kwargs):
if self.image and not self.image_field:
try:
img_temp = NamedTemporaryFile(delete=True)
img_temp.write(urllib2.urlopen(self.image).read())
img_temp.flush()
self.image_field.save(
self.image.split('/')[-1].split('#')[0].split('?')[0],
File(img_temp))
except Exception as ex:
pass
super(MarketNewsItemData, self).save(*args, **kwargs)
def item_save_image_field(sender, instance, *args, **kwargs):
img_temp = NamedTemporaryFile(delete=True)
if "http" not in instance.image:
response = requests.get(settings.MEDIA_HOST + instance.image)
else:
response = requests.get(instance.image)
img_temp.write(response.content)
img_temp.flush()
file = File(img_temp)
extension = os.path.splitext(instance.image)[1]
instance.image_field.save(instance.slug + extension, file, False)
