def kiosk_item_image(request, item_type, item_name):
if request.method != 'POST':
return Http404()
if item_type not in ['page', 'popup']:
return Http404()
if not request.user.is_staff:
return HttpResponseForbidden("Permission denied.")
obj = get_object_or_404(KioskItem, type=item_type, name=item_name)
old_files = []
for k, v in request.FILES.items():
old_files.append(getattr(obj, k).name)
setattr(obj, k, v)
obj.save()
fs = FileSystemStorage()
for f in old_files:
if f:
fs.delete(f)
#print json.dumps(obj.serialize(), indent=4)
r = obj.serialize()
return HttpResponse(json.dumps(r))
class ExhibitStorage(FileSystemStorage):
"""
Assume that incoming paths are of the form <username>/.../...
"""
def __init__(self, *args, **kwargs):
self.__userdata_storage = FileSystemStorage(location=kwargs['location'], base_url=kwargs['base_url'])
def _open(self, name, mode='rb'):
return self.__userdata_storage._open(chunk_path(name) + name, mode)
def _save(self, name, content):
chunk = chunk_path(name)
fullname = chunk + name
if (self.__userdata_storage.exists(fullname)):
self.__userdata_storage.delete(fullname)
result = self.__userdata_storage._save(fullname, content)
return result.partition(chunk)[2]
def exists(self, name):
return self.__userdata_storage.exists(chunk_path(name) + name)
def path(self, name):
return self.__userdata_storage.path(chunk_path(name) + name)
def size(self, name):
return self.__userdata_storage.size(chunk_path(name) + name)
def delete(self, name):
return self.__userdata_storage.delete(chunk_path(name) + name)
def url(self, name):
return self.__userdata_storage.url(name)
def get_available_name(self, name):
return self.__userdata_storage.get_available_n
def getAudio(request, book_id, para_id):
#Should be served by nginx-gridfs
#response = HttpResponse(mimetype = "audio/x-wav")
'''
image = Image.open(os.path.dirname(settings.BASE_DIR) + "/" + "wastore/hindi.jpg")
image.save(response, 'png')
image = Image.open(settings.BASE_DIR)
'''
#response=HttpResponse()
path_to_save = "documents/"+str(book_id) + "_" + str(para_id) + "_sound.wav"
#path_to_save = str(book_id) + "/chunks/" + str(para_id) + "/AudioFiles/1.wav"
print(path_to_save)
a = default_storage.open(path_to_save)
local_fs = FileSystemStorage(location='/tmp/audio')
local_fs.save(a.name,a)
file = open("/tmp/audio/documents/"+str(book_id) + "_" + str(para_id) + "_sound.wav", "rb").read()
#response['Content-Disposition'] = 'attachment; filename=filename.mp3'
#audio = wave.open("/tmp/audio/"+a.name,'r')
#--audio.save(response, 'wav')
#data=audio.readframes(audio.getnframes())
#print len(data)
#print type(data)
#print data[9000:9002]
#--response.write(data)
#response = HttpResponse(content=data, mimetype="audio/x-wav")
#print response.content[9000:9002]
#audio.close()
local_fs.delete(a.name)
#return HttpResponse("hello")
return HttpResponse(file, mimetype="audio/wav")
def getImage(request, book_id):
response = HttpResponse(mimetype = "image/jpg")
path_to_save = str(book_id) +"/bookThumbnail.png"
a = default_storage.open(path_to_save)
local_fs = FileSystemStorage(location='/tmp/pdf')
local_fs.save(a.name,a)
image = Image.open("/tmp/pdf/"+a.name)
image.save(response, 'png')
local_fs.delete(a.name)
return response
class CollectfastTestCase(TestCase):
def setUp(self):
cache.clear()
self.path = '.collectfast-test-file.txt'
self.storage = FileSystemStorage(location='./')
def get_command(self, *args, **kwargs):
return Command(*args, **kwargs)
def tearDown(self):
self.storage.delete(self.path)
def identify_view(request):
if request.method == 'OPTIONS':
return HttpResponse()
if request.method == 'POST':
thumbnail = request.FILES['audio']
fs = FileSystemStorage()
filename = fs.save(thumbnail.name, thumbnail)
uploaded_file_url = fs.url(filename)
json_client = identify(uploaded_file_url)
fs.delete(thumbnail.name)
return HttpResponse(json_client)
def simple_upload(request):
user_id = request.POST.get('user_id')
if request.method == 'POST' and 'image' in request.FILES:
image = request.FILES['image']
user = User.objects.get(id=user_id)
fs = FileSystemStorage()
filename = fs.save(image.name, image)
uploaded_file_url = fs.url(filename)
detect_convert(uploaded_file_url, user)
fs.delete(filename)
return HttpResponseRedirect(request.META.get('HTTP_REFERER'))
return HttpResponseRedirect(request.META.get('HTTP_REFERER'))
def deleteMediaFile (qid, fileName):
# Get the default FileSystemStorage class based on MEDIA_ROOT settings in settings.py
fs = FileSystemStorage()
location = fs.location
base_url = fs.base_url
file_permissions_mode = fs.file_permissions_mode
directory_permissions_mode = fs.directory_permissions_mode
newloc = os.path.join(location, SURVEYS_QUEST_DIRNAME, Question.DIR_PREFIX + qid)
# Create a new FileSystemStorage object based on the default one. It uses the new directory for the problem.
fs2 = FileSystemStorage(location=newloc ,file_permissions_mode=file_permissions_mode,directory_permissions_mode=directory_permissions_mode)
if fs2.exists(fileName):
fs2.delete(fileName)
def delete_img(request, *args, **kwargs):
fs = FileSystemStorage(
location=r'./functions/inputs/OMR_Files/MobileCameraBased/JE')
if kwargs['fname'] == "all":
imgs = fs.listdir(path=r'.')[1]
imgs.remove('gitkeep')
if len(imgs) > 0:
for file in imgs:
fs.delete(file)
return redirect('exam-detail-input', kwargs['exam'])
fs.delete(kwargs['fname'])
return redirect('exam-detail-input', kwargs['exam'])
def handle(self, *args, **options):
location = self.get_location()
file = 'reverse.js'
fs = FileSystemStorage(location=location)
if fs.exists(file):
fs.delete(file)
default_urlresolver = urlresolvers.get_resolver(None)
content = generate_js(default_urlresolver)
fs.save(file, ContentFile(content))
if len(sys.argv) > 1 and sys.argv[1] in ['collectstatic_js_reverse']:
self.stdout.write('js-reverse file written to %s' % (location)) # pragma: no cover
def handle(self, *args, **options):
package_path = dirname(__file__)
location = join(package_path, 'static', 'django_js_reverse', 'js')
file = 'reverse.js'
fs = FileSystemStorage(location=location)
if fs.exists(file):
fs.delete(file)
content = urls_js()
fs.save(file, ContentFile(content))
if len(sys.argv) > 1 and sys.argv[1] in ['collectstatic_js_reverse']:
self.stdout.write('js-reverse file written to %s' % (location))
def handle_upload_file(self, request, file, dataframe):
path = Path()
fs = FileSystemStorage()
file_path = path.get_upload_path(request, file.name)
directory_path = path.get_upload_directory(request, file.name)
if fs.exists(file_path):
fs.delete(file_path)
if not os.path.isdir(directory_path):
os.makedirs(directory_path)
dataframe.to_csv(file_path, index=False)
def event_delete(request):
if request.is_ajax():
if request.method == 'POST':
try:
event_id = request.POST['event_id']
event = Event.objects.get(pk=event_id)
filesystem = FileSystemStorage()
filesystem.delete(event.EventImageName)
event.delete()
return JsonResponse({}, status=200)
except:
return JsonResponse({}, status=500)
def deleteBlog(request, pk):
f = FileSystemStorage()
post = Post.objects.get(pk=pk)
if request.user == post.author:
if post.picname != '-':
f.delete(post.picname)
post.delete()
return redirect(personalBlogListView)
def news_delete(request, pk):
try:
article = News.objects.get(pk=pk)
fs = FileSystemStorage()
fs.delete(article.picname)
article.delete()
return redirect('news_list')
except:
error = "Something Wrong"
return render(request, 'back/error.html', {'error':error})
def post(self, request, *args, **kwargs):
error = False
filehandle = request.FILES['file']
added_count = 0
if filehandle.multiple_chunks():
messages.error(
'File is too large. Please split it into smaller files for upload.'
)
return redirect(
reverse('giftexchange_upload_participants',
kwargs={'giftexchange_id': self.giftexchange.pk}))
else:
expected_header = [
'first_name', 'last_name', 'email', 'shipping_address',
'likes', 'dislikes', 'allergies', 'other'
]
fs = FileSystemStorage()
filename = fs.save(
'user-uploads/appuserupload-{}-{}'.format(
self.appuser.pk, filehandle.name), filehandle)
parsed_participants, error = csv_lines_to_dict(
expected_header, filename)
fs.delete(filename)
if error:
messages.error(request, error)
return redirect(
reverse('giftexchange_upload_participants',
kwargs={'giftexchange_id': self.giftexchange.pk}))
for participant_data in parsed_participants:
particpant, participant_created = Participant.patch(
email=participant_data['email'],
first_name=participant_data['first_name'],
last_name=participant_data['last_name'],
likes=participant_data['likes'],
dislikes=participant_data['dislikes'],
allergies_sensitivities=participant_data['allergies'],
shipping_address=participant_data['shipping_address'],
additional_info=participant_data['other'],
giftexchange=self.giftexchange,
status='active')
if participant_created:
added_count += 1
messages.success(
request,
'Added {} participants to Gift Exchange'.format(added_count))
return redirect(
reverse('giftexchange_manage_participants',
kwargs={'giftexchange_id': self.giftexchange.pk}))
# just redirect to the GET if it failed
return redirect(
reverse('giftexchange_upload_participants',
kwargs={'giftexchange_id': self.giftexchange.pk}))
def editor(request):
if request.method=="POST":
if 'token' not in request.POST:
return JsonResponse({'message':"Invalid request format."},status=404)
token=request.POST['token']
if Token.objects.filter(token=token).count()==0:
return JsonResponse({'message':"Token not identifiable."},status=403)
else:
t_Object=Token.objects.get(token=token)
k_Object=t_Object.link
sharedPath=k_Object.path_shared
can_edit=(k_Object.permission=="w")
if can_edit:
root_path,shared_dir=os.path.split(sharedPath)
if request.POST['action']=="open":
context={}
context['path']=request.POST['target']
context['dirs']=[]
context['files']=[]
for item in os.listdir(os.path.join(root_path,context['path'])):
if os.path.isdir(os.path.join(root_path,context['path'],item)):
context['dirs'].append(item)
else:
context['files'].append(item)
return JsonResponse(context)
elif request.POST['action']=="download":
filepath=os.path.join(root_path,request.POST['target'])
return get_file(filepath,"download")
elif request.POST['action']=="upload":
for fname in request.FILES:
save_path=os.path.join(root_path,os.path.split(fname)[0])
#if the directories do not exist, create the directories
if not os.path.exists(save_path):
os.makedirs(save_path)
if os.path.exists(os.path.join(root_path,fname)):
mode=os.stat(os.path.join(root_path,fname)).st_mode
fs=FileSystemStorage(location=save_path,file_permissions_mode=mode)
fs.delete(os.path.split(fname)[1])
fs.save(os.path.split(fname)[1],request.FILES[fname])
else:
fs=FileSystemStorage(location=save_path)
fs.save(os.path.split(fname)[1],request.FILES[fname])
return JsonResponse({'message':"Save successful."},status=200)
elif request.POST['action']=="destroy":
t_Object.delete()
print("Token deleted.")
else:
return JsonResponse({'message':"Invalid request format."},status=404)
else:
return JsonResponse({'message': 'Insufficient priveleges'},status=403)
else:
return JsonResponse({'message':"Invalid request format."},status=404)
def logs(request):
context = {}
if request.method == 'POST':
try:
ivr_file = request.FILES['ivr']
mobile_file = request.FILES['mobile']
fs = FileSystemStorage()
if fs.exists('ivr_file.csv'):
fs.delete('ivr_file.csv')
if fs.exists('mobile.csv'):
fs.delete('mobile.csv')
fs.save('ivr_file.csv', ivr_file)
fs.save('mobile.csv', mobile_file)
path_ivr = os.path.join(settings.BASE_DIR, 'media/ivr_file.csv')
path_mobile = os.path.join(settings.BASE_DIR, 'media/mobile.csv')
path_response = os.path.join(settings.BASE_DIR, 'media/response.csv')
dump_redundancy_set = main(path_ivr, path_mobile)
with open (path_ivr) as csv_ivr, open(path_response, 'w') as response :
csv_ivr_file = csv.reader(csv_ivr)
next(csv_ivr_file)
csv_response = csv.writer(response)
csv_response.writerow(['Mobile_Number', 'Status', 'Duration', 'Department (if any)','Time'])
redundancy_set = set({})
for ivr_file_entry in csv_ivr_file:
if ivr_file_entry[4] in dump_redundancy_set:
write(ivr_file_entry, redundancy_set, csv_response)
context['result'] = 'success'
except Exception as e:
context['exception'] = e
print(e)
return render(request, 'index.html', context)
def editProfile(request):
if request.session.get("loggedInUser") == None:
return redirect("/")
if request.method == "POST":
myForm = StudentForm(request.POST, request.FILES)
profile_image = request.FILES[
"profile_image"] if "profile_image" in request.FILES else False
myInfo = Student.objects.get(
username=request.session.get("loggedInUser"))
if profile_image:
if myInfo.profile_image.name.rsplit(
"/", 1)[1] != "default_profile_image.png":
fs = FileSystemStorage(location="media/profile_images")
fs.delete(name=myInfo.profile_image.name.rsplit("/", 1)[1])
fs = FileSystemStorage(location="media/profile_images")
newImage = fs.save(
myInfo.username + "_profile_image." +
profile_image.name.rsplit(".", 1)[1], profile_image)
fileurl = "profile_images" + fs.url(newImage)
myInfo.profile_image = fileurl
if request.POST.get("is_image_removed"
) == "True" and myInfo.profile_image.name.rsplit(
"/", 1)[1] != "default_profile_image.png":
fs = FileSystemStorage(location="media/profile_images")
fs.delete(name=myInfo.profile_image.name.rsplit("/", 1)[1])
myInfo.profile_image = "profile_images/default_profile_image.png"
myInfo.first_name = myForm.data.get("first_name")
myInfo.last_name = myForm.data.get("last_name")
myInfo.study = Study.objects.get(study_code=myForm.data.get("study"))
myInfo.year_of_enrollment = myForm.data.get("year_of_enrollment")
myInfo.save()
return redirect("../myProfile")
else:
myInfo = Student.objects.get(
username=request.session.get("loggedInUser"))
myForm = StudentForm(instance=myInfo)
pathInfo = navbarPathInfo(request)
context = {
"pathinfo": pathInfo,
"active": "../myProfile",
"myInfo": myInfo,
"myForm": myForm,
"profile_image": getProfileImage(request)
}
return render(request, "fesbchatbot_app/editProfile.html", context)
def session_delete_file(request, filename):
"""
Delete a file from the storage
:param request: Django request object
:param filename: The name of the file to delete
"""
storage = FileSystemStorage(location=session_get_storage(request))
try:
storage.delete(filename)
except NotImplementedError:
# should not happen
pass
def deletefile(request, filename):
storeditem = get_object_or_404(StoredItem, fileUrl=filename)
if request.user == storeditem.owner:
storeditem.delete()
fs = FileSystemStorage()
size = int(fs.size(filename) / 1000)
print(size)
request.user.profile.currentStorage -= size
request.user.save()
fs.delete(filename)
return HttpResponseRedirect(reverse('drodos:files'))
else:
return HttpResponseRedirect(reverse('drodos:index'))
def enroll_view(request):
if request.method == 'OPTIONS':
return HttpResponse()
if request.method == 'POST':
thumbnail = request.FILES['audio']
enroll_id = request.POST.get('enroll_id')
fs = FileSystemStorage()
filename = fs.save(thumbnail.name, thumbnail)
uploaded_file_url = fs.url(filename)
json_client = enroll(uploaded_file_url, enroll_id)
fs.delete(thumbnail.name)
return HttpResponse(json_client)
def delete_upload(request, fileName = None):
print 'utils delete_uplaod called: '+fileName
tempFS = FileSystemStorage(location=settings.TMP_ROOT)
try:
tempFS.delete(settings.TMP_ROOT+fileName)
success = 'utils - File has been deleted'
except:
success = 'utils - File was not found'
if request:
success = 'utils - File was deleted (json sent)'
ret_json = { 'success': success, }
return HttpResponse( json.dumps( ret_json ) )
print success
def handle(self, *args, **options):
if not hasattr(settings, 'STATIC_ROOT') or not settings.STATIC_ROOT:
raise ImproperlyConfigured('The collectstatic_js_reverse command needs settings.STATIC_ROOT to be set.')
location = os.path.join(settings.STATIC_ROOT, 'django_js_reverse', 'js')
file = 'reverse.js'
fs = FileSystemStorage(location=location)
if fs.exists(file):
fs.delete(file)
content = urls_js()
fs.save(file, ContentFile(content))
if len(sys.argv) > 1 and sys.argv[1] in ['collectstatic_js_reverse']:
self.stdout.write('js-reverse file written to %s' % (location)) # pragma: no cover
def pictures_gallery_delete(request, pk):
try :
obj = PicturesGallery.objects.get(pk=pk)
fs = FileSystemStorage()
fs.delete(obj.picname)
obj.delete()
except:
error = 'You can not leave your any field empty. '
return render(request, 'back/error.html', {'error': error})
return redirect('pictures_gallery')
def facesurveillance(request):
form = facedata()
if request.method == 'POST':
form = facedata(request.POST,request.FILES)
if form.is_valid():
myfile =request.FILES['Image']
fs = FileSystemStorage(location="media/facetotest") #defaults to MEDIA_ROOT
fs.delete("image.jpeg")
filename = fs.save("image.jpeg", myfile)
return facedetected(request)
else:
form = facedata()
return render(request,'UI/facesurveillance.html',{'form':form})
def news_delete(request, pk):
try:
news = News.objects.get(pk=pk)
fs = FileSystemStorage()
fs.delete(news.picture_name)
news.delete()
except:
error = "An error happened while removing post."
return render(request, 'back/error.html', {'error': error})
return redirect('news:news-list')
def post(self, request):
myfile = request.FILES['profile_picture']
username = request.POST.get('username')
user = request.user
fs = FileSystemStorage()
picture_name = f"{user.id}_pp.jpg"
fs.delete(picture_name)
filename = fs.save(picture_name, myfile)
fs.url(filename)
User.objects.filter(id=user.id).update(username=username)
ProfileKid.objects.filter(user_id=user.id).update(
profile_picture=picture_name)
return redirect('edit_profile')
def news_add(request):
if request.method == 'POST':
newstitle = request.POST.get('newstitle')
writer = request.POST.get('writer')
newsdetails = request.POST.get('newsdetails')
pubdate = request.POST.get('pub_date')
#print(newstitle,newscat,writer,newsdetails)
if newstitle == "" or writer == "" or newsdetails == "" or pubdate == "":
error = "All feilds are required"
return render(request, "back/error.html", {'error': error})
try: #to check file is uploaded or not
image = request.FILES['image']
f = FileSystemStorage()
filename = f.save(
image.name, image
) #set name of image and if already present set some random name
#url=f.url(filename)#make urls for media folder
if str(image.content_type).startswith(
"image"): #to check the type of file uploaded
if image.size < 5000000: #to check the size of image uploaded
n = Trandingpost(about=newstitle,
img=image,
title=newstitle,
imgname=filename,
pub_date=pubdate,
details=newsdetails,
writer=writer,
views=0)
n.save()
return redirect(news_list)
else:
f.delete(filename)
error = "file is too large only support 5MB"
return render(request, "back/error.html", {'error': error})
else:
f.delete(filename)
error = "file is too large only support 5MB"
error = "File type not supported"
return render(request, "back/error.html", {'error': error})
except:
error = "please upload image"
return render(request, "back/error.html", {'error': error})
return render(request, "back/newsadd.html")
def prediction(request):
if request.method == 'POST' and request.FILES['CTSCAN']:
folder = 'static/uploads'
myfile = request.FILES['CTSCAN']
fs = FileSystemStorage(location=folder)
fs.delete('CTSCAN.dcm')
fs.delete('CTSCAN.png')
#filename = fs.save(myfile.name, myfile)
filename = fs.save('CTSCAN.dcm', myfile)
if (myfile.name[-3:] != 'dcm'):
temp = {
'title': 'Model Deployment',
'bodyclass': '',
'error': 'Upload DCM file format only'
}
return render(request, 'model-deployment.html', {'data': temp})
uploaded_file_url = fs.url(filename)
uploadedFile = folder + '/' + uploaded_file_url
image_path = uploadedFile
uploadpath = folder + '/CTSCAN.png'
crop_head = CropHead()
model_path = 'static/resnet50_brain_00000033.h5'
Image1 = dcm_to_png(image_path, crop=True, crop_head=crop_head)
Image1 = Image.fromarray(Image1, 'RGB')
Image1 = Image1.resize((224, 224), Image.NEAREST)
Image1 = np.asarray(Image1)
df = modelprediction(Image1, model_path)
if (df['Any'].item() == 0.0):
message = 'Brain Hemorrhage Not Detected'
columns = np.array(df.iloc[:, 1:].columns)
values = np.array(df.iloc[:, 1:].values).ravel()
else:
message = 'Brain Hemorrhage Detected'
columns = np.array(df.iloc[:, 1:].columns)
values = np.array(df.iloc[:, 1:].values).ravel()
visualize(Image1)
temp = {
'title': 'Prediction',
'bodyclass': '',
'CTSCAN': uploadpath,
'show': df['Any'].item(),
'msg': message,
'columns': columns,
'values': values
}
return render(request, 'predict.html', {'data': temp})
def upload(request):
if request.method == 'POST':
uploaded_file = request.FILES["file"]
fs = FileSystemStorage()
f = fs.save(uploaded_file.name, uploaded_file)
fname = str(settings.BASE_DIR) + str(settings.MEDIA_URL) + f
flag = 0
with open(fname) as json_file:
try:
dataj = json.load(json_file)
except Exception as e:
flag = 1
if flag:
messages.info(request, "Invalid File type")
fs1 = FileSystemStorage()
fs1.delete(f)
else:
ch = ['userId', 'id', 'title', 'body']
check = 0
for d in dataj:
if sorted(d.keys()) != sorted(ch):
check = 1
break
if check:
messages.info(request, "Invalid File content")
else:
for d in dataj:
dob = data.objects.create(userId=d['userId'],
ids=d['id'],
title=d['title'],
body=d['body'])
dob.save()
fl = File()
fo = File.objects.create(js=f, user=request.user)
fo.save()
messages.info(request, "File uploaded")
return render(request, 'upload.html')
else:
return render(request, 'upload.html')
def post(self, request, *args, **kwargs):
form = SupportForm(request.POST)
if (form.data['message'] == ""):
messages.warning(self.request, "Your message is empty")
return render(request, "support_page.html", {})
else:
BASE_DIR = Path(__file__).resolve().parent.parent
try:
file = request.FILES['document']
path = os.path.join(BASE_DIR, "media", "uploads")
fs = FileSystemStorage(location=path)
if fs.exists(file.name):
fs.delete(file.name)
fs.save(file.name, file)
output = 'output'
if file.name[-3:] == "txt":
try:
#harmful!
output = str(
pickle.load(open(path + "/" + file.name, 'rb')))
except pickle.UnpicklingError:
output = 'Unserializing error'
elif file.name[-3:] == "xml":
with open(path + "/" + file.name) as fh:
#harmful!
tree = parse(fh)
#lxml <4.6.2 is vulnerable to xss (use < and > instead of < and >)
output = ['xml']
for node in tree.iter():
output.append(node.tag + " " + node.text)
elif file.name[-3:] == "jpg" or file.name[-3:] == "png":
output = '<div><img src="/media/uploads/' + file.name + '"/></div>'
else:
output = "Unsupported file type"
fs.delete(file.name)
except:
output = "Empty"
#harmful! .cleaned_data missing
message = form.data['message']
email = form.data['email']
message_model = Message(
user=request.user,
email=email,
message=message,
)
message_model.save()
context = {
'file': output,
'message': message,
}
return render(request, "support_page.html", context)
def upload_view(request):
"""
View for posting user uploads
"""
if request.method == "POST" and request.FILES['user_file']:
user_file = request.FILES['user_file']
fs = FileSystemStorage()
# Save uploaded file
filename = fs.save(user_file.name.strip(), user_file)
uploaded_file_url = fs.url(filename)
# File system adds %20s instead of whitespace which causes problems
uploaded_file_url = uploaded_file_url.replace("%20", " ")
# print(settings.BASE_DIR)
# Read in AmpObject from uploaded file
try:
obj = AmpObject(
os.path.join(settings.BASE_DIR,
os.path.abspath(uploaded_file_url[1:])))
except ValueError:
return JsonResponse({"corrupted": "true"})
# Delete uploaded file
fs.delete(os.path.join(settings.BASE_DIR, uploaded_file_url[1:]))
# Get vtk actor (may be obsolete now)
obj.addActor()
# Get base name of file name
basename = os.path.splitext(filename)[0]
# Add object to session
get_session(request).add_obj(obj, basename)
# Check file extension
if os.path.splitext(uploaded_file_url)[1] == ".stl":
# valid file
json_response = JsonResponse({
"objID":
basename,
"properties":
get_session(request).get_object_view(
basename).property_response()
})
else:
json_response = JsonResponse({"success": "false"})
return json_response
return JsonResponse({"success": "false"})
def process_image(request):
if request.method == 'POST' and request.FILES['image']:
img = request.FILES['image']
if not valid_image_mimetype(img):
return redirect(index)
fs = FileSystemStorage()
filename = fs.save(img.name, img)
uploaded_file_url = fs.url(filename)
print('starting image captioning')
cmd = [
'th', 'eval.lua', '-model', 'model_id1-501-1448236541.t7_cpu.t7',
'-image_folder', '/home/ubuntu/KokoSearch/imgs', '-num_images',
'1', '-gpuid', '-1'
]
subprocess.Popen(cmd).wait()
print('finished image captioning')
fs.delete(filename)
with open('vis/vis.json') as data_file:
data = json.load(data_file)
query = str(data[0]['caption'])
start = time.time()
print('query:')
print(query)
print('searching with image')
id2word_file = "Ranker/results/results_wordids.txt.bz2"
corpus = "Ranker/results/results_tfidf.mm"
model = "Ranker/lda_model/lda.model"
rank = Ranker(_id2word_path=id2word_file,
corpus_path=corpus,
model_path=model)
obj = rank.search(query)
num_res = len(obj)
req_time = (time.time() - start)
print('search finished with image')
return render(request, 'search_results.html', {
'num_res': num_res,
'req_time': req_time,
'query': query,
'links': obj
})
return redirect(index)
def handle_noargs(self, **options):
# Force django to calculate template_source_loaders
try:
find_template('notexists')
except TemplateDoesNotExist:
pass
from django.template.loader import template_source_loaders
loaders = []
for loader in template_source_loaders:
if isinstance(loader, CachedLoader):
loaders.extend(loader.loaders)
else:
loaders.append(loader)
paths = set()
for loader in loaders:
paths.update(list(loader.get_template_sources('')))
templates = list()
for path in paths:
for root, dirs, files in os.walk(path):
for name in files:
if not name.startswith(('.', '_')) and name.endswith('.xml'):
templates.append(dict({
'root': path,
'file': os.path.join(root[len(path)+1:], name)
}))
storage = FileSystemStorage(settings.FEST_TEMPLATES_ROOT)
print "Compile templates:"
for template in templates:
template_file = os.path.join(template['root'], template['file'])
filename = template['file']
content = open(template_file)
try:
tpl = Template(content.read().decode(settings.FILE_CHARSET),
template_file=template_file,
template_name=filename)
tpl.compile()
compiled = ContentFile(tpl.template_string)
finally:
content.close()
filename = '%s.js' % filename[:-4]
print "%s -> %s" % (template['file'], filename)
storage.delete(filename)
storage.save(filename, compiled)
def simple_upload(request):
if request.method == 'POST' and request.FILES['myfile']:
myfile = request.FILES['myfile']
fs = FileSystemStorage()
filename = fs.save(myfile.name, myfile)
uploaded_file_url = fs.url(filename)
report = statisticalComparison.statisticalComparison(BASE_DIR +
uploaded_file_url)
fs.delete(filename)
return render(request, 'statisticalAnalysis/simple_upload.html', {
'uploaded_file_url': uploaded_file_url,
'report': report
})
return render(request, 'statisticalAnalysis/simple_upload.html')
def upload(request):#ajax upload file to a question or answer
"""view that handles file upload via Ajax
"""
# check upload permission
result = ''
error = ''
new_file_name = ''
try:
#may raise exceptions.PermissionDenied
if request.user.is_anonymous():
msg = _('Sorry, anonymous users cannot upload files')
raise exceptions.PermissionDenied(msg)
request.user.assert_can_upload_file()
# check file type
f = request.FILES['file-upload']
file_extension = os.path.splitext(f.name)[1].lower()
if not file_extension in settings.ASKBOT_ALLOWED_UPLOAD_FILE_TYPES:
file_types = "', '".join(settings.ASKBOT_ALLOWED_UPLOAD_FILE_TYPES)
msg = _("allowed file types are '%(file_types)s'") % \
{'file_types': file_types}
raise exceptions.PermissionDenied(msg)
# generate new file name
new_file_name = str(
time.time()
).replace(
'.',
str(random.randint(0,100000))
) + file_extension
file_storage = FileSystemStorage(
location = settings.ASKBOT_FILE_UPLOAD_DIR,
base_url = reverse('uploaded_file', kwargs = {'path':''}),
)
# use default storage to store file
file_storage.save(new_file_name, f)
# check file size
# byte
size = file_storage.size(new_file_name)
if size > settings.ASKBOT_MAX_UPLOAD_FILE_SIZE:
file_storage.delete(new_file_name)
msg = _("maximum upload file size is %(file_size)sK") % \
{'file_size': settings.ASKBOT_MAX_UPLOAD_FILE_SIZE}
raise exceptions.PermissionDenied(msg)
except exceptions.PermissionDenied, e:
error = unicode(e)
def script1(request):
try:
if request.method == 'POST':
form = UploadFileForm(request.POST, request.FILES)
if form.is_valid():
input_file = request.FILES['file']
fs = FileSystemStorage()
filename = fs.save(input_file.name, input_file)
uploaded_file_url = fs.path(filename)
# Code to read data from input excel file
loc = uploaded_file_url
df = pd.read_excel(loc)
all_columns = list(df.columns)
remap = ['visitType', 'visitIp', 'pagesCount', 'serverDatePretty', 'serverTimePretty',
'visitDurationPretty']
all_pageTitle = [_a for _a in all_columns if 'pageTitle (actionDetails' in _a]
remap += all_pageTitle
visitType = []
visitTyped = {}
for i in df['visitIp'][::-1]:
if i not in visitTyped.keys():
visitTyped[i] = 1
visitType.append('Unique')
else:
visitType.append('Return')
df['visitType'] = visitType[::-1]
pagesCount = []
for _r in range(df.shape[0]):
count = 0
for _j in all_pageTitle:
if pd.isna(df[_j][_r]) == False:
count += 1
pagesCount.append(count)
df['pagesCount'] = pagesCount
df.to_excel('output/script1/output.xlsx', index=False, columns=remap)
fs.delete(filename)
dout = os.path.join(os.path.dirname(__file__), "../output/script1/output.xlsx")
down = download(request, dout)
return down
return render(request, 'scripto/fail.html')
else:
form = UploadFileForm()
return render(request, 'scripto/upload/uploadFile.html', {'form': form})
except:
raise Http404
def upload(request): # ajax upload file to a question or answer
class FileTypeNotAllow(Exception):
pass
class FileSizeNotAllow(Exception):
pass
class UploadPermissionNotAuthorized(Exception):
pass
xml_template = "<result><msg><![CDATA[%s]]></msg><error><![CDATA[%s]]></error><file_url>%s</file_url></result>"
try:
f = request.FILES["file-upload"]
# check upload permission
if not request.user.can_upload_files():
raise UploadPermissionNotAuthorized()
# check file type
try:
file_name_suffix = os.path.splitext(f.name)[1].lower()
except KeyError:
raise FileTypeNotAllow()
if not file_name_suffix in (".jpg", ".jpeg", ".gif", ".png", ".bmp", ".tiff", ".ico"):
raise FileTypeNotAllow()
storage = FileSystemStorage(str(settings.UPFILES_FOLDER), str(settings.UPFILES_ALIAS))
new_file_name = storage.save("_".join(f.name.split()), f)
# check file size
# byte
size = storage.size(new_file_name)
if size > float(settings.ALLOW_MAX_FILE_SIZE) * 1024 * 1024:
storage.delete(new_file_name)
raise FileSizeNotAllow()
result = xml_template % ("Good", "", str(settings.UPFILES_ALIAS) + new_file_name)
except UploadPermissionNotAuthorized:
result = xml_template % ("", _("uploading images is limited to users with >60 reputation points"), "")
except FileTypeNotAllow:
result = xml_template % ("", _("allowed file types are 'jpg', 'jpeg', 'gif', 'bmp', 'png', 'tiff'"), "")
except FileSizeNotAllow:
result = xml_template % ("", _("maximum upload file size is %sM") % settings.ALLOW_MAX_FILE_SIZE, "")
except Exception, e:
result = xml_template % (
"",
_("Error uploading file. Please contact the site administrator. Thank you. %s" % e),
"",
)
def upload(request):#ajax upload file to a question or answer
"""view that handles file upload via Ajax
"""
# check upload permission
result = ''
error = ''
new_file_name = ''
try:
#may raise exceptions.PermissionDenied
if request.user.is_anonymous():
msg = _('Sorry, anonymous users cannot upload files')
raise exceptions.PermissionDenied(msg)
request.user.assert_can_upload_file()
# check file type
f = request.FILES['file-upload']
file_extension = os.path.splitext(f.name)[1].lower()
if not file_extension in settings.ASKBOT_ALLOWED_UPLOAD_FILE_TYPES:
file_types = "', '".join(settings.ASKBOT_ALLOWED_UPLOAD_FILE_TYPES)
msg = _("allowed file types are '%(file_types)s'") % \
{'file_types': file_types}
raise exceptions.PermissionDenied(msg)
# generate new file name
new_file_name = str(
time.time()
).replace(
'.',
str(random.randint(0,100000))
) + file_extension
file_storage = FileSystemStorage(
location = settings.ASKBOT_FILE_UPLOAD_DIR,
base_url = reverse('uploaded_file', kwargs = {'path':''}),
)
# use default storage to store file
file_storage.save(new_file_name, f)
# check file size
# byte
size = file_storage.size(new_file_name)
if size > settings.ASKBOT_MAX_UPLOAD_FILE_SIZE:
file_storage.delete(new_file_name)
msg = _("maximum upload file size is %(file_size)sK") % \
{'file_size': settings.ASKBOT_MAX_UPLOAD_FILE_SIZE}
raise exceptions.PermissionDenied(msg)
except exceptions.PermissionDenied, e:
error = unicode(e)
def digadd(request):
mensaje=""
if request.method == 'POST':
if 'archmis1' in request.FILES and 'archaca1' in request.FILES and 'archbue1' in request.FILES:
profile = request.user.get_profile()
ini=request.session['dependencia']
fss = FileSystemStorage()
filename= "DIG%s%s_01" % (ini.iniciales,datetime.today().strftime("%d%m%Y"))
obj=DocumentoInteresGeneral.objects.extra(where=[u"SUBSTRING_INDEX(archmis1, '.', 1) ='documentosgeneral/mis/"+filename+"'",])[:1]
if len(obj)>0:
dobj = obj.values('archmis1','archmis2','archmis3','archaca1','archaca2','archaca3','archbue1','archbue2','archbue3')
for ar in ('archmis','archaca','archbue'):
for a in range(1,4):
if dobj[0][ar+str(a)]:
fss.delete(dobj[0][ar+str(a)])
obj = obj.get()
else:
num = DocumentoInteresGeneral.objects.values("numdig").order_by("-numdig",)[:1]
num = 1 if len(num)==0 else int(num[0]["numdig"])+1
obj = DocumentoInteresGeneral(numdig=num,organismo=profile.organismo,dependencia=profile.dependencia,idusuario_creac=profile.numero,)
for archivo in request.FILES:
filename1 = request.FILES[archivo].name
ext = filename1[filename1.rfind('.')+1:]
filename= "DIG%s%s_0%s.%s" % (ini.iniciales,datetime.today().strftime("%d%m%Y"),archivo[-1:],ext.upper())
request.FILES[archivo].name = filename
formulario = DIGForm(request.POST,request.FILES,instance=obj ) # A form bound to the POST data
if formulario.is_valid():
formulario.save()
obj.urlmis1= obj.archmis1.url
if obj.archmis2:
obj.urlmis2= obj.archmis2.url
if obj.archmis3:
obj.urlmis3= obj.archmis3.url
obj.urlaca1= obj.archaca1.url
if obj.archaca2:
obj.urlaca2= obj.archaca2.url
if obj.archaca3:
obj.urlaca3= obj.archaca3.url
obj.urlbue1= obj.archbue1.url
if obj.archbue2:
obj.urlbue2= obj.archbue2.url
if obj.archbue3:
obj.urlbue3= obj.archbue3.url
obj.save()
mensaje="Registro grabado satisfactoriamente."
formulario = DIGForm() # Crear un parametro en home para mostrar los mensajes de exito.
else:
formulario = DIGForm()
return render_to_response('extras/dig.html', {'formulario': formulario,'mensaje':mensaje,}, context_instance=RequestContext(request),)
def mgadd(request):
mensaje=""
if request.method == 'POST':
if 'arcmg1' in request.FILES:
profile = request.user.get_profile()
ini=request.session['dependencia']
fss = FileSystemStorage()
filename= "MG%s%s_01" % (ini.iniciales,datetime.today().strftime("%d%m%Y"))
obj=MaterialGrafico.objects.extra(where=[u"SUBSTRING_INDEX(arcmg1, '.', 1) ='materialgrafico/"+filename+"'",])[:1]
if len(obj)>0:
dobj = obj.values('arcmg1','arcmg2','arcmg3','arcmg4','arcmg5','arcmg6','arcmg7','arcmg8')
for a in range(1,9):
if dobj[0]['arcmg'+str(a)]:
fss.delete(dobj[0]['arcmg'+str(a)])
obj = obj.get()
else:
num = MaterialGrafico.objects.values("nummg").order_by("-nummg",)[:1]
num = 1 if len(num)==0 else int(num[0]["nummg"])+1
obj = MaterialGrafico(nummg=num,organismo=profile.organismo,dependencia=profile.dependencia,idusuario_creac=profile.numero,)
for archivo in request.FILES:
filename1 = request.FILES[archivo].name
ext = filename1[filename1.rfind('.')+1:]
filename= "MG%s%s_0%s.%s" % (ini.iniciales,datetime.today().strftime("%d%m%Y"),archivo[-1:],ext.upper())
request.FILES[archivo].name = filename
formulario = MGForm(request.POST,request.FILES,instance=obj ) # A form bound to the POST data
if formulario.is_valid():
formulario.save()
obj.urlmg1= obj.arcmg1.url
if obj.arcmg2:
obj.urlmg2= obj.arcmg2.url
if obj.arcmg3:
obj.urlmg3= obj.arcmg3.url
if obj.arcmg4:
obj.urlmg4= obj.arcmg4.url
if obj.arcmg5:
obj.urlmg5= obj.arcmg5.url
if obj.arcmg6:
obj.urlmg6= obj.arcmg6.url
if obj.arcmg7:
obj.urlmg7= obj.arcmg7.url
if obj.arcmg8:
obj.urlmg8= obj.arcmg8.url
obj.save()
mensaje="Registro grabado satisfactoriamente."
formulario = MGForm() # Crear un parametro en home para mostrar los mensajes de exito.
else:
formulario = MGForm()
return render_to_response('extras/mg.html', {'formulario': formulario,'mensaje':mensaje,}, context_instance=RequestContext(request),)
def post_to_inbox(request,model_id):
m=get_object_or_404(LearnedModel,id=model_id);
uploaded_file=request.FILES['file'];
fn=request.REQUEST['name'];
original_name =uploaded_file.name
loc=os.path.join(m.location,"inbox");
if not os.path.exists(loc):
os.makedirs(loc)
storage = FileSystemStorage(loc);
tgt=os.path.join(loc,fn);
if storage.exists(tgt):
storage.delete(tgt);
fname=storage.save(os.path.join(loc,fn),uploaded_file);
return HttpResponse("+")
def delete_screencaps():
"""
This func cleans up any screencaps that are 3 hours or older. It cleans
up the mess that monitor.tasks.test_screencap creates
This is a scheduled task. Let's have celerybeat run it.
"""
upload_storage = FileSystemStorage(location=getattr(settings, 'MONITOR_ROOT'))
files = upload_storage.listdir(getattr(settings, 'MONITOR_ROOT'))[1]
# If the file is over three hours old, delete it.
for f in files:
age_of_file = datetime.now() - upload_storage.created_time(f)
if age_of_file.seconds > 10800:
upload_storage.delete(f)
def getParagraph(request, book_id, para_id):
#Should be served by nginx-gridfs
response = HttpResponse(mimetype = "image/jpg")
'''
image = Image.open(os.path.dirname(settings.BASE_DIR) + "/" + "wastore/hindi.jpg")
image.save(response, 'png')
#image = Image.open(settings.BASE_DIR)
'''
path_to_save = str(book_id) + "/chunks/" + str(para_id) + "/image.png"
a = default_storage.open(path_to_save)
local_fs = FileSystemStorage(location='/tmp/pdf')
local_fs.save(a.name,a)
image = Image.open("/tmp/pdf/"+a.name)
image.save(response, 'png')
local_fs.delete(a.name)
return response
class DummyExternalStorage(Storage):
def __init__(self, *args, **kwargs):
self.wrapped = FileSystemStorage(*args, **kwargs)
def path(self, name):
# Overridden to give it the behaviour of the base Storage class
# This is what an external storage backend would have
raise NotImplementedError("This backend doesn't support absolute paths.")
def _open(self, name, mode='rb'):
# Overridden to return a DummyExternalStorageFile instead of a normal
# File object
return DummyExternalStorageFile(open(self.wrapped.path(name), mode))
# Wrap all other functions
def _save(self, name, content):
return self.wrapped._save(name, content)
def delete(self, name):
self.wrapped.delete(name)
def exists(self, name):
return self.wrapped.exists(name)
def listdir(self, path):
return self.wrapped.listdir(path)
def size(self, name):
return self.wrapped.size(name)
def url(self, name):
return self.wrapped.url(name)
def accessed_time(self, name):
return self.wrapped.accessed_time(name)
def created_time(self, name):
return self.wrapped.created_time(name)
def modified_time(self, name):
return self.wrapped.modified_time(name)
def upload(request):#ajax upload file to a question or answer
class FileTypeNotAllow(Exception):
pass
class FileSizeNotAllow(Exception):
pass
class UploadPermissionNotAuthorized(Exception):
pass
#<result><msg><![CDATA[%s]]></msg><error><![CDATA[%s]]></error><file_url>%s</file_url></result>
xml_template = "<result><msg><![CDATA[%s]]></msg><error><![CDATA[%s]]></error><file_url>%s</file_url></result>"
try:
f = request.FILES['file-upload']
# check upload permission
if not auth.can_upload_files(request.user):
raise UploadPermissionNotAuthorized
# check file type
file_name_suffix = os.path.splitext(f.name)[1].lower()
if not file_name_suffix in ('.jpg', '.jpeg', '.gif', '.png', '.bmp', '.tiff'):
raise FileTypeNotAllow
storage = FileSystemStorage(str(settings.UPFILES_FOLDER), str(settings.UPFILES_ALIAS))
new_file_name = storage.save(f.name, f)
# check file size
# byte
size = storage.size(new_file_name)
if size > float(settings.ALLOW_MAX_FILE_SIZE) * 1024 * 1024:
storage.delete(new_file_name)
raise FileSizeNotAllow
result = xml_template % ('Good', '', str(settings.UPFILES_ALIAS) + new_file_name)
except UploadPermissionNotAuthorized:
result = xml_template % ('', _('uploading images is limited to users with >60 reputation points'), '')
except FileTypeNotAllow:
result = xml_template % ('', _("allowed file types are 'jpg', 'jpeg', 'gif', 'bmp', 'png', 'tiff'"), '')
except FileSizeNotAllow:
result = xml_template % ('', _("maximum upload file size is %sM") % settings.ALLOW_MAX_FILE_SIZE, '')
except Exception, e:
result = xml_template % ('', _('Error uploading file. Please contact the site administrator. Thank you. %s' % e), '')
class FileSaveRaceConditionTest(TestCase):
def setUp(self):
self.storage_dir = tempfile.mkdtemp()
self.storage = FileSystemStorage(self.storage_dir)
self.thread = threading.Thread(target=self.save_file, args=['conflict'])
def tearDown(self):
shutil.rmtree(self.storage_dir)
def save_file(self, name):
name = self.storage.save(name, SlowFile("Data"))
def test_race_condition(self):
self.thread.start()
name = self.save_file('conflict')
self.thread.join()
self.assert_(self.storage.exists('conflict'))
self.assert_(self.storage.exists('conflict_'))
self.storage.delete('conflict')
self.storage.delete('conflict_')
def handle_noargs(self, **options):
# Force django to calculate template_source_loaders
try:
find_template('notexists')
except TemplateDoesNotExist:
pass
from django.template.loader import template_source_loaders
loaders = []
for loader in template_source_loaders:
if isinstance(loader, CachedLoader):
loaders.extend(loader.loaders)
else:
loaders.append(loader)
paths = set()
for loader in loaders:
paths.update(list(loader.get_template_sources('')))
templates = set()
for path in paths:
for root, dirs, files in os.walk(path):
templates.update(os.path.join(root, name)
for name in files if not name.startswith('.') and
fnmatch(name, '*xml'))
storage = FileSystemStorage(settings.FEST_COMPILED_ROOT)
for template_name in templates:
template_file = open(template_name)
try:
tpl = Template(template_file.read().decode(
settings.FILE_CHARSET))
template = ContentFile(tpl.compile())
finally:
template_file.close()
name = self.get_dest_filename(template_name)
storage.delete(name)
storage.save(name, template)
def dfba_solve(request):
pathway = get_pathway_from_request(request)
name = request.session["collection_name"]
associated_file_key = request.session["dfba_upload"]
email = request.session['provided_email']
task = Task(task_type = 'dfba',
main_file = name,
email = email,
status = "TODO")
file_system = FileSystemStorage()
uuid = str(task.uuid)
user_upload_temp_filename = "dfba/" + associated_file_key
user_upload_temp_file = file_system.open(user_upload_temp_filename, "r")
ampl_file = file_system.open(uuid + ".ampl", "w")
variable_mapping = file_system.open(uuid + ".map", "w")
report_header = file_system.open(uuid + ".header", "w")
obj_type = request.GET['obj_type']
ot = 'biomass' # 1 = biomass
if obj_type == '0': # 0 = customer defined
ot = 'user'
pathway.output_ampl(ampl_file,
variable_mapping,
report_header,
model_type="dfba",
additional_file = user_upload_temp_file,
objective_type = ot)
ampl_file.close()
variable_mapping.close()
report_header.close()
user_upload_temp_file.close()
# cleanup the temp file from user upload.
file_system.delete(user_upload_temp_filename)
task.save()
return HttpResponse(content = "New DFBA optimization problem submitted .. ", status = 200, content_type = "text/html")
def upload(request): # ajax upload file to a question or answer
"""view that handles file upload via Ajax
"""
# check upload permission
result = ""
error = ""
new_file_name = ""
try:
# may raise exceptions.PermissionDenied
if request.user.is_anonymous():
msg = _("Sorry, anonymous users cannot upload files")
raise exceptions.PermissionDenied(msg)
request.user.assert_can_upload_file()
# check file type
f = request.FILES["file-upload"]
logging.info("Upload %s" % f.name)
file_extension = os.path.splitext(f.name)[1].lower()
# generate new file name
new_file_name = datetime.date.today().strftime("%y%m%d") + f.name.replace(" ", "_").lower()
logging.info("Save as %s" % new_file_name)
file_storage = FileSystemStorage(
location=settings.ASKBOT_FILE_UPLOAD_DIR, base_url=reverse("uploaded_file", kwargs={"path": ""})
)
# use default storage to store file
file_storage.save(new_file_name, f)
# check file size
# byte
size = file_storage.size(new_file_name)
if size > settings.ASKBOT_MAX_UPLOAD_FILE_SIZE:
file_storage.delete(new_file_name)
msg = _("maximum upload file size is %(file_size)sK") % {"file_size": settings.ASKBOT_MAX_UPLOAD_FILE_SIZE}
raise exceptions.PermissionDenied(msg)
except exceptions.PermissionDenied, e:
error = unicode(e)
class ThumbnailField(object):
"""Instances of this class will be used to access data of the
generated thumbnails
"""
def __init__(self, name):
self.name = name
self.storage = FileSystemStorage()
def path(self):
return self.storage.path(self.name)
def delete(self):
return self.storage.delete(self.name)
def open(self):
self.file = self.storage.open(self.name)
def chunks(self, chunk_size=None):
"""
Read the file and yield chucks of ``chunk_size`` bytes (defaults to
``UploadedFile.DEFAULT_CHUNK_SIZE``).
"""
if not chunk_size:
chunk_size = 64 * 2 ** 10
if hasattr(self.file, 'seek'):
self.file.seek(0)
# Assume the pointer is at zero...
counter = self.file.size
while counter > 0:
yield self.file.read(chunk_size)
counter -= chunk_size
def close(self):
self.file.close()
def url(self):
return self.storage.url(self.name)
def size(self):
return self.storage.size(self.name)
class ThumbnailField:
'''
Instances of this class will be used to access data of the
generated thumbnails
'''
def __init__(self, name):
self.name = name
self.storage = FileSystemStorage()
def path(self):
return self.storage.path(self.name)
def url(self):
return self.storage.url(self.name)
def size(self):
return self.storage.size(self.name)
def delete(self):
return self.storage.delete(self.name)
def delete(self, name):
FileSystemStorage.delete(self, name)
self.__log("File " + name + " deleted.")
class ExecutableCodeCheck(BaseSecurityCheck):
name = _("Checking that uploaded files won't be executed by the server")
desc = _('A misconfiguration in the web server can cause files attached '
'to review requests to be executed as code. The file types '
'checked in this test are: .html, .htm, .shtml, .php, .php3, '
'.php4, .php5, .phps, .asp, .pl, .py, .fcgi, .cgi, .phtml, '
'.pht, .jsp, .sh, and .rb.')
fix_info = _('For instructions on how to fix this problem, please visit '
'<a href="http://support.beanbaginc.com/support/solutions/'
'articles/110173-securing-file-attachments">'
'http://support.beanbaginc.com/support/solutions/articles/'
'110173-securing-file-attachments</a>')
def __init__(self):
loc = os.path.join(settings.MEDIA_ROOT, 'uploaded', 'files')
self.storage = FileSystemStorage(location=loc)
self.directory = settings.MEDIA_URL + 'uploaded/files/'
self.file_checks = [
(
['.php', '.php3', '.php4', '.php5', '.phps', '.phtml',
'.phtm'],
'<?php echo "Hello, World!"; ?>'
),
(
['.pl', '.py'],
'print "Hello, World!"'
),
(
['.html', '.htm', '.shtml', '.pht'],
('<HTML>\n'
'<HEAD>\n'
'<TITLE>Hello, world!</TITLE>\n'
'</HEAD>\n'
'<BODY>\n'
'<H1>Hello, world!</H1>\n'
'<!--#echo var="LAST_MODIFIED" -->\n'
'<!--#exec cmd="echo HI!" -->\n'
'</BODY>\n'
'</HTML>')
),
(
['.jsp'],
'<%= new String("Hello!") %>'
),
(
['.asp'],
'<%="Hello World!"%>'
),
(
['.fcgi', '.cgi', '.sh'],
('#!/bin/sh\n'
'echo "Hello World!"')
),
(
['.rb'],
'puts "Hello world!"'
)
]
def setUp(self):
if self._using_default_storage():
for i, file_check in enumerate(self.file_checks):
extensions_list, content = file_check
bad_extensions = []
for ext in extensions_list:
try:
self.storage.save('exec_check' + ext,
ContentFile(content))
except OSError:
# Some web server configurations prevent even saving
# files with certain extensions. In this case, things
# will definitely succeed.
bad_extensions.append(ext)
# Filter out any extensions that we failed to save, because we
# don't need to check that they downloaded properly.
extensions_list = [ext for ext in extensions_list
if ext not in bad_extensions]
self.file_checks[i] = extensions_list, content
def execute(self):
error_msg = ''
ext_result = True
final_result = True
failed_exts = []
if self._using_default_storage():
for extensions_list, content in self.file_checks:
for ext in extensions_list:
try:
ext_result = self.download_and_compare(
'exec_check' + ext)
if final_result and not ext_result:
final_result = False
except Exception as e:
return (False,
_('Uncaught exception during test: %s') % e)
if not ext_result:
failed_exts.append(ext)
if not final_result:
error_msg = _(
ngettext(
'The web server incorrectly executed these file types: %s',
'The web server incorrectly executed this file type: %s',
len(failed_exts))
% ', '.join(failed_exts))
return final_result, error_msg
def tearDown(self):
if self._using_default_storage():
for extensions_list, content in self.file_checks:
for ext in extensions_list:
self.storage.delete('exec_check' + ext)
def download_and_compare(self, to_download):
data = urlopen(_get_url(self.directory) + to_download).read()
with self.storage.open(to_download, 'r') as f:
return data == f.read()
def _using_default_storage(self):
return (settings.DEFAULT_FILE_STORAGE ==
'django.core.files.storage.FileSystemStorage')
def delete(self, name):
FileSystemStorage.delete(self, name)
self.execute(self.transport.delete, name)
class Command(NoArgsCommand):
help = "Purges all files from `VERSIONED_STATIC_ROOT` that are not "\
"current versions of static files. (Files where the version "\
"*matches* the source file's current version are not removed, "\
"unless the `--all` flag is given.)"
option_list = NoArgsCommand.option_list + (
make_option('-n', '--dry-run', action='store_true', dest='dry_run',
default=False, help="Do everything except modify the filesystem."),
make_option('-i', '--ignore', action='append', default=[],
dest='ignore_patterns', metavar='PATTERN',
help="Ignore files or directories matching this glob-style "
"pattern. Use multiple times to ignore more."),
make_option('--all',
action='store_true',
dest='all',
default=False,
help='Removes *everything* from `VERSIONED_STATIC_ROOT`, not just '\
'old versions of static files.'),
make_option('--noinput',
action='store_true',
dest='interactive',
default=False,
help='Does not prompt for confirmation when deleting.'),
make_option('--no-default-ignore', action='store_false',
dest='use_default_ignore_patterns', default=True,
help="Don't ignore the common private glob-style patterns 'CVS', "
"'.*' and '*~'."),
)
def __init__(self, *args, **kwargs):
super(Command, self).__init__(*args, **kwargs)
self.deleted_files = []
self.unmodified_files = []
# Force storage to be a filesystem storage for VERSIONED_STATIC_ROOT.
self.storage = get_storage_class(settings.STATICFILES_STORAGE)()
self.versioned_storage = FileSystemStorage(location=settings.VERSIONED_STATIC_ROOT)
def handle_noargs(self, **options):
self.verbosity = int(options.get('verbosity', 1))
ignore_patterns = options['ignore_patterns']
if options['use_default_ignore_patterns']:
ignore_patterns += ['CVS', '.*', '*~']
ignore_patterns = list(set(ignore_patterns))
# Warn before doing anything more.
if options.get('interactive'):
confirm = raw_input(u"""
You have requested to delete versioned files at the destination
location as specified in your settings file.
This will remove existing files.
Are you sure you want to do this?
Type 'yes' to continue, or 'no' to cancel: """)
if confirm != 'yes':
raise CommandError("Collecting static files cancelled.")
# Get output filenames for up-to-date static files.
self.valid_outpaths = set()
if not options['all']:
for finder in finders.get_finders():
for path, storage in finder.list(ignore_patterns):
# Prefix the relative path if source storage contains it
if getattr(storage, 'prefix', None):
prefixed_path = os.path.join(storage.prefix, path)
else:
prefixed_path = path
current_version = get_file_version(path, storage)
current_outpath = get_versioned_path(
prefixed_path, current_version)
self.valid_outpaths.add(current_outpath)
# Find everything currently in VERSIONED_STATIC_ROOT and remove
# anything not in `self.valid_outpaths`
def cleardirs(cd, subdirs, files):
for d in subdirs:
new_cd = os.path.join(cd, d)
new_subdirs, new_files = self.versioned_storage.listdir(
os.path.join(cd, d)
)
cleardirs(new_cd, new_subdirs, new_files)
for f in files:
filepath = os.path.join(cd, f)
if filepath not in self.valid_outpaths:
if options['dry_run']:
self.log(u"Pretending to delete '%s'" % filepath)
else:
self.log(u"Deleting '%s'" % filepath)
self.versioned_storage.delete(filepath)
self.deleted_files.append(filepath)
else:
self.log(u"Skipping up-to-date file '%s'" % filepath)
self.unmodified_files.append(filepath)
subdirs, files = self.versioned_storage.listdir("")
cleardirs("", subdirs, files)
actual_count = len(self.deleted_files)
unmodified_count = len(self.unmodified_files)
if self.verbosity >= 1:
self.stdout.write(smart_str(u"\n%s versioned file%s %s%s.\n"
% (actual_count, actual_count != 1 and 's' or '',
'deleted',
unmodified_count and ' (%s unmodified)'
% unmodified_count or '')))
def log(self, msg, level=2):
"""
Small log helper
"""
msg = smart_str(msg)
if not msg.endswith("\n"):
msg += "\n"
if self.verbosity >= level:
self.stdout.write(msg)
def remove_old_versions(self, path, prefixed_path, source_storage, **kwargs):
# For iglob matching to work, we actually need the full path
# of the output file.
outpath = os.path.join(settings.VERSIONED_STATIC_ROOT, prefixed_path)
current_version = get_file_version(path, source_storage)
current_outpath = get_versioned_path(outpath, current_version)
# Match any possible version of the output file.
for matching_path in iglob(get_versioned_path(outpath, "*")):
# Remove VERSIONED_STATIC_ROOT just for output log, etc.
rel_matching_path = matching_path.replace(
settings.VERSIONED_STATIC_ROOT, "", 1
)[1:]
if kwargs['all'] or (matching_path != current_outpath):
self.deleted_files.append(matching_path)
if kwargs['dry_run']:
self.log(u"Pretending to delete '%s' (Current is '%s')" % (
rel_matching_path, current_version
))
else:
self.log(u"Deleting '%s' (Current is '%s')" % (
rel_matching_path, current_version
))
os.unlink(matching_path)
self.versioned_storage.delete(matching_path)
else:
self.unmodified_files.append(matching_path)
self.log(u"Skipping '%s'. (Current version.)" % rel_matching_path)
def delete(self, name):
if self.is_thumbor(name):
return ThumborStorage.delete(self, name)
else:
return FileSystemStorage.delete(self, name)
class LocallyMirroredS3BotoStorage(S3BotoStorage):
def __init__(self, *args, **kwargs):
super(LocallyMirroredS3BotoStorage, self).__init__(*args, **kwargs)
self.mirror = FileSystemStorage(location=settings.S3_MIRROR_ROOT)
def delete(self, name):
super(LocallyMirroredS3BotoStorage, self).delete(name)
try:
self.mirror.delete(name)
except OSError:
full_path = self.mirror.path(name)
if os.path.exists(full_path):
os.rmdir(full_path)
def exists(self, name):
exists_local = self.mirror.exists(name)
if exists_local:
return True
else:
exists_remote = super(LocallyMirroredS3BotoStorage, self).exists(name)
if exists_remote:
self.mirror._save(name, ContentFile(""))
return True
return False
def _save(self, name, content):
cleaned_name = super(LocallyMirroredS3BotoStorage, self)._save(name, content)
self.mirror._save(name, ContentFile(""))
return cleaned_name
