def _copy(self, source, target_dir, name):
"""
Copy file into another file.
"""
fp = self._fopen(source)
# place the file contents in a temp file
# this is necessary for remote storages since reading in chunks may not be supported
tmp_file = tempfile.NamedTemporaryFile()
tmp_file.write(fp.read())
fp.close()
self._options['storage'].save(self._join_path(target_dir, name),
DjangoFile(tmp_file))
tmp_file.close()
def _move_from_local(self, path, dst, name):
"""
Move from local file to storage file.
"""
if os.path.isdir(path):
for p in map(lambda x: os.path.join(path, x), os.listdir(path)):
self._move_from_local(p, self._join_path(dst, name),
os.path.basename(p))
shutil.rmtree(path)
else:
dst_path = self._join_path(dst, name)
fp = open(path)
self._options['storage'].save(dst_path, DjangoFile(fp))
fp.close()
os.unlink(path)
def create_fixtures(self):
self.img = create_image()
self.image_name = 'test_file.jpg'
self.filename = os.path.join(os.path.dirname(__file__),
self.image_name)
self.img.save(self.filename, 'JPEG')
self.file = DjangoFile(open(self.filename), name=self.image_name)
self.zipfilename = 'test_zip.zip'
self.zip = ZipFile(self.zipfilename, 'a')
self.zip.write(self.filename)
self.zip.close()
def test_article_detail_show_featured_image(self):
author = self.create_person()
with open(FEATURED_IMAGE_PATH, 'rb') as f:
file_obj = DjangoFile(f, name='featured_image.jpg')
image = Image.objects.create(owner=author.user,
original_filename='featured_image.jpg',
file=file_obj,
subject_location='fooobar')
article = self.create_article(author=author, featured_image=image)
response = self.client.get(article.get_absolute_url())
image_url = get_thumbnailer(article.featured_image).get_thumbnail({
'size': (800, 450),
'crop': True,
'subject_location': article.featured_image.subject_location
}).url
self.assertContains(response, image_url)
def excel2xhtml(self, pth, filename):
try:
converted_pth = pth.replace("files", "converted")
converted_pth = converted_pth.split(".")[0] + ".xhtml"
subprocess.check_call([
'/usr/bin/python3', '/usr/bin/unoconv', '-f', 'xhtml', '-o',
converted_pth, pth
])
res = open(converted_pth, 'rb')
self.pdf_doc.save(filename + ".xhtml", DjangoFile(res))
# read = res.read()
# r=read.decode("utf-8")
# self.html = r
except:
raise
def url_to_file(url, folder):
# url = 'http://www.astrosurf.com/astrospace/images/ss/Satellite%2008.jpg'
local_name, local_path = download(url, 'tmp')
dj_file = DjangoFile(open(local_path), name=local_name)
obj, created = Image.objects.get_or_create(original_filename=local_name,
file=dj_file,
folder=folder,
is_public=True)
os.remove(local_path)
return obj
def excel2xhtml(self, pth, filename):
try:
res_pdf_path = pth.replace("files", "converted")
res_pdf_path = self.exclude_extension(res_pdf_path) + ".xhtml"
subprocess.check_call(
['/usr/bin/python3', '/usr/bin/unoconv', '-f', 'xhtml',
'-o', res_pdf_path,
pth]
)
res = open(res_pdf_path, 'rb')
self.pdf_doc.save(filename + ".xhtml", DjangoFile(res))
read = res.read()
r = read.decode("utf-8")
self.html = r
except:
raise
def setUp(self):
self.superuser = create_superuser()
self.client.login(username='******', password='******')
self.img = create_image()
self.image_name = 'test_file.jpg'
self.filename = os.path.join(settings.FILE_UPLOAD_TEMP_DIR, self.image_name)
self.img.save(self.filename, 'JPEG')
self.file = DjangoFile(open(self.filename), name=self.image_name)
# This is actually a "file" for filer considerations
self.image = Image.objects.create(owner=self.superuser,
original_filename=self.image_name,
file=self.file)
self.clipboard = Clipboard.objects.create(user=self.superuser)
self.clipboard.append_file(self.image)
self.folder = Folder.objects.create(name='test_folder')
def download(self, request, *args, **kwargs):
"""Download a Malware instance as a raw or compressed file"""
# kwargs provided by URLs/Routers: project_name, sha256
# malware_sha256 = self.kwargs.get("sha256", None)
serializer = self.get_serializer(data=request.data)
serializer.is_valid(raise_exception=True)
# get instance
instance = self.get_object()
dl_file = open(get_sample_path(instance.sha256), 'rb')
# TODO(frennkie) encoding?!?! CLRF, LF ?! XXX
response = HttpResponse(DjangoFile(dl_file),
content_type=instance.mime)
response['Content-Disposition'] = 'attachment; filename={}'.format(
instance.name)
return response
def _save(self, fp, dir_, name):
"""
Create new file and write into it from file pointer.
Return new file path or raise an ``Exception``.
"""
#place the file contents in a temp file
#this is necessary for remote storages since reading in chunks may not be supported
tmp_file = tempfile.NamedTemporaryFile()
tmp_file.write(fp.read())
fp.close()
tmp_file.seek(0)
path = self._join_path(dir_, name)
self._options['storage'].save(path, DjangoFile(tmp_file))
tmp_file.close()
return path
def create_django_file_info_image(self, text, filename=None):
"""
Create a django.core.files.base.File instance:
1. fill a PIL image with a colorful gradient
2. draw the given >text< on it
usable to attach this to: models.ImageField()
"""
if filename is None:
filename = "dummy.%s" % self.format.lower()
pil_image = self.create_info_image(text)
temp = io.BytesIO()
pil_image.save(temp, format=self.format)
django_file = DjangoFile(temp, name=filename)
return django_file
def createCompileBrokenSubmission(self, user):
'''
Create a submission that cannot be compiled.
'''
sf = self.createCompileBrokenSubmissionFile()
sub = Submission(assignment=self.validatedAssignment,
submitter=user.user,
notes=uccrap + "This is a non-compilable submission.",
state=Submission.TEST_COMPILE_PENDING,
file_upload=sf)
fname = "reverse_support_files.zip"
shutil.copyfile(rootdir + '/opensubmit/tests/submfiles/' + fname,
settings.MEDIA_ROOT + fname)
sub.assignment.attachment_test_support = DjangoFile(
open(settings.MEDIA_ROOT + fname))
sub.assignment.save()
sub.save()
return sub
def _save_image(self, rss_import, field_id, image_url, item_complete):
warnings.warn(
'Starting multimedia processing of the entry width id %s' %
rss_import.id)
try:
source = rss_import.source
warnings.warn("Processing Multimedia for %s (%s)" %
(source.name, source.url))
item_to_save = rss_import.content
if "multimedia" not in item_to_save:
item_to_save["multimedia"] = {}
if field_id not in item_to_save["multimedia"]:
from filer.models.foldermodels import Folder
parent, parent_created = Folder.objects.get_or_create(
name='RSS Files')
rss_folder, rss_created = Folder.objects.get_or_create(
name=source.name, parent=parent)
from django.core.files.temp import NamedTemporaryFile
with NamedTemporaryFile(delete=True) as file_tmp_obj:
img_file = urllib2.urlopen(image_url)
file_tmp_obj.write(img_file.read())
file_tmp_obj.flush()
file_name = image_url.rsplit('/', 1)[-1]
from django.core.files import File as DjangoFile
file_obj = DjangoFile(file_tmp_obj, name=file_name)
from filer.models.imagemodels import Image
image = Image.objects.create(original_filename=file_name,
file=file_obj,
folder=rss_folder)
file_obj.close()
img_file.close()
item_to_save["multimedia"][field_id] = image.pk
rss_import.content = item_to_save
except Exception, e:
warnings.warn('Error processing the image: %s' % str(e))
item_to_save["multimedia"][field_id] = None
def linker(self, path=None, base_folder=None, key=None):
path = unicode(os.path.normpath(path))
# key = '2011-09-25_event-now'
if base_folder:
base_folder = unicode(os.path.normpath(base_folder))
print u"The directory structure will be imported in %s" % (
base_folder, )
if self.verbosity >= 1:
print u"Import the folders and files in %s" % (path, )
root_folder_name = os.path.basename(path)
for root, dirs, files in os.walk(path):
rel_folders = root.partition(path)[2].strip(os.path.sep).split(
os.path.sep)
while '' in rel_folders:
rel_folders.remove('')
if base_folder:
folder_names = base_folder.split('/') + [root_folder_name
] + rel_folders
else:
folder_names = [root_folder_name] + rel_folders
folder = self.get_or_create_folder(folder_names)
for file in files:
if file.startswith(key):
print 'MATCH'
src = os.path.join(root, file)
print src
""""""
dj_file = DjangoFile(open(src), name=file)
dj_file = self.import_file(file=dj_file, folder=folder)
print src
print dj_file.path
# ugly hack - but we just want symlinks
os.unlink(dj_file.path)
os.symlink(src, dj_file.path)
def copy_file(pending_file: File, token: str) -> None:
with requests.get(
f'{DKC_API}/file/{pending_file.legacy_file_id}/download',
stream=True,
headers={'Girder-Token': token},
) as resp:
if 400 <= resp.status_code < 500:
print('4xx encountered, deleting file')
pending_file.delete()
return
resp.raise_for_status()
with tempfile.SpooledTemporaryFile(max_size=64 << 20) as tmp:
for chunk in resp.iter_content(chunk_size=1 << 20):
tmp.write(chunk)
tmp.seek(0)
pending_file.blob = DjangoFile(tmp, name=pending_file.name)
pending_file.save(update_fields=['blob'])
def setUp(self):
try:
from django.contrib.auth import get_user_model
User = get_user_model()
except ImportError:
from django.contrib.auth.models import User # NOQA
self.superuser = create_superuser()
self.client.login(username='******', password='******')
self.unauth_user = User.objects.create(username='******')
self.owner = User.objects.create(username='******')
self.test_user1 = User.objects.create(username='******', password='******')
self.test_user2 = User.objects.create(username='******', password='******')
self.test_user3 = User.objects.create(username='******', password='******')
self.group1 = Group.objects.create(name='name1')
self.group2 = Group.objects.create(name='name2')
self.group3 = Group.objects.create(name='name3')
self.group4 = Group.objects.create(name='name4')
self.test_user1.groups.add(self.group1)
self.test_user1.groups.add(self.group2)
self.test_user2.groups.add(self.group3)
self.test_user2.groups.add(self.group4)
self.img = create_image()
self.image_name = 'test_file.jpg'
self.filename = os.path.join(settings.FILE_UPLOAD_TEMP_DIR, self.image_name)
self.img.save(self.filename, 'JPEG')
self.file = DjangoFile(open(self.filename, 'rb'), name=self.image_name)
# This is actually a "file" for filer considerations
self.image = Image.objects.create(owner=self.superuser,
original_filename=self.image_name,
file=self.file)
self.clipboard = Clipboard.objects.create(user=self.superuser)
self.clipboard.append_file(self.image)
self.folder = Folder.objects.create(name='test_folder')
self.folder_perm = Folder.objects.create(name='test_folder2')
def save_file_and_audio_tracks(owner, content):
filename = content.name[:content.name.rfind('.')]
extension = content.name[content.name.rfind('.') + 1:]
content_tempfile_path = '{temp_dir}{filename}.{extension}'.format(
temp_dir=settings.FILE_UPLOAD_TEMP_DIR,
filename=filename,
extension=extension)
with open(content_tempfile_path, 'wb+') as destination:
destination.write(content.read())
clip_type = extensions_dict[extension]['type']
if clip_type == 'video':
audio = VideoFileClip(content_tempfile_path).audio
elif clip_type == 'audio':
audio = AudioFileClip(content_tempfile_path)
asset_file = AssetFile(owner=owner, content=content, name=content.name)
asset_file.save()
time_frames = generate_time_frames(
clip_duration=audio.duration,
frame_size=settings.ASSETS_ASSET_FILE_PART_SIZE)
for part_number, frame in enumerate(time_frames, start=1):
audio_clip_part = audio.subclip(*frame)
audio_path = '{temp_dir}{filename}_{part_number}.{extension}'.format(
temp_dir=settings.FILE_UPLOAD_TEMP_DIR,
filename=filename,
part_number=part_number,
extension='mp3')
audio_clip_part.write_audiofile(audio_path)
with open(audio_path, 'rb') as audio_file:
file_content = DjangoFile(audio_file)
file_part = AssetFilePart(asset_file=asset_file, order=part_number)
file_part.content.save(audio_path, file_content)
file_part.save()
return asset_file
def create_test_cleaned_file(account=None, portfolio=None, uploaded_file=None, file_file=None):
if account is None:
account = create_test_account()
if portfolio is None:
portfolio = create_test_portfolio(account=account)
if uploaded_file is None:
uploaded_file = UploadedFile.objects.create(
portfolio=portfolio,
uploaded_by=account,
file=file_file,
)
if file_file is None:
file_file = NamedTemporaryFile(dir=settings.MEDIA_ROOT)
file_file = DjangoFile(file_file)
return CleanedFile.objects.create(
uploaded_by=account,
uploaded_file=uploaded_file,
file=file_file
)
def create_django_image_obj(self):
"""
Create a django image file object suitable for FileField
It also sets the following attributes:
* ``self.image_name``: the image base name
* ``self.filename``: the complete image path
:return: django file object
It requires Pillow installed in the environment to work
"""
from django.core.files import File as DjangoFile
img = self.create_image()
self.image_name = 'test_file.jpg'
self.filename = os.path.join(settings.FILE_UPLOAD_TEMP_DIR, self.image_name)
img.save(self.filename, 'JPEG')
return DjangoFile(open(self.filename, 'rb'), name=self.image_name)
def doc2pdf(self, pth, ext, filename):
try:
res_pdf_path = pth.replace("files", "converted")
res_pdf_path = self.exclude_extension(res_pdf_path) + '.pdf'
if ext == "pdf":
res = open(pth, 'rb')
else:
subprocess.check_call([
'/usr/bin/python3', '/usr/bin/unoconv', '-f', 'pdf', '-o',
res_pdf_path, '-d', 'document', pth
])
res = open(res_pdf_path, 'rb')
if ext != "pdf":
res = open(res_pdf_path, 'rb')
else:
res = open(pth, 'rb')
if filename.endswith('.pdf'):
filename = filename + '.pdf'
self.pdf_doc.save(filename, DjangoFile(res))
except:
raise
def test_output(self):
"""
Test the standard picture
"""
file_obj = SimpleUploadedFile("file.jpg",
b"file_content",
content_type="image/jpeg")
image, _ = Image.objects.get_or_create(file=DjangoFile(file_obj),
defaults={"name": "file.jpg"})
item = create_content_item(PictureItem,
image=image,
caption="file.jpg")
self.assertHTMLEqual(
render_content_items([item]).html,
'<figure class="picture ">'
f'<img src="{item.image.url}" width="0.0" height="0.0" alt="" />'
'<figcaption>file.jpg</figcaption>'
'</figure>',
)
def create(self, validated_data):
sound = Sound.objects.create(**validated_data)
clip = AudioSegment.from_wav(sound.audio)
temp_path = settings.MEDIA_ROOT + 'temp.wav'
rev = clip.reverse()
id = sound.id
root = settings.MEDIA_ROOT
root = root + 'temp.wav'
rev.export(root, format='wav')
print(temp_path)
temp = DjangoFile(open(temp_path, mode='rb'), name='temp.wav')
x = {'og': id, 'reversedaudio': temp}
revserial = RevSerializer(data=x)
if revserial.is_valid():
r = revserial.save()
if os.path.isfile(temp_path):
os.remove(temp_path)
else:
print(revserial.errors)
return {'RevAudio': r.reversedaudio.url}
def test_file_change_upload_to_destination(self):
"""
Test that the file is actualy move from the private to the public
directory when the is_public is checked on an existing private file.
"""
file_obj = DjangoFile(open(self.filename, 'rb'), name=self.image_name)
image = Image.objects.create(owner=self.superuser,
is_public=False,
original_filename=self.image_name,
file=file_obj)
image.save()
self.assertTrue(image.file.path.startswith(filer_settings.FILER_PRIVATEMEDIA_STORAGE.location))
image.is_public = True
image.save()
self.assertTrue(image.file.path.startswith(filer_settings.FILER_PUBLICMEDIA_STORAGE.location))
self.assertEqual(len(image.icons), len(filer_settings.FILER_ADMIN_ICON_SIZES))
image.is_public = False
image.save()
self.assertTrue(image.file.path.startswith(filer_settings.FILER_PRIVATEMEDIA_STORAGE.location))
self.assertEqual(len(image.icons), len(filer_settings.FILER_ADMIN_ICON_SIZES))
def build_cache(self):
# get settings
formats_download = FORMATS_DOWNLOAD
waveform_sizes = WAVEFORM_SIZES
# cleanup:
# delete everything that 'could' be in the cache so far...
print '# formats_download:'
for format, variations in formats_download.iteritems():
for variation in variations:
filename = '%s_%s.%s' % ('download', variation, format)
tmp_directory = self.convert_(filename, self.folder, format,
variation)
"""
if sucessfully converted, create a django/filer file from temp_file
"""
if tmp_directory:
tmp_path = tmp_directory + '/' + filename
try:
tmp_file = DjangoFile(open(tmp_path), name=filename)
file, created = File.objects.get_or_create(
original_filename=tmp_file.name,
file=tmp_file,
folder=self.folder,
is_public=False)
self.status = 1
except Exception, e:
self.status = 2
print e
try:
shutil.rmtree(tmp_directory)
except Exception, e:
print e
self.save()
def populate():
printOnTerminal("app_search/populate", "populate.py called")
rootDict = {}
fileList = []
prevRoot = None
count = 1
for rootPath in rootPaths:
#collecting data for rootData.py
if prevRoot != None:
rootDict[prevRoot].append(count - 1)
rootDict[rootPath] = []
rootDict[rootPath].append(count)
prevRoot = rootPath
#actual populate function starts here
for root, directories, files in os.walk(rootPath):
for file in files:
if os.path.splitext(file)[1] in ext:
filePath = root + '/' + file
with open(filePath, mode='r', encoding='utf8') as f:
content = DjangoFile(f).read()
#creating new File object for every file
fileList.append({
'id': count,
'root': root,
'fileName': file,
'content': content
})
printOnTerminal("app_search/populate",
"reading " + filePath)
count += 1
rootDict[prevRoot].append(count - 1)
printOnTerminal("app_search/populate",
str(count - 1) + " files found in the root path(s)")
esFunctions.addToEs(fileList)
writeToRootData(rootDict)
def setUp(self):
activate('en')
super(BaseTest, self).setUp()
self.category_1 = BlogCategory.objects.create(name=u'category 1')
self.category_1.set_current_language('it', initialize=True)
self.category_1.name = u'categoria 1'
self.category_1.save()
self.thumb_1 = ThumbnailOption.objects.create(
name='base', width=100, height=100, crop=True, upscale=False
)
self.thumb_2 = ThumbnailOption.objects.create(
name='main', width=200, height=200, crop=False, upscale=False
)
img = create_image()
self.image_name = 'test_file.jpg'
self.filename = os.path.join(settings.FILE_UPLOAD_TEMP_DIR,
self.image_name)
img.save(self.filename, 'JPEG')
file_obj = DjangoFile(open(self.filename, 'rb'), name=self.image_name)
self.img = Image.objects.create(owner=self.user,
original_filename=self.image_name,
file=file_obj)
def walker(self, path=None, base_folder=None):
"""
This method walk a directory structure and create the
Folders and Files as they appear.
"""
path = path or self.path
base_folder = base_folder or self.base_folder
# prevent trailing slashes and other inconsistencies on path.
# cast to unicode so that os.walk returns path names in unicode
# (prevents encoding/decoding errors)
path = unicode(os.path.normpath(path))
if base_folder:
base_folder = unicode(os.path.normpath(base_folder))
print u"The directory structure will be imported in %s" % (
base_folder, )
if self.verbosity >= 1:
print u"Import the folders and files in %s" % (path, )
root_folder_name = os.path.basename(path)
for root, dirs, files in os.walk(path):
rel_folders = root.partition(path)[2].strip(os.path.sep).split(
os.path.sep)
while '' in rel_folders:
rel_folders.remove('')
if base_folder:
folder_names = base_folder.split('/') + [root_folder_name
] + rel_folders
else:
folder_names = [root_folder_name] + rel_folders
folder = self.get_or_create_folder(folder_names)
for file in files:
dj_file = DjangoFile(open(os.path.join(root, file)), name=file)
self.import_file(file=dj_file, folder=folder)
if self.verbosity >= 1:
print ('folder_created #%s / file_created #%s / ' + \
'image_created #%s') % (
self.folder_created, self.file_created,
self.image_created)
def create_fixtures(self):
self.superuser = create_superuser()
self.client.login(username='******', password='******')
self.unauth_user = User.objects.create(username='******')
self.owner = User.objects.create(username='******')
self.img = create_image()
self.image_name = 'test_file.jpg'
self.filename = os.path.join(os.path.dirname(__file__),
self.image_name)
self.img.save(self.filename, 'JPEG')
self.file = DjangoFile(open(self.filename), name=self.image_name)
# This is actually a "file" for filer considerations
self.image = Image.objects.create(owner=self.superuser,
original_filename=self.image_name,
file=self.file)
self.clipboard = Clipboard.objects.create(user=self.superuser)
self.clipboard.append_file(self.image)
self.folder = Folder.objects.create(name='test_folder')
def createSubmissionFile():
with NamedTemporaryFile(mode="wt", delete=False, prefix=settings.MEDIA_ROOT) as tmpfile:
# Submission file
tmpfile.write("The quick brown fox jumps over the lazy dog.")
tmpfile.close()
sf = SubmissionFile(attachment=DjangoFile(tmpfile.name))
sf.save()
# os.remove(tmpfile.name)
# Test results
val_result = SubmissionTestResult()
val_result.submission_file = sf
val_result.kind = SubmissionTestResult.VALIDITY_TEST
val_result.result = "Validation test result for student"
val_result.result_tutor = "Validation test result for tutor"
val_result.machine = machine
val_result.save()
full_result = SubmissionTestResult()
full_result.submission_file = sf
full_result.kind = SubmissionTestResult.FULL_TEST
full_result.result_tutor = "Full test result for tutor"
full_result.machine = machine
full_result.save()
return sf
def img2pdf(self, pth, filename):
try:
converted_pth = pth.replace("files", "converted")
im = Image.open(pth)
width, height = im.size
if height >= width:
orientation = 'P'
w = 210
h = 297
else:
orientation = 'L'
w = 297
h = 210
pdf = FPDF()
pdf.add_page(orientation=orientation)
pdf.image(pth, x=0, y=0, w=w, h=h)
pdf.output(converted_pth, "F")
res = open(converted_pth, 'rb')
self.pdf_doc.save(filename + ".pdf", DjangoFile(res))
# self.original_pdf.save(filename+".pdf", DjangoFile(res))
except:
raise