def __init__(self, label):
self.label = label
key = DataKey("bar.jpg", "0123456789abcdef")
other_key = DataKey("subdirectory/baz.png", "fedcba9876543210")
#key.url = "http://example.com/my_data/bar.jpg"
self.output_data = [key, other_key]
self.datastore = MockDataStore()
def test__delete__should_remove_files(self):
assert os.path.exists(os.path.join(self.root_dir, 'test_file1'))
digest = hashlib.sha1(self.test_data).hexdigest()
keys = [DataKey(path, digest) for path in self.test_files]
self.ds.delete(*keys)
self.assert_(
not os.path.exists(os.path.join(self.root_dir, 'test_file1')))
def test__get_content__should_truncate_long_files(self):
self.ds.find_new_data(self.now)
digest = hashlib.sha1(self.test_data).hexdigest()
key = DataKey('%s/test_file1' % self.now.strftime(TIMESTAMP_FORMAT),
digest)
content = self.ds.get_content(key, max_length=10)
self.assertEqual(content, self.test_data[:10])
def test__get_content__should_return_short_file_content(self):
self.ds.find_new_data(self.now)
digest = hashlib.sha1(self.test_data).hexdigest()
key = DataKey('%s/test_file1' % self.now.strftime(TIMESTAMP_FORMAT),
digest)
content = self.ds.get_content(key)
self.assertEqual(content, self.test_data)
def download_file(request, project, label):
label = unescape(label)
path = request.GET['path']
digest = request.GET['digest']
data_key = DataKey(path, digest)
record = models.Record.objects.get(label=label, project__id=project)
data_store = get_data_store(record.datastore.type,
eval(record.datastore.parameters))
mimetype, encoding = mimetypes.guess_type(path)
try:
content = data_store.get_content(data_key)
except (IOError, KeyError):
raise Http404
dir, fname = os.path.split(path)
response = HttpResponse(mimetype=mimetype)
response['Content-Disposition'] = 'attachment; filename=%s' % fname
response.write(content)
return response
def show_image(request, project, label):
label = unescape(label)
path = request.GET['path']
digest = request.GET['digest']
data_key = DataKey(path, digest)
mimetype, encoding = mimetypes.guess_type(path)
if mimetype in ("image/png", "image/jpeg", "image/gif"):
record = models.Record.objects.get(label=label, project__id=project)
data_store = get_data_store(record.datastore.type,
eval(record.datastore.parameters))
try:
content = data_store.get_content(data_key)
except (IOError, KeyError):
raise Http404
response = HttpResponse(mimetype=mimetype)
response.write(content)
return response
else:
return HttpResponse(
mimetype="image/png") # should return a placeholder image?
def show_file(request, project, label):
label = unescape(label)
path = request.GET['path']
digest = request.GET['digest']
type = request.GET.get('type', 'output')
show_script = request.GET.get('show_script', False)
data_key = DataKey(path, digest)
if show_script: # record_list.html: user click the main file cell
print 'digest: %s' % digest
if 'truncate' in request.GET:
if request.GET['truncate'].lower() == 'false':
max_display_length = None
else:
max_display_length = int(request.GET['truncate']) * 1024
else:
max_display_length = DEFAULT_MAX_DISPLAY_LENGTH
record = models.Record.objects.get(label=label, project__id=project)
if type == 'output':
data_store = get_data_store(record.datastore.type,
eval(record.datastore.parameters))
else:
data_store = get_data_store(record.input_datastore.type,
eval(record.input_datastore.parameters))
truncated = False
mimetype, encoding = mimetypes.guess_type(path)
try:
if mimetype == "text/csv":
content = data_store.get_content(data_key,
max_length=max_display_length)
if max_display_length is not None and len(
content) >= max_display_length:
truncated = True
# dump the last truncated line (if any)
content = content.rpartition('\n')[0]
lines = content.splitlines()
reader = csv.reader(lines)
return render_to_response(
"show_csv.html", {
'path': path,
'label': label,
'digest': digest,
'project_name': project,
'reader': reader,
'truncated': truncated
})
elif mimetype == None or mimetype.split("/")[0] == "text":
content = data_store.get_content(data_key,
max_length=max_display_length)
if max_display_length is not None and len(
content) >= max_display_length:
truncated = True
return render_to_response(
"show_file.html", {
'path': path,
'label': label,
'digest': digest,
'project_name': project,
'content': content,
'truncated': truncated,
})
elif mimetype in ("image/png", "image/jpeg",
"image/gif"): # need to check digests match
return render_to_response(
"show_image.html", {
'path': path,
'label': label,
'digest': digest,
'project_name': project,
})
elif mimetype == 'application/zip':
import zipfile
if zipfile.is_zipfile(path):
zf = zipfile.ZipFile(path, 'r')
contents = zf.namelist()
zf.close()
return render_to_response("show_file.html", {
'path': path,
'label': label,
'content': "\n".join(contents)
})
else:
raise IOError("Not a valid zip file")
else:
return render_to_response(
"show_file.html", {
'path':
path,
'label':
label,
'project_name':
project,
'content':
"Can't display this file (mimetype assumed to be %s)" %
mimetype
})
except (IOError, KeyError), e:
return render_to_response(
"show_file.html", {
'path': path,
'label': label,
'project_name': project,
'content': "File not found.",
'errmsg': e
})
def test__get_content__should_truncate_long_files(self):
digest = hashlib.sha1(self.test_data).hexdigest()
key = DataKey('test_file1', digest, creation=self.now)
content = self.ds.get_content(key, max_length=10)
self.assertEqual(content, self.test_data[:10])
def test__get_content__should_return_short_file_content(self):
digest = hashlib.sha1(self.test_data).hexdigest()
key = DataKey('test_file1', digest, creation=None)
content = self.ds.get_content(key)
self.assertEqual(content, self.test_data)