def post(self, request, *args, **kwargs):
form = ImportISATabFileForm(request.POST, request.FILES)
if form.is_valid():
f = form.cleaned_data['isa_tab_file']
url = form.cleaned_data['isa_tab_url']
logger.debug("ISA-Tab URL: %s", url)
context = RequestContext(request, {'form': form})
if url:
# TODO: replace with chain
# http://docs.celeryproject.org/en/latest/userguide/tasks.html#task-synchronous-subtasks
u = urlparse(url)
file_name = u.path.split('/')[-1]
temp_file_path = os.path.join(get_temp_dir(), file_name)
try:
# TODO: refactor download_file to take file handle instead
# of path
download_file(url, temp_file_path)
except DownloadError as e:
logger.error("Problem downloading ISA-Tab file. %s", e)
error = "Problem downloading ISA-Tab file from: " + url
context = RequestContext(request,
{'form': form, 'error': error})
return render_to_response(self.template_name,
context_instance=context)
else:
temp_file_path = os.path.join(get_temp_dir(), f.name)
try:
handle_uploaded_file(f, temp_file_path)
except IOError as e:
error_msg = "Error writing ISA-Tab file to disk."
error_msg += " IOError: %s, file name: %s, error: %s"
logger.error(error_msg, e.errno, e.filename, e.strerror)
error = "Error writing ISA-Tab file to disk"
context = RequestContext(request,
{'form': form, 'error': error})
return render_to_response(self.template_name,
context_instance=context)
logger.debug("Temp file name: '%s'", temp_file_path)
dataset_uuid = (parse_isatab.delay(
request.user.username,
False,
temp_file_path
).get())[0]
# TODO: exception handling (OSError)
os.unlink(temp_file_path)
if dataset_uuid:
# TODO: redirect to the list of analysis samples for the given
# UUID
return HttpResponseRedirect(
reverse(self.success_view_name, args=(dataset_uuid,)))
else:
error = 'Problem parsing ISA-Tab file'
context = RequestContext(request,
{'form': form, 'error': error})
return render_to_response(self.template_name,
context_instance=context)
else: # submitted form is not valid
context = RequestContext(request, {'form': form})
return render_to_response(self.template_name,
context_instance=context)
def import_isa_tab(request):
'''Process imported ISA-Tab file sent via POST request
'''
#TODO: change implementation to a class-based view
#TODO: change from hardcoded URLs to using reverse()
if request.method == 'POST':
form = ImportISATabFileForm(request.POST, request.FILES)
if form.is_valid():
f = form.cleaned_data['isa_tab_file']
url = form.cleaned_data['isa_tab_url']
if url:
#TODO: replace with chain (http://docs.celeryproject.org/en/latest/userguide/tasks.html#task-synchronous-subtasks)
u = urlparse(url)
file_name = u.path.split('/')[-1]
temp_file_name = os.path.join(get_temp_dir(), file_name)
try:
download_file.delay(url, temp_file_name).get()
except DownloadError as e:
logger.error("Problem downloading ISA-Tab file. %s", e)
error = "Problem downloading ISA-Tab file from: " + url
context = RequestContext(request, {'form': form, 'error': error})
return render_to_response('data_set_manager/import.html', context_instance=context)
else:
temp_file_name = os.path.join(get_temp_dir(), f.name)
try:
handle_uploaded_file(f, temp_file_name)
except IOError as e:
logger.error("Error writing ISA-Tab file to disk\nIOError: %s, file name: %s, error: %s",
e.errno, e.filename, e.strerror)
error = "Error writing ISA-Tab file to disk"
context = RequestContext(request, {'form': form, 'error': error})
return render_to_response('data_set_manager/import.html', context_instance=context)
logger.debug("Temp file name: '%s'", temp_file_name)
dataset_uuid = parse_isatab.delay(request.user.username, False, temp_file_name).get()
os.unlink(temp_file_name)
if dataset_uuid:
#TODO: redirect to the list of analysis samples for the given UUID
return HttpResponseRedirect('/data_sets/' + dataset_uuid + '/')
else:
error = 'Problem parsing ISA-Tab file'
context = RequestContext(request, {'form': form, 'error': error})
return render_to_response('data_set_manager/import.html', context_instance=context)
else: # submitted form is not valid
context = RequestContext(request, {'form': form})
else: # this was not a POST request
form = ImportISATabFileForm()
context = RequestContext(request, {'form': form})
return render_to_response('data_set_manager/import.html', context_instance=context)
def import_by_url(self, url):
# TODO: replace with chain
# http://docs.celeryproject.org/en/latest/userguide/tasks.html#task-synchronous-subtasks
parsed_url = urlparse.urlparse(url)
file_name = parsed_url.path.split('/')[-1]
temp_file_path = os.path.join(get_temp_dir(), file_name)
try:
# TODO: refactor download_file to take file handle instead
# of path
download_file(url, temp_file_path)
except RuntimeError as exc:
error_msg = "Problem downloading ISA-Tab file from: " + url
logger.error("%s. %s", error_msg, exc)
return {
"success": False,
"message": error_msg
}
return {
"success": True,
"message": "File imported.",
"data": {
"temp_file_path": temp_file_path
}
}
def import_by_file(self, file):
temp_file_path = os.path.join(get_temp_dir(), file.name)
try:
handle_uploaded_file(file, temp_file_path)
except IOError as e:
error_msg = "Error writing ISA-Tab file to disk"
logger.error(
"%s. IOError: %s, file name: %s, error: %s.",
error_msg,
e.errno,
e.filename,
e.strerror
)
return {
"success": False,
"message": error_msg
}
return {
"success": True,
"message": "File imported.",
"data": {
"temp_file_path": temp_file_path
}
}
def import_by_url(self, url):
# TODO: replace with chain
# http://docs.celeryproject.org/en/latest/userguide/tasks.html#task-synchronous-subtasks
parsed_url = urlparse.urlparse(url)
file_name = parsed_url.path.split('/')[-1]
temp_file_path = os.path.join(get_temp_dir(), file_name)
try:
# TODO: refactor download_file to take file handle instead
# of path
download_file(url, temp_file_path)
except DownloadError as e:
error_msg = "Problem downloading ISA-Tab file from: " + url
logger.error("%s. %s", (error_msg, e))
return {
"success": False,
"message": error_msg
}
return {
"success": True,
"message": "File imported.",
"data": {
"temp_file_path": temp_file_path
}
}
def import_by_file(self, file):
temp_file_path = os.path.join(get_temp_dir(), file.name)
try:
handle_uploaded_file(file, temp_file_path)
except IOError as e:
error_msg = "Error writing ISA-Tab file to disk"
logger.error(
"%s. IOError: %s, file name: %s, error: %s.",
error_msg,
e.errno,
e.filename,
e.strerror
)
return {
"success": False,
"message": error_msg
}
return {
"success": True,
"message": "File imported.",
"data": {
"temp_file_path": temp_file_path
}
}
def get(self, request, *args, **kwargs):
# a workaround for automatic ISA archive import after logging in
try:
url_from_cookie = request.COOKIES[self.isa_tab_cookie_name]
except KeyError:
logger.info("ISA-Tab URL was not provided")
form = ImportISATabFileForm()
context = RequestContext(request, {'form': form})
return render_to_response(self.template_name,
context_instance=context)
form = ImportISATabFileForm({'isa_tab_url': url_from_cookie})
if form.is_valid():
url = form.cleaned_data['isa_tab_url']
else:
context = RequestContext(request, {'form': form})
response = render_to_response(self.template_name,
context_instance=context)
response.delete_cookie(self.isa_tab_cookie_name)
return response
u = urlparse.urlparse(url)
file_name = u.path.split('/')[-1]
temp_file_path = os.path.join(get_temp_dir(), file_name)
try:
# TODO: refactor download_file to take file handle instead of path
download_file(url, temp_file_path)
except DownloadError as e:
logger.error("Problem downloading ISA-Tab file. %s", e)
error = "Problem downloading ISA-Tab file from: '{}'".format(url)
context = RequestContext(request, {'form': form, 'error': error})
response = render_to_response(self.template_name,
context_instance=context)
response.delete_cookie(self.isa_tab_cookie_name)
return response
logger.debug("Temp file name: '%s'", temp_file_path)
dataset_uuid = parse_isatab.delay(request.user.username, False,
temp_file_path).get()[0]
# TODO: exception handling
os.unlink(temp_file_path)
if dataset_uuid:
if 'ajax' in kwargs and kwargs['ajax']:
return HttpResponse(
json.dumps({'new_data_set_uuid': dataset_uuid}),
'application/json'
)
else:
response = HttpResponseRedirect(
reverse(self.success_view_name, args=(dataset_uuid,)))
response.delete_cookie(self.isa_tab_cookie_name)
return response
else:
error = "Problem parsing ISA-Tab file"
context = RequestContext(request, {'form': form, 'error': error})
response = render_to_response(self.template_name,
context_instance=context)
response.delete_cookie(self.isa_tab_cookie_name)
return response
def get(self, request, *args, **kwargs):
# a workaround for automatic ISA archive import after logging in
try:
url_from_cookie = request.COOKIES[self.isa_tab_cookie_name]
except KeyError:
logger.info("ISA-Tab URL was not provided")
form = ImportISATabFileForm()
context = RequestContext(request, {'form': form})
return render_to_response(self.template_name,
context_instance=context)
form = ImportISATabFileForm({'isa_tab_url': url_from_cookie})
if form.is_valid():
url = form.cleaned_data['isa_tab_url']
else:
context = RequestContext(request, {'form': form})
response = render_to_response(self.template_name,
context_instance=context)
response.delete_cookie(self.isa_tab_cookie_name)
return response
u = urlparse.urlparse(url)
file_name = u.path.split('/')[-1]
temp_file_path = os.path.join(get_temp_dir(), file_name)
try:
# TODO: refactor download_file to take file handle instead of path
download_file(url, temp_file_path)
except DownloadError as e:
logger.error("Problem downloading ISA-Tab file. %s", e)
error = "Problem downloading ISA-Tab file from: '{}'".format(url)
context = RequestContext(request, {'form': form, 'error': error})
response = render_to_response(self.template_name,
context_instance=context)
response.delete_cookie(self.isa_tab_cookie_name)
return response
logger.debug("Temp file name: '%s'", temp_file_path)
dataset_uuid = parse_isatab.delay(request.user.username, False,
temp_file_path).get()[0]
# TODO: exception handling
os.unlink(temp_file_path)
if dataset_uuid:
if 'ajax' in kwargs and kwargs['ajax']:
return HttpResponse(
json.dumps({'new_data_set_uuid': dataset_uuid}),
'application/json')
else:
response = HttpResponseRedirect(
reverse(self.success_view_name, args=(dataset_uuid, )))
response.delete_cookie(self.isa_tab_cookie_name)
return response
else:
error = "Problem parsing ISA-Tab file"
context = RequestContext(request, {'form': form, 'error': error})
response = render_to_response(self.template_name,
context_instance=context)
response.delete_cookie(self.isa_tab_cookie_name)
return response
def test_get_temp_dir(self):
"""Check that the file store temp dir is reported correctly"""
self.assertEqual(get_temp_dir(), FILE_STORE_TEMP_DIR)
def import_isa_tab(request):
'''Process imported ISA-Tab file sent via POST request
'''
#TODO: change implementation to a class-based view
#TODO: change from hardcoded URLs to using reverse()
if request.method == 'POST':
form = ImportISATabFileForm(request.POST, request.FILES)
if form.is_valid():
f = form.cleaned_data['isa_tab_file']
url = form.cleaned_data['isa_tab_url']
if url:
#TODO: replace with chain (http://docs.celeryproject.org/en/latest/userguide/tasks.html#task-synchronous-subtasks)
u = urlparse(url)
file_name = u.path.split('/')[-1]
temp_file_name = os.path.join(get_temp_dir(), file_name)
try:
download_file.delay(url, temp_file_name).get()
except DownloadError as e:
logger.error("Problem downloading ISA-Tab file. %s", e)
error = "Problem downloading ISA-Tab file from: " + url
context = RequestContext(request, {
'form': form,
'error': error
})
return render_to_response('data_set_manager/import.html',
context_instance=context)
else:
temp_file_name = os.path.join(get_temp_dir(), f.name)
try:
handle_uploaded_file(f, temp_file_name)
except IOError as e:
logger.error(
"Error writing ISA-Tab file to disk\nIOError: %s, file name: %s, error: %s",
e.errno, e.filename, e.strerror)
error = "Error writing ISA-Tab file to disk"
context = RequestContext(request, {
'form': form,
'error': error
})
return render_to_response('data_set_manager/import.html',
context_instance=context)
logger.debug("Temp file name: '%s'", temp_file_name)
dataset_uuid = parse_isatab.delay(request.user.username, False,
temp_file_name).get()
os.unlink(temp_file_name)
if dataset_uuid:
#TODO: redirect to the list of analysis samples for the given UUID
return HttpResponseRedirect('/data_sets/' + dataset_uuid + '/')
else:
error = 'Problem parsing ISA-Tab file'
context = RequestContext(request, {
'form': form,
'error': error
})
return render_to_response('data_set_manager/import.html',
context_instance=context)
else: # submitted form is not valid
context = RequestContext(request, {'form': form})
else: # this was not a POST request
form = ImportISATabFileForm()
context = RequestContext(request, {'form': form})
return render_to_response('data_set_manager/import.html',
context_instance=context)
def get(self, request, *args, **kwargs):
# a workaround for automatic ISA archive import after logging in
try:
url_from_cookie = request.COOKIES[self.isa_tab_cookie_name]
except KeyError:
logger.info("ISA-Tab URL was not provided")
form = ImportISATabFileForm()
context = RequestContext(request, {'form': form})
return render_to_response(self.template_name,
context_instance=context)
form = ImportISATabFileForm({'isa_tab_url': url_from_cookie})
if form.is_valid():
url = form.cleaned_data['isa_tab_url']
else:
context = RequestContext(request, {'form': form})
response = render_to_response(self.template_name,
context_instance=context)
response.delete_cookie(self.isa_tab_cookie_name)
return response
u = urlparse.urlparse(url)
file_name = u.path.split('/')[-1]
temp_file_path = os.path.join(get_temp_dir(), file_name)
try:
# TODO: refactor download_file to take file handle instead of path
download_file(url, temp_file_path)
except RuntimeError as exc:
logger.error("Problem downloading ISA-Tab file. %s", exc)
error = "Problem downloading ISA-Tab file from: '{}'".format(url)
context = RequestContext(request, {'form': form, 'error': error})
response = render_to_response(self.template_name,
context_instance=context)
response.delete_cookie(self.isa_tab_cookie_name)
return response
logger.debug("Temp file name: '%s'", temp_file_path)
try:
dataset_uuid = parse_isatab(
request.user.username,
False,
temp_file_path
)
except ParserException as e:
error_message = "{} {}".format(
PARSER_ERROR_MESSAGE,
e.message
)
logger.error(error_message)
return HttpResponseBadRequest(error_message)
except Exception as e:
error_message = "{} {}".format(
PARSER_UNEXPECTED_ERROR_MESSAGE,
traceback.format_exc(e)
)
logger.error(error_message)
return HttpResponseBadRequest(
PARSER_UNEXPECTED_ERROR_MESSAGE +
e.message
)
try:
os.unlink(temp_file_path)
except OSError as exc:
logger.error("Couldn't unlink ISATab's `temp_file_path`: %s %s",
temp_file_path, exc)
if dataset_uuid:
if 'ajax' in kwargs and kwargs['ajax']:
return JsonResponse({'new_data_set_uuid': dataset_uuid})
else:
response = HttpResponseRedirect(
reverse(self.success_view_name, args=(dataset_uuid,)))
response.delete_cookie(self.isa_tab_cookie_name)
return response
else:
error = "Problem parsing ISA-Tab file"
context = RequestContext(request, {'form': form, 'error': error})
response = render_to_response(self.template_name,
context_instance=context)
response.delete_cookie(self.isa_tab_cookie_name)
return response
def import_file(uuid, permanent=False, refresh=False, file_size=1):
"""Download or copy file specified by UUID.
:param permanent: Flag for adding the FileStoreItem to cache.
:type permanent: bool.
:param refresh: Flag for forcing update of the file.
:type refresh: bool.
:param file_size: size of the remote file.
:type file_size: int.
:returns: FileStoreItem -- model instance or None if importing failed.
"""
logger.debug("Importing FileStoreItem with UUID '%s'", uuid)
item = FileStoreItem.objects.get_item(uuid=uuid)
if not item:
logger.error("FileStoreItem with UUID '%s' not found", uuid)
return None
# save task ID for looking up file import status
item.import_task_id = import_file.request.id
item.save()
# if file is ready to be used then return it,
# otherwise delete it if update is requested
if item.is_local():
if refresh:
item.delete_datafile()
else:
return item
# if source is an absolute file system path then copy,
# otherwise assume it is a URL and download
if os.path.isabs(item.source):
if os.path.isfile(item.source):
# check if source file can be opened
try:
srcfile = File(open(item.source))
except IOError:
logger.error("Could not open file: %s", item.source)
return None
srcfilename = os.path.basename(item.source)
# TODO: copy file in chunks to display progress report
# model is saved by default if FileField.save() is called
item.datafile.save(srcfilename, srcfile)
srcfile.close()
logger.info("File copied")
else:
logger.error("Copying failed: source is not a file")
return None
else:
req = urllib2.Request(item.source)
# check if source file can be downloaded
try:
response = urllib2.urlopen(req)
except urllib2.HTTPError as e:
logger.error("Could not open URL '%s'", item.source)
return None
except urllib2.URLError as e:
logger.error("Could not open URL '%s'. Reason: '%s'",
item.source, e.reason)
return None
except ValueError as e:
logger.error("Could not open URL '%s'. Reason: '%s'",
item.source, e.message)
return None
tmpfile = NamedTemporaryFile(dir=get_temp_dir(), delete=False)
# provide a default value in case Content-Length is missing
remotefilesize = int(
response.info().getheader("Content-Length", file_size)
)
logger.debug("Starting download from '%s'", item.source)
# download and save the file
localfilesize = 0
blocksize = 8 * 2 ** 10 # 8 Kbytes
for buf in iter(lambda: response.read(blocksize), ''):
localfilesize += len(buf)
tmpfile.write(buf)
# check if we have a sane value for file size
if remotefilesize > 0:
percent_done = localfilesize * 100. / remotefilesize
else:
percent_done = 0
import_file.update_state(
state="PROGRESS",
meta={"percent_done": "%3.2f%%" % percent_done,
'current': localfilesize, 'total': remotefilesize}
)
# cleanup
# TODO: delete temp file if download failed
tmpfile.flush()
tmpfile.close()
response.close()
logger.debug("Finished downloading from '%s'", item.source)
# get the file name from URL (remove query string)
u = urlparse(item.source)
src_file_name = os.path.basename(u.path)
# construct destination path based on source file name
rel_dst_path = item.datafile.storage.get_available_name(
file_path(item, src_file_name)
)
abs_dst_path = os.path.join(FILE_STORE_BASE_DIR, rel_dst_path)
# move the temp file into the file store
try:
if not os.path.exists(os.path.dirname(abs_dst_path)):
os.makedirs(os.path.dirname(abs_dst_path))
os.rename(tmpfile.name, abs_dst_path)
except OSError as e:
logger.error(
"Error moving temp file into the file store. "
"OSError: %s, file name: %s, error: %s",
e.errno, e.filename, e.strerror)
return False
# temp file is only accessible by the owner by default which prevents
# access by the web server if it is running as it's own user
try:
mode = os.stat(abs_dst_path).st_mode
os.chmod(abs_dst_path,
mode | stat.S_IRGRP | stat.S_IWGRP | stat.S_IROTH)
except OSError as e:
logger.error("Failed changing permissions on %s", abs_dst_path)
logger.error("OSError: %s, file name %s, error: %s",
e.errno, e.filename, e.strerror)
# assign new path to datafile
item.datafile.name = rel_dst_path
# save the model instance
item.save()
# TODO: if permanent is False then add to cache
return item
def test_get_temp_dir(self):
'''Check that the file store temp dir is reported correctly.
'''
self.assertEqual(get_temp_dir(), FILE_STORE_TEMP_DIR)
