def test_file_path(self):
"""Check that the file store path contains share name and file name"""
# TODO: replace with assertRegexpMatches()?
path = file_path(self.item_from_url, self.filename)
self.assertIn(self.sharename, path)
self.assertIn(self.filename, path)
path = file_path(self.item_from_path, self.filename)
self.assertIn(self.sharename, path)
self.assertIn(self.filename, path)
def test_file_path(self):
'''Check that the file store path contains share name and file name.
'''
#TODO: replace with assertRegexpMatches()?
path = file_path(self.item_from_url, self.filename)
self.assertIn(self.sharename, path)
self.assertIn(self.filename, path)
path = file_path(self.item_from_path, self.filename)
self.assertIn(self.sharename, path)
self.assertIn(self.filename, path)
def test_file_path_parens(self):
'''Check if the parentheses are replaced with underscores in the file name
'''
filename = 'Kc.dMi-2(Q4443).wig_5.tdf'
new_filename = 'Kc.dMi-2_Q4443_.wig_5.tdf'
path_source = os.path.join('/example/path', filename)
item_from_path = FileStoreItem.objects.create(source=path_source, sharename=self.sharename)
url_source = urljoin('http://example.org/', filename)
item_from_url = FileStoreItem.objects.create(source=url_source, sharename=self.sharename)
path = file_path(item_from_url, filename)
self.assertIn(self.sharename, path)
self.assertIn(new_filename, path)
path = file_path(item_from_path, filename)
self.assertIn(self.sharename, path)
self.assertIn(new_filename, path)
def import_file(uuid, permanent=False, refresh=False, file_size=1):
"""Download or copy file specified by UUID.
:param permanent: Flag for adding the FileStoreItem to cache.
:type permanent: bool.
:param refresh: Flag for forcing update of the file.
:type refresh: bool.
:param file_size: size of the remote file.
:type file_size: int.
:returns: FileStoreItem -- model instance or None if importing failed.
"""
logger.debug("Importing FileStoreItem with UUID '%s'", uuid)
item = FileStoreItem.objects.get_item(uuid=uuid)
if not item:
logger.error("FileStoreItem with UUID '%s' not found", uuid)
return None
# save task ID for looking up file import status
item.import_task_id = import_file.request.id
item.save()
# if file is ready to be used then return it,
# otherwise delete it if update is requested
if item.is_local():
if refresh:
item.delete_datafile()
else:
return item
# if source is an absolute file system path then copy,
# otherwise assume it is a URL and download
if os.path.isabs(item.source):
if os.path.isfile(item.source):
# check if source file can be opened
try:
srcfile = File(open(item.source))
except IOError:
logger.error("Could not open file: %s", item.source)
return None
srcfilename = os.path.basename(item.source)
# TODO: copy file in chunks to display progress report
# model is saved by default if FileField.save() is called
item.datafile.save(srcfilename, srcfile)
srcfile.close()
logger.info("File copied")
else:
logger.error("Copying failed: source is not a file")
return None
else:
req = urllib2.Request(item.source)
# check if source file can be downloaded
try:
response = urllib2.urlopen(req)
except urllib2.HTTPError as e:
logger.error("Could not open URL '%s'", item.source)
return None
except urllib2.URLError as e:
logger.error("Could not open URL '%s'. Reason: '%s'",
item.source, e.reason)
return None
except ValueError as e:
logger.error("Could not open URL '%s'. Reason: '%s'",
item.source, e.message)
return None
tmpfile = NamedTemporaryFile(dir=get_temp_dir(), delete=False)
# provide a default value in case Content-Length is missing
remotefilesize = int(
response.info().getheader("Content-Length", file_size)
)
logger.debug("Starting download from '%s'", item.source)
# download and save the file
localfilesize = 0
blocksize = 8 * 2 ** 10 # 8 Kbytes
for buf in iter(lambda: response.read(blocksize), ''):
localfilesize += len(buf)
tmpfile.write(buf)
# check if we have a sane value for file size
if remotefilesize > 0:
percent_done = localfilesize * 100. / remotefilesize
else:
percent_done = 0
import_file.update_state(
state="PROGRESS",
meta={"percent_done": "%3.2f%%" % percent_done,
'current': localfilesize, 'total': remotefilesize}
)
# cleanup
# TODO: delete temp file if download failed
tmpfile.flush()
tmpfile.close()
response.close()
logger.debug("Finished downloading from '%s'", item.source)
# get the file name from URL (remove query string)
u = urlparse(item.source)
src_file_name = os.path.basename(u.path)
# construct destination path based on source file name
rel_dst_path = item.datafile.storage.get_available_name(
file_path(item, src_file_name)
)
abs_dst_path = os.path.join(FILE_STORE_BASE_DIR, rel_dst_path)
# move the temp file into the file store
try:
if not os.path.exists(os.path.dirname(abs_dst_path)):
os.makedirs(os.path.dirname(abs_dst_path))
os.rename(tmpfile.name, abs_dst_path)
except OSError as e:
logger.error(
"Error moving temp file into the file store. "
"OSError: %s, file name: %s, error: %s",
e.errno, e.filename, e.strerror)
return False
# temp file is only accessible by the owner by default which prevents
# access by the web server if it is running as it's own user
try:
mode = os.stat(abs_dst_path).st_mode
os.chmod(abs_dst_path,
mode | stat.S_IRGRP | stat.S_IWGRP | stat.S_IROTH)
except OSError as e:
logger.error("Failed changing permissions on %s", abs_dst_path)
logger.error("OSError: %s, file name %s, error: %s",
e.errno, e.filename, e.strerror)
# assign new path to datafile
item.datafile.name = rel_dst_path
# save the model instance
item.save()
# TODO: if permanent is False then add to cache
return item
