def test_file_path(self): """Check that the file store path contains share name and file name""" # TODO: replace with assertRegexpMatches()? path = file_path(self.item_from_url, self.filename) self.assertIn(self.sharename, path) self.assertIn(self.filename, path) path = file_path(self.item_from_path, self.filename) self.assertIn(self.sharename, path) self.assertIn(self.filename, path)
def test_file_path(self): '''Check that the file store path contains share name and file name. ''' #TODO: replace with assertRegexpMatches()? path = file_path(self.item_from_url, self.filename) self.assertIn(self.sharename, path) self.assertIn(self.filename, path) path = file_path(self.item_from_path, self.filename) self.assertIn(self.sharename, path) self.assertIn(self.filename, path)
def test_file_path_parens(self): '''Check if the parentheses are replaced with underscores in the file name ''' filename = 'Kc.dMi-2(Q4443).wig_5.tdf' new_filename = 'Kc.dMi-2_Q4443_.wig_5.tdf' path_source = os.path.join('/example/path', filename) item_from_path = FileStoreItem.objects.create(source=path_source, sharename=self.sharename) url_source = urljoin('http://example.org/', filename) item_from_url = FileStoreItem.objects.create(source=url_source, sharename=self.sharename) path = file_path(item_from_url, filename) self.assertIn(self.sharename, path) self.assertIn(new_filename, path) path = file_path(item_from_path, filename) self.assertIn(self.sharename, path) self.assertIn(new_filename, path)
def import_file(uuid, permanent=False, refresh=False, file_size=1): """Download or copy file specified by UUID. :param permanent: Flag for adding the FileStoreItem to cache. :type permanent: bool. :param refresh: Flag for forcing update of the file. :type refresh: bool. :param file_size: size of the remote file. :type file_size: int. :returns: FileStoreItem -- model instance or None if importing failed. """ logger.debug("Importing FileStoreItem with UUID '%s'", uuid) item = FileStoreItem.objects.get_item(uuid=uuid) if not item: logger.error("FileStoreItem with UUID '%s' not found", uuid) return None # save task ID for looking up file import status item.import_task_id = import_file.request.id item.save() # if file is ready to be used then return it, # otherwise delete it if update is requested if item.is_local(): if refresh: item.delete_datafile() else: return item # if source is an absolute file system path then copy, # otherwise assume it is a URL and download if os.path.isabs(item.source): if os.path.isfile(item.source): # check if source file can be opened try: srcfile = File(open(item.source)) except IOError: logger.error("Could not open file: %s", item.source) return None srcfilename = os.path.basename(item.source) # TODO: copy file in chunks to display progress report # model is saved by default if FileField.save() is called item.datafile.save(srcfilename, srcfile) srcfile.close() logger.info("File copied") else: logger.error("Copying failed: source is not a file") return None else: req = urllib2.Request(item.source) # check if source file can be downloaded try: response = urllib2.urlopen(req) except urllib2.HTTPError as e: logger.error("Could not open URL '%s'", item.source) return None except urllib2.URLError as e: logger.error("Could not open URL '%s'. Reason: '%s'", item.source, e.reason) return None except ValueError as e: logger.error("Could not open URL '%s'. Reason: '%s'", item.source, e.message) return None tmpfile = NamedTemporaryFile(dir=get_temp_dir(), delete=False) # provide a default value in case Content-Length is missing remotefilesize = int( response.info().getheader("Content-Length", file_size) ) logger.debug("Starting download from '%s'", item.source) # download and save the file localfilesize = 0 blocksize = 8 * 2 ** 10 # 8 Kbytes for buf in iter(lambda: response.read(blocksize), ''): localfilesize += len(buf) tmpfile.write(buf) # check if we have a sane value for file size if remotefilesize > 0: percent_done = localfilesize * 100. / remotefilesize else: percent_done = 0 import_file.update_state( state="PROGRESS", meta={"percent_done": "%3.2f%%" % percent_done, 'current': localfilesize, 'total': remotefilesize} ) # cleanup # TODO: delete temp file if download failed tmpfile.flush() tmpfile.close() response.close() logger.debug("Finished downloading from '%s'", item.source) # get the file name from URL (remove query string) u = urlparse(item.source) src_file_name = os.path.basename(u.path) # construct destination path based on source file name rel_dst_path = item.datafile.storage.get_available_name( file_path(item, src_file_name) ) abs_dst_path = os.path.join(FILE_STORE_BASE_DIR, rel_dst_path) # move the temp file into the file store try: if not os.path.exists(os.path.dirname(abs_dst_path)): os.makedirs(os.path.dirname(abs_dst_path)) os.rename(tmpfile.name, abs_dst_path) except OSError as e: logger.error( "Error moving temp file into the file store. " "OSError: %s, file name: %s, error: %s", e.errno, e.filename, e.strerror) return False # temp file is only accessible by the owner by default which prevents # access by the web server if it is running as it's own user try: mode = os.stat(abs_dst_path).st_mode os.chmod(abs_dst_path, mode | stat.S_IRGRP | stat.S_IWGRP | stat.S_IROTH) except OSError as e: logger.error("Failed changing permissions on %s", abs_dst_path) logger.error("OSError: %s, file name %s, error: %s", e.errno, e.filename, e.strerror) # assign new path to datafile item.datafile.name = rel_dst_path # save the model instance item.save() # TODO: if permanent is False then add to cache return item