def test_basic(self): """ Upload a basic file using the FileUploadResource, in just a single chunk. """ upload_dir = FilePath(self.mktemp()) upload_dir.makedirs() temp_dir = FilePath(self.mktemp()) temp_dir.makedirs() fields = { "file_name": "resumableFilename", "mime_type": "resumableType", "total_size": "resumableTotalSize", "chunk_number": "resumableChunkNumber", "chunk_size": "resumableChunkSize", "total_chunks": "resumableTotalChunks", "content": "file", "on_progress": "on_progress", "session": "session" } mock_session = Mock() resource = FileUploadResource(upload_dir.path, temp_dir.path, fields, mock_session) multipart_body = b"""-----------------------------478904261175205671481632460\r\nContent-Disposition: form-data; name="resumableChunkNumber"\r\n\r\n1\r\n-----------------------------478904261175205671481632460\r\nContent-Disposition: form-data; name="resumableChunkSize"\r\n\r\n1048576\r\n-----------------------------478904261175205671481632460\r\nContent-Disposition: form-data; name="resumableCurrentChunkSize"\r\n\r\n16\r\n-----------------------------478904261175205671481632460\r\nContent-Disposition: form-data; name="resumableTotalSize"\r\n\r\n16\r\n-----------------------------478904261175205671481632460\r\nContent-Disposition: form-data; name="resumableType"\r\n\r\ntext/plain\r\n-----------------------------478904261175205671481632460\r\nContent-Disposition: form-data; name="resumableIdentifier"\r\n\r\n16-examplefiletxt\r\n-----------------------------478904261175205671481632460\r\nContent-Disposition: form-data; name="resumableFilename"\r\n\r\nexamplefile.txt\r\n-----------------------------478904261175205671481632460\r\nContent-Disposition: form-data; name="resumableRelativePath"\r\n\r\nexamplefile.txt\r\n-----------------------------478904261175205671481632460\r\nContent-Disposition: form-data; name="resumableTotalChunks"\r\n\r\n1\r\n-----------------------------478904261175205671481632460\r\nContent-Disposition: form-data; name="on_progress"\r\n\r\ncom.example.upload.on_progress\r\n-----------------------------478904261175205671481632460\r\nContent-Disposition: form-data; name="session"\r\n\r\n6891276359801283\r\n-----------------------------478904261175205671481632460\r\nContent-Disposition: form-data; name="file"; filename="blob"\r\nContent-Type: application/octet-stream\r\n\r\nhello Crossbar!\n\r\n-----------------------------478904261175205671481632460--\r\n""" d = renderResource( resource, b"/", method="POST", headers={ b"content-type": [b"multipart/form-data; boundary=---------------------------478904261175205671481632460"], b"Content-Length": [b"1678"] }, body=multipart_body ) res = self.successResultOf(d) res.setResponseCode.assert_called_once_with(200) self.assertEqual(len(mock_session.method_calls), 2) # Starting the upload self.assertEqual(mock_session.method_calls[0][1][0], u"com.example.upload.on_progress") self.assertEqual(mock_session.method_calls[0][1][1]["status"], "started") self.assertEqual(mock_session.method_calls[0][1][1]["id"], "examplefile.txt") # Upload complete self.assertEqual(mock_session.method_calls[1][1][0], u"com.example.upload.on_progress") self.assertEqual(mock_session.method_calls[1][1][1]["status"], "finished") self.assertEqual(mock_session.method_calls[1][1][1]["id"], "examplefile.txt") # Nothing in the temp dir, one file in the upload self.assertEqual(len(temp_dir.listdir()), 0) self.assertEqual(len(upload_dir.listdir()), 1) with upload_dir.child("examplefile.txt").open("rb") as f: self.assertEqual(f.read(), b"hello Crossbar!\n")
class Fs(CrawlJob): def __init__(self, cfg, db_pool, data): super(Fs, self).__init__(cfg, db_pool, data) self.cfg = cfg self.data = data self.protocol = 'filesystem' self.creator = None self.fs = None self.connection_closed = False def create(self, path=None): self.fs = FilePath(path) try: return self.fs.getPermissions() except: return False @defer.inlineCallbacks def getDirectory(self, path='/'): self.fs = yield FilePath(path) if not self.fs.getPermissions(): defer.returnValue(False) files = [] for f in self.fs.listdir(): if f == '/': continue fp = path+f fs = FilePath(fp) # dont follow symlinks if fs.realpath().path != fp: continue perm = None isdir = fs.isdir() size = fs.getsize() modified = datetime.utcfromtimestamp(fs.getModificationTime()) df = DiscoveredFile( resource_id=self.data['resource_id'], file_path=path, file_name=f, file_isdir=isdir, file_size=size, file_modified=modified, file_perm=perm ) print '[%s] LIST %s.' % (self.data['resource_name'], fp if not fp.endswith('.') else fp) files.append(df) defer.returnValue(files)
def _get_capinfos(self, username, ip_addr): """ Get capinfos of user's pcap. This should be run afte the account has expired and after executing pcapsummarizer on the user's directory. :param username (str): account username :param ip_addr (IPv4Address): IP address allocated for the account. :return: String with capinfos content. """ user_dir = "{}_{}".format(username, str(ip_addr)) user_dir = os.path.join(self.path['pcaps'], user_dir) user_dir_fp = FilePath(user_dir) capinfos = None for f in user_dir_fp.listdir(): filename = os.path.join(user_dir, f) fp = FilePath(filename) if re.match(".*\.capinfos$", fp.basename()): if capinfos is None: capinfos = fp.getContent() else: capinfos = "{}\n\n{}".format(capinfos, fp.getContent()) return capinfos
def _setupTheme(self): """ Download and setup the theme. """ logging.info("\tBuilding theme...") try: tarball = urlopen(self.theme_url).read() except HTTPError: print("Error downloading theme from %s" % self.theme_url) sys.exit(1) workPath = FilePath(mkdtemp()) sourceFile = workPath.child("theme.tar.gz") # change dir to fix issue with sphinx & themes os.chdir(self.docPath.path) o = open(sourceFile.path , "w") o.write(tarball) o.close() tar = opentar(sourceFile.path, mode='r:*') tar.extractall(workPath.path) theme = None for d in workPath.listdir(): theme = workPath.child(d) if theme.isdir(): theme = theme.child("source").child("themes") dest = self.docPath.child("themes") theme.moveTo(dest) break
def __init__(self, static_path, templates_path): resource.Resource.__init__(self) self.log = logging.getLogger('webui.core') for action in 'online', 'offline', 'scan': self.putChild(action, WebUIAction(self, 'nm_{}'.format(action))) if not isinstance(static_path, FilePath): static_path = FilePath(static_path) for p in static_path.listdir(): self.putChild(p, File(static_path.child(p).path)) if not isinstance(templates_path, FilePath): templates_path = FilePath(templates_path) self.templates = jinja2.Environment( loader=jinja2.FileSystemLoader(templates_path.path) ) self.templates.filters['json'] = self.jinja2_json self.templates.filters['unless_false'] = self.jinja2_unless_false self.events = ClientEvents(self.handle_command) self.putChild('events', SockJSResource(self.events))
def list_measurements(compute_size=False, order=None): measurements = [] measurement_path = FilePath(config.measurements_directory) if not measurement_path.exists(): return measurements for measurement_id in measurement_path.listdir(): try: measurements.append(get_measurement(measurement_id, compute_size)) except: log.err("Failed to get metadata for measurement {0}".format(measurement_id)) if order is None: return measurements if order.lower() in ['asc', 'desc']: reverse = {'asc': False, 'desc': True}[order.lower()] measurements.sort(key=operator.itemgetter('test_start_time'), reverse=reverse) return measurements else: raise ValueError("order must be either 'asc' 'desc' or None")
def __init__(self, static_path, templates_path): resource.Resource.__init__(self) self.log = logging.getLogger('webui.core') for action in 'online', 'offline', 'scan': self.putChild(action, WebUIAction(self, 'nm_{}'.format(action))) if not isinstance(static_path, FilePath): static_path = FilePath(static_path) for p in static_path.listdir(): self.putChild(p, File(static_path.child(p).path)) if not isinstance(templates_path, FilePath): templates_path = FilePath(templates_path) self.templates = jinja2.Environment( loader=jinja2.FileSystemLoader(templates_path.path)) self.templates.filters['json'] = self.jinja2_json self.templates.filters['unless_false'] = self.jinja2_unless_false self.events = ClientEvents(self.handle_command) self.putChild('events', SockJSResource(self.events))
def __init__(self, static_path, templates_path): resource.Resource.__init__(self) self.log = logging.getLogger("webui.core") for action in b"online", b"offline", b"scan": self.putChild(action, WebUIAction(self, "nm_{}".format(action))) if not isinstance(static_path, FilePath): static_path = FilePath(static_path) for p in static_path.listdir(): self.putChild(bytes(p), File(static_path.child(p).path)) if not isinstance(templates_path, FilePath): templates_path = FilePath(templates_path) self.templates = jinja2.Environment( loader=jinja2.FileSystemLoader(templates_path.path) ) self.templates.filters["json"] = self.jinja2_json self.templates.filters["unless_false"] = self.jinja2_unless_false self.events = ClientEvents(self.handle_command) self.putChild("events", SockJSResource(self.events))
def list_measurements(compute_size=False, order=None): measurements = [] measurement_path = FilePath(config.measurements_directory) if not measurement_path.exists(): return measurements for measurement_id in measurement_path.listdir(): try: measurements.append(get_measurement(measurement_id, compute_size)) except Exception as exc: log.err("Failed to get metadata for measurement {0}".format(measurement_id)) log.exception(exc) if order is None: return measurements if order.lower() in ['asc', 'desc']: reverse = {'asc': False, 'desc': True}[order.lower()] measurements.sort(key=operator.itemgetter('test_start_time'), reverse=reverse) return measurements else: raise ValueError("order must be either 'asc' 'desc' or None")
class FileUploadResource(Resource): """ Twisted Web resource that handles file uploads over `HTTP/POST` requests. """ log = make_logger() def __init__(self, upload_directory, temp_directory, form_fields, upload_session, options=None): """ :param upload_directory: The target directory where uploaded files will be stored. :type upload_directory: str :param temp_directory: A temporary directory where chunks of a file being uploaded are stored. :type temp_directory: str :param form_fields: Names of HTML form fields used for uploading. :type form_fields: dict :param upload_session: An instance of `ApplicationSession` used for publishing progress events. :type upload_session: obj :param options: Options for file upload. :type options: dict or None """ Resource.__init__(self) self._dir = FilePath(upload_directory) self._tempDir = FilePath(temp_directory) self._form_fields = form_fields self._fileupload_session = upload_session self._options = options or {} self._max_file_size = self._options.get('max_file_size', 10 * 1024 * 1024) self._fileTypes = self._options.get('file_types', None) self._file_permissions = self._options.get('file_permissions', None) # track uploaded files / chunks self._uploads = {} # scan the temp dir for uploaded chunks and fill the _uploads dict with it # so existing uploads can be resumed for fileTempDir in self._tempDir.listdir(): ft = self._tempDir.child(fileTempDir) if ft.isdir(): self._uploads[fileTempDir] = { 'chunk_list': {}, 'origin': 'startup' } for chunk in ft.listdir(): if chunk[:6] == 'chunk_': self._uploads[fileTempDir]['chunk_list'][int( chunk[6:])] = True self.log.debug("Scanned pending uploads: {uploads}", uploads=self._uploads) def render_POST(self, request): headers = { x.decode('iso-8859-1'): y.decode('iso-8859-1') for x, y in request.getAllHeaders().items() } origin = headers['host'] content = cgi.FieldStorage(fp=request.content, headers=headers, environ={"REQUEST_METHOD": "POST"}) f = self._form_fields filename = content[f['file_name']].value totalSize = int(content[f['total_size']].value) totalChunks = int(content[f['total_chunks']].value) chunkSize = int(content[f['chunk_size']].value) chunkNumber = int(content[f['chunk_number']].value) fileContent = content[f['content']].value fileId = filename # # prepare user specific upload areas # # NOT YET IMPLEMENTED # # # if 'auth_id' in f and f['auth_id'] in content: # auth_id = content[f['auth_id']].value # mydir = os.path.join(self._dir, auth_id) # my_temp_dir = os.path.join(self._tempDir, auth_id) # # # check if auth_id is a valid directory_name # # # if auth_id != auth_id.encode('ascii', 'ignore'): # msg = "The requestor auth_id must be an ascii string." # if self._debug: # log.msg(msg) # # 415 Unsupported Media Type # request.setResponseCode(415, msg) # return msg # else: # auth_id = 'anonymous' # create user specific folder # mydir = self._dir # my_temp_dir = self._tempDir # if not os.path.exists(mydir): # os.makedirs(mydir) # if not os.path.exists(my_temp_dir): # os.makedirs(my_temp_dir) if 'on_progress' in f and f[ 'on_progress'] in content and self._fileupload_session != {}: topic = content[f['on_progress']].value if 'session' in f and f['session'] in content: session = int(content[f['session']].value) publish_options = PublishOptions(eligible=[session]) else: publish_options = None def fileupload_publish(payload): self._fileupload_session.publish(topic, payload, options=publish_options) else: def fileupload_publish(payload): pass # Register upload right at the start to avoid overlapping upload conflicts if fileId not in self._uploads: self._uploads[fileId] = {'chunk_list': {}, 'origin': origin} chunk_is_first = True else: chunk_is_first = False self.log.debug( 'Started upload of file: file_name={file_name}, total_size={total_size}, total_chunks={total_chunks}, chunk_size={chunk_size}, chunk_number={chunk_number}', file_name=fileId, total_size=totalSize, total_chunks=totalChunks, chunk_size=chunkSize, chunk_number=chunkNumber) # check file size # if totalSize > self._max_file_size: msg = "Size {} of file to be uploaded exceeds maximum {}".format( totalSize, self._max_file_size) self.log.debug(msg) # 413 Request Entity Too Large request.setResponseCode(413, msg.encode('utf8')) return msg.encode('utf8') # check file extensions # extension = os.path.splitext(filename)[1] if self._fileTypes and extension not in self._fileTypes: msg = "Type '{}' of file to be uploaded is in allowed types {}".format( extension, self._fileTypes) self.log.debug(msg) # 415 Unsupported Media Type request.setResponseCode(415, msg.encode('utf8')) return msg.encode('utf8') # check if another session is uploading this file already # If the chunks are read at startup of crossbar any client may resume the pending upload ! # try: upl = self._uploads[fileId] if upl['origin'] != origin and upl['origin'] != 'startup': msg = "File being uploaded is already uploaded in a different session" self.log.debug(msg) # 409 Conflict request.setResponseCode(409, msg.encode('utf8')) return msg.encode('utf8') except Exception: pass # TODO: check mime type fileTempDir = self._tempDir.child(fileId) chunkName = fileTempDir.child('chunk_' + str(chunkNumber)) _chunkName = fileTempDir.child('#kfhfkzuru578e38viokbjhfvz4w__' + 'chunk_' + str(chunkNumber)) if chunk_is_first: # first chunk of file # clean the temp dir once per file upload self._remove_stale_uploads() # publish file upload start # fileupload_publish({ "id": fileId, "chunk": chunkNumber, "name": filename, "total": totalSize, "remaining": totalSize, "status": "started", "progress": 0. }) if totalChunks == 1: # only one chunk overall -> write file directly finalFileName = self._dir.child(fileId) _finalFileName = self._dir.child( '#kfhfkzuru578e38viokbjhfvz4w__' + fileId) with open(_finalFileName.path, 'wb') as finalFile: finalFile.write(fileContent) _finalFileName.moveTo(finalFileName) self._uploads[fileId]['chunk_list'][chunkNumber] = True if self._file_permissions: perm = int(self._file_permissions, 8) try: finalFileName.chmod(perm) except Exception as e: finalFileName.remove() msg = "Could not change file permissions of uploaded file" self.log.debug(msg) self.log.debug(e) request.setResponseCode(500, msg.encode('utf8')) return msg.encode('utf8') else: self.log.debug( "Changed permissions on {file_name} to {permissions}", file_name=finalFileName, permissions=self._file_permissions) self._uploads.pop(fileId, None) # publish file upload progress to file_progress_URI fileupload_publish({ "id": fileId, "chunk": chunkNumber, "name": filename, "total": totalSize, "remaining": 0, "status": "finished", "progress": 1. }) else: # first of more chunks fileTempDir.makedirs() with open(_chunkName.path, 'wb') as chunk: chunk.write(fileContent) _chunkName.moveTo(chunkName) self._uploads[fileId]['chunk_list'][chunkNumber] = True # publish file upload progress # fileupload_publish({ "id": fileId, "chunk": chunkNumber, "name": filename, "total": totalSize, "remaining": totalSize - chunkSize, "status": "progress", "progress": round(float(chunkSize) / float(totalSize), 3) }) else: # intermediate chunk with open(_chunkName.path, 'wb') as chunk: chunk.write(fileContent) _chunkName.moveTo(chunkName) self._uploads[fileId]['chunk_list'][chunkNumber] = True received = sum( fileTempDir.child(f).getsize() for f in fileTempDir.listdir()) fileupload_publish({ "id": fileId, "chunk": chunkNumber, "name": filename, "total": totalSize, "remaining": totalSize - received, "status": "progress", "progress": round(float(received) / float(totalSize), 3) }) # every chunk has to check if it is the last chunk written, except in a single chunk scenario if totalChunks > 1 and len( self._uploads[fileId]['chunk_list']) == totalChunks: # last chunk self.log.debug('Finished file upload after chunk {chunk_number}', chunk_number=chunkNumber) # Merge all files into one file and remove the temp files # TODO: How to avoid the extra file IO ? finalFileName = self._dir.child(fileId) _finalFileName = self._dir.child( '#kfhf3kz412uru578e38viokbjhfvz4w__' + fileId) with open(_finalFileName.path, 'wb') as finalFile: for tfileName in fileTempDir.listdir(): with open(fileTempDir.child(tfileName).path, 'rb') as tfile: finalFile.write(tfile.read()) _finalFileName.moveTo(finalFileName) if self._file_permissions: perm = int(self._file_permissions, 8) try: finalFileName.chmod(perm) except Exception as e: msg = "file upload resource - could not change file permissions of uploaded file" self.log.debug(msg) self.log.debug(e) request.setResponseCode(500, msg.encode('utf8')) return msg.encode('utf8') else: self.log.debug( "Changed permissions on {file_name} to {permissions}", file_name=finalFileName, permissions=self._file_permissions) # publish file upload progress to file_progress_URI fileupload_publish({ "id": fileId, "chunk": chunkNumber, "name": filename, "total": totalSize, "remaining": 0, "status": "finished", "progress": 1. }) # remove the file temp folder self._remove_temp_dir(fileTempDir) self._uploads.pop(fileId, None) request.setResponseCode(200) return b'' def _remove_temp_dir(self, fileTempDir): fileTempDir.remove() def _remove_stale_uploads(self): """ This only works if there is a temp folder exclusive for crossbar file uploads if the system temp folder is used then crossbar creates a "crossbar-uploads" there and uses that as the temp folder for uploads If you don't clean up regularly an attacker could fill up the OS file system """ for _dir in self._tempDir.listdir(): fileTempDir = self._tempDir.child(_dir) if fileTempDir.isdir() and _dir not in self._uploads: self._remove_temp_dir(fileTempDir) def render_GET(self, request): """ This method can be used to check whether a chunk has been uploaded already. It returns with HTTP status code `200` if yes and `404` if not. The request needs to contain the file identifier and the chunk number to check for. """ for param in ['file_name', 'chunk_number']: if not self._form_fields[param].encode( 'iso-8859-1') in request.args: msg = "file upload resource - missing request query parameter '{}', configured from '{}'".format( self._form_fields[param], param) self.log.debug(msg) # 400 Bad Request request.setResponseCode(400, msg.encode('utf8')) return msg.encode('utf8') file_name = request.args[self._form_fields['file_name'].encode( 'iso-8859-1')][0].decode('utf8') chunk_number = int( request.args[self._form_fields['chunk_number'].encode( 'iso-8859-1')][0].decode('utf8')) # a complete upload will be repeated an incomplete upload will be resumed if file_name in self._uploads and chunk_number in self._uploads[ file_name]['chunk_list']: self.log.debug( "Skipping chunk upload {file_name} of chunk {chunk_number}", file_name=file_name, chunk_number=chunk_number) msg = b"chunk of file already uploaded" request.setResponseCode(200, msg) return msg else: msg = b"chunk of file not yet uploaded" request.setResponseCode(404, msg) return msg
class FileUploadResource(Resource): """ Twisted Web resource that handles file uploads over `HTTP/POST` requests. """ log = make_logger() def __init__(self, upload_directory, temp_directory, form_fields, upload_session, options=None): """ :param upload_directory: The target directory where uploaded files will be stored. :type upload_directory: str :param temp_directory: A temporary directory where chunks of a file being uploaded are stored. :type temp_directory: str :param form_fields: Names of HTML form fields used for uploading. :type form_fields: dict :param upload_session: An instance of `ApplicationSession` used for publishing progress events. :type upload_session: obj :param options: Options for file upload. :type options: dict or None """ Resource.__init__(self) self._dir = FilePath(upload_directory) self._tempDir = FilePath(temp_directory) self._form_fields = form_fields self._fileupload_session = upload_session self._options = options or {} self._max_file_size = self._options.get('max_file_size', 10 * 1024 * 1024) self._fileTypes = self._options.get('file_types', None) self._file_permissions = self._options.get('file_permissions', None) # track uploaded files / chunks self._uploads = {} # scan the temp dir for uploaded chunks and fill the _uploads dict with it # so existing uploads can be resumed for fileTempDir in self._tempDir.listdir(): ft = self._tempDir.child(fileTempDir) if ft.isdir(): self._uploads[fileTempDir] = {'chunk_list': {}, 'origin': 'startup'} for chunk in ft.listdir(): if chunk[:6] == 'chunk_': self._uploads[fileTempDir]['chunk_list'][int(chunk[6:])] = True self.log.debug("Scanned pending uploads: {uploads}", uploads=self._uploads) def render_POST(self, request): headers = {x.decode('iso-8859-1'): y.decode('iso-8859-1') for x, y in request.getAllHeaders().items()} origin = headers['host'] content = cgi.FieldStorage( fp=request.content, headers=headers, environ={"REQUEST_METHOD": "POST"}) f = self._form_fields filename = content[f['file_name']].value totalSize = int(content[f['total_size']].value) totalChunks = int(content[f['total_chunks']].value) chunkSize = int(content[f['chunk_size']].value) chunkNumber = int(content[f['chunk_number']].value) fileContent = content[f['content']].value fileId = filename # # prepare user specific upload areas # # NOT YET IMPLEMENTED # # # if 'auth_id' in f and f['auth_id'] in content: # auth_id = content[f['auth_id']].value # mydir = os.path.join(self._dir, auth_id) # my_temp_dir = os.path.join(self._tempDir, auth_id) # # # check if auth_id is a valid directory_name # # # if auth_id != auth_id.encode('ascii', 'ignore'): # msg = "The requestor auth_id must be an ascii string." # if self._debug: # log.msg(msg) # # 415 Unsupported Media Type # request.setResponseCode(415, msg) # return msg # else: # auth_id = 'anonymous' # create user specific folder # mydir = self._dir # my_temp_dir = self._tempDir # if not os.path.exists(mydir): # os.makedirs(mydir) # if not os.path.exists(my_temp_dir): # os.makedirs(my_temp_dir) if 'on_progress' in f and f['on_progress'] in content and self._fileupload_session != {}: topic = content[f['on_progress']].value if 'session' in f and f['session'] in content: session = int(content[f['session']].value) publish_options = PublishOptions(eligible=[session]) else: publish_options = None def fileupload_publish(payload): self._fileupload_session.publish(topic, payload, options=publish_options) else: def fileupload_publish(payload): pass # Register upload right at the start to avoid overlapping upload conflicts if fileId not in self._uploads: self._uploads[fileId] = {'chunk_list': {}, 'origin': origin} chunk_is_first = True else: chunk_is_first = False self.log.debug('Started upload of file: file_name={file_name}, total_size={total_size}, total_chunks={total_chunks}, chunk_size={chunk_size}, chunk_number={chunk_number}', file_name=fileId, total_size=totalSize, total_chunks=totalChunks, chunk_size=chunkSize, chunk_number=chunkNumber) # check file size # if totalSize > self._max_file_size: msg = "Size {} of file to be uploaded exceeds maximum {}".format(totalSize, self._max_file_size) self.log.debug(msg) # 413 Request Entity Too Large request.setResponseCode(413, msg.encode('utf8')) return msg.encode('utf8') # check file extensions # extension = os.path.splitext(filename)[1] if self._fileTypes and extension not in self._fileTypes: msg = "Type '{}' of file to be uploaded is in allowed types {}".format(extension, self._fileTypes) self.log.debug(msg) # 415 Unsupported Media Type request.setResponseCode(415, msg.encode('utf8')) return msg.encode('utf8') # check if another session is uploading this file already # If the chunks are read at startup of crossbar any client may resume the pending upload ! # try: upl = self._uploads[fileId] if upl['origin'] != origin and upl['origin'] != 'startup': msg = "File being uploaded is already uploaded in a different session" self.log.debug(msg) # 409 Conflict request.setResponseCode(409, msg.encode('utf8')) return msg.encode('utf8') except Exception: pass # TODO: check mime type fileTempDir = self._tempDir.child(fileId) chunkName = fileTempDir.child('chunk_' + str(chunkNumber)) _chunkName = fileTempDir.child('#kfhfkzuru578e38viokbjhfvz4w__' + 'chunk_' + str(chunkNumber)) if chunk_is_first: # first chunk of file # clean the temp dir once per file upload self._remove_stale_uploads() # publish file upload start # fileupload_publish({ "id": fileId, "chunk": chunkNumber, "name": filename, "total": totalSize, "remaining": totalSize, "status": "started", "progress": 0. }) if totalChunks == 1: # only one chunk overall -> write file directly finalFileName = self._dir.child(fileId) _finalFileName = self._dir.child('#kfhfkzuru578e38viokbjhfvz4w__' + fileId) with open(_finalFileName.path, 'wb') as finalFile: finalFile.write(fileContent) _finalFileName.moveTo(finalFileName) self._uploads[fileId]['chunk_list'][chunkNumber] = True if self._file_permissions: perm = int(self._file_permissions, 8) try: finalFileName.chmod(perm) except Exception as e: finalFileName.remove() msg = "Could not change file permissions of uploaded file" self.log.debug(msg) self.log.debug(e) request.setResponseCode(500, msg.encode('utf8')) return msg.encode('utf8') else: self.log.debug("Changed permissions on {file_name} to {permissions}", file_name=finalFileName, permissions=self._file_permissions) self._uploads.pop(fileId, None) # publish file upload progress to file_progress_URI fileupload_publish({ "id": fileId, "chunk": chunkNumber, "name": filename, "total": totalSize, "remaining": 0, "status": "finished", "progress": 1. }) else: # first of more chunks fileTempDir.makedirs() with open(_chunkName.path, 'wb') as chunk: chunk.write(fileContent) _chunkName.moveTo(chunkName) self._uploads[fileId]['chunk_list'][chunkNumber] = True # publish file upload progress # fileupload_publish({ "id": fileId, "chunk": chunkNumber, "name": filename, "total": totalSize, "remaining": totalSize - chunkSize, "status": "progress", "progress": round(float(chunkSize) / float(totalSize), 3) }) else: # intermediate chunk with open(_chunkName.path, 'wb') as chunk: chunk.write(fileContent) _chunkName.moveTo(chunkName) self._uploads[fileId]['chunk_list'][chunkNumber] = True received = sum(fileTempDir.child(f).getsize() for f in fileTempDir.listdir()) fileupload_publish({ "id": fileId, "chunk": chunkNumber, "name": filename, "total": totalSize, "remaining": totalSize - received, "status": "progress", "progress": round(float(received) / float(totalSize), 3) }) # every chunk has to check if it is the last chunk written, except in a single chunk scenario if totalChunks > 1 and len(self._uploads[fileId]['chunk_list']) == totalChunks: # last chunk self.log.debug('Finished file upload after chunk {chunk_number}', chunk_number=chunkNumber) # Merge all files into one file and remove the temp files # TODO: How to avoid the extra file IO ? finalFileName = self._dir.child(fileId) _finalFileName = self._dir.child('#kfhf3kz412uru578e38viokbjhfvz4w__' + fileId) with open(_finalFileName.path, 'wb') as finalFile: for tfileName in fileTempDir.listdir(): with open(fileTempDir.child(tfileName).path, 'rb') as tfile: finalFile.write(tfile.read()) _finalFileName.moveTo(finalFileName) if self._file_permissions: perm = int(self._file_permissions, 8) try: finalFileName.chmod(perm) except Exception as e: msg = "file upload resource - could not change file permissions of uploaded file" self.log.debug(msg) self.log.debug(e) request.setResponseCode(500, msg.encode('utf8')) return msg.encode('utf8') else: self.log.debug("Changed permissions on {file_name} to {permissions}", file_name=finalFileName, permissions=self._file_permissions) # publish file upload progress to file_progress_URI fileupload_publish({ "id": fileId, "chunk": chunkNumber, "name": filename, "total": totalSize, "remaining": 0, "status": "finished", "progress": 1. }) # remove the file temp folder self._remove_temp_dir(fileTempDir) self._uploads.pop(fileId, None) request.setResponseCode(200) return b'' def _remove_temp_dir(self, fileTempDir): fileTempDir.remove() def _remove_stale_uploads(self): """ This only works if there is a temp folder exclusive for crossbar file uploads if the system temp folder is used then crossbar creates a "crossbar-uploads" there and uses that as the temp folder for uploads If you don't clean up regularly an attacker could fill up the OS file system """ for _dir in self._tempDir.listdir(): fileTempDir = self._tempDir.child(_dir) if fileTempDir.isdir() and _dir not in self._uploads: self._remove_temp_dir(fileTempDir) def render_GET(self, request): """ This method can be used to check whether a chunk has been uploaded already. It returns with HTTP status code `200` if yes and `404` if not. The request needs to contain the file identifier and the chunk number to check for. """ for param in ['file_name', 'chunk_number']: if not self._form_fields[param].encode('iso-8859-1') in request.args: msg = "file upload resource - missing request query parameter '{}', configured from '{}'".format(self._form_fields[param], param) self.log.debug(msg) # 400 Bad Request request.setResponseCode(400, msg.encode('utf8')) return msg.encode('utf8') file_name = request.args[self._form_fields['file_name'].encode('iso-8859-1')][0].decode('utf8') chunk_number = int(request.args[self._form_fields['chunk_number'].encode('iso-8859-1')][0].decode('utf8')) # a complete upload will be repeated an incomplete upload will be resumed if file_name in self._uploads and chunk_number in self._uploads[file_name]['chunk_list']: self.log.debug("Skipping chunk upload {file_name} of chunk {chunk_number}", file_name=file_name, chunk_number=chunk_number) msg = b"chunk of file already uploaded" request.setResponseCode(200, msg) return msg else: msg = b"chunk of file not yet uploaded" request.setResponseCode(404, msg) return msg
def test_basic(self): """ Upload a basic file using the FileUploadResource, in just a single chunk. """ upload_dir = FilePath(self.mktemp()) upload_dir.makedirs() temp_dir = FilePath(self.mktemp()) temp_dir.makedirs() fields = { "file_name": "resumableFilename", "mime_type": "resumableType", "total_size": "resumableTotalSize", "chunk_number": "resumableChunkNumber", "chunk_size": "resumableChunkSize", "total_chunks": "resumableTotalChunks", "content": "file", "on_progress": "on_progress", "session": "session" } mock_session = Mock() resource = FileUploadResource(upload_dir.path, temp_dir.path, fields, mock_session) mp = Multipart() mp.add_part(b"resumableChunkNumber", b"1") mp.add_part(b"resumableChunkSize", b"1048576") mp.add_part(b"resumableCurrentChunkSize", b"16") mp.add_part(b"resumableTotalSize", b"16") mp.add_part(b"resumableType", b"text/plain") mp.add_part(b"resumableIdentifier", b"16-examplefiletxt") mp.add_part(b"resumableFilename", b"examplefile.txt") mp.add_part(b"resumableRelativePath", b"examplefile.txt") mp.add_part(b"resumableTotalChunks", b"1") mp.add_part(b"on_progress", b"com.example.upload.on_progress") mp.add_part(b"session", b"6891276359801283") mp.add_part(b"file", b"hello Crossbar!\n", content_type=b"application/octet-stream", filename=b"blob") body, headers = mp.render() d = renderResource( resource, b"/", method="POST", headers=headers, body=body ) res = self.successResultOf(d) self.assertEqual(res.code, 200) self.assertEqual(len(mock_session.method_calls), 2) # Starting the upload self.assertEqual(mock_session.method_calls[0][1][0], u"com.example.upload.on_progress") self.assertEqual(mock_session.method_calls[0][1][1]["status"], "started") self.assertEqual(mock_session.method_calls[0][1][1]["id"], "examplefile.txt") # Upload complete self.assertEqual(mock_session.method_calls[1][1][0], u"com.example.upload.on_progress") self.assertEqual(mock_session.method_calls[1][1][1]["status"], "finished") self.assertEqual(mock_session.method_calls[1][1][1]["id"], "examplefile.txt") # Nothing in the temp dir, one file in the upload self.assertEqual(len(temp_dir.listdir()), 0) self.assertEqual(len(upload_dir.listdir()), 1) with upload_dir.child("examplefile.txt").open("rb") as f: self.assertEqual(f.read(), b"hello Crossbar!\n")
def test_resumed_upload(self): """ Uploading part of a file, simulating a Crossbar restart, and continuing to upload works. """ upload_dir = FilePath(self.mktemp()) upload_dir.makedirs() temp_dir = FilePath(self.mktemp()) temp_dir.makedirs() fields = { "file_name": "resumableFilename", "mime_type": "resumableType", "total_size": "resumableTotalSize", "chunk_number": "resumableChunkNumber", "chunk_size": "resumableChunkSize", "total_chunks": "resumableTotalChunks", "content": "file", "on_progress": "on_progress", "session": "session" } mock_session = Mock() resource = FileUploadResource(upload_dir.path, temp_dir.path, fields, mock_session) # Make some stuff in the temp dir that wasn't there when it started but # is put there before a file upload temp_dir.child("otherjunk").makedirs() # # Chunk 1 mp = Multipart() mp.add_part(b"resumableChunkNumber", b"1") mp.add_part(b"resumableChunkSize", b"10") mp.add_part(b"resumableCurrentChunkSize", b"10") mp.add_part(b"resumableTotalSize", b"16") mp.add_part(b"resumableType", b"text/plain") mp.add_part(b"resumableIdentifier", b"16-examplefiletxt") mp.add_part(b"resumableFilename", b"examplefile.txt") mp.add_part(b"resumableRelativePath", b"examplefile.txt") mp.add_part(b"resumableTotalChunks", b"2") mp.add_part(b"on_progress", b"com.example.upload.on_progress") mp.add_part(b"session", b"6891276359801283") mp.add_part(b"file", b"hello Cros", content_type=b"application/octet-stream", filename=b"blob") body, headers = mp.render() d = renderResource( resource, b"/", method="POST", headers=headers, body=body ) res = self.successResultOf(d) self.assertEqual(res.code, 200) # One directory in the temp dir, nothing in the upload dir, temp dir # contains one chunk self.assertEqual(len(temp_dir.listdir()), 1) self.assertEqual(len(temp_dir.child("examplefile.txt").listdir()), 1) with temp_dir.child("examplefile.txt").child("chunk_1").open("rb") as f: self.assertEqual(f.read(), b"hello Cros") self.assertEqual(len(upload_dir.listdir()), 0) del resource # Add some random junk in there that Crossbar isn't expecting temp_dir.child("junk").setContent(b"just some junk") temp_dir.child("examplefile.txt").child("hi").setContent(b"what") # Simulate restarting Crossbar by reinitialising the FileUploadResource resource = FileUploadResource(upload_dir.path, temp_dir.path, fields, mock_session) # # Chunk 2 mp = Multipart() mp.add_part(b"resumableChunkNumber", b"2") mp.add_part(b"resumableChunkSize", b"10") mp.add_part(b"resumableCurrentChunkSize", b"6") mp.add_part(b"resumableTotalSize", b"16") mp.add_part(b"resumableType", b"text/plain") mp.add_part(b"resumableIdentifier", b"16-examplefiletxt") mp.add_part(b"resumableFilename", b"examplefile.txt") mp.add_part(b"resumableRelativePath", b"examplefile.txt") mp.add_part(b"resumableTotalChunks", b"2") mp.add_part(b"on_progress", b"com.example.upload.on_progress") mp.add_part(b"session", b"6891276359801283") mp.add_part(b"file", b"sbar!\n", content_type=b"application/octet-stream", filename=b"blob") body, headers = mp.render() d = renderResource( resource, b"/", method="POST", headers=headers, body=body ) res = self.successResultOf(d) self.assertEqual(res.code, 200) self.assertEqual(len(mock_session.method_calls), 4) # Starting the upload self.assertEqual(mock_session.method_calls[0][1][0], u"com.example.upload.on_progress") self.assertEqual(mock_session.method_calls[0][1][1]["status"], "started") self.assertEqual(mock_session.method_calls[0][1][1]["id"], "examplefile.txt") self.assertEqual(mock_session.method_calls[0][1][1]["chunk"], 1) # Progress, first chunk done self.assertEqual(mock_session.method_calls[1][1][0], u"com.example.upload.on_progress") self.assertEqual(mock_session.method_calls[1][1][1]["status"], "progress") self.assertEqual(mock_session.method_calls[1][1][1]["id"], "examplefile.txt") self.assertEqual(mock_session.method_calls[1][1][1]["chunk"], 1) # Progress, second chunk done self.assertEqual(mock_session.method_calls[2][1][0], u"com.example.upload.on_progress") self.assertEqual(mock_session.method_calls[2][1][1]["status"], "progress") self.assertEqual(mock_session.method_calls[2][1][1]["id"], "examplefile.txt") self.assertEqual(mock_session.method_calls[2][1][1]["chunk"], 2) # Upload complete self.assertEqual(mock_session.method_calls[3][1][0], u"com.example.upload.on_progress") self.assertEqual(mock_session.method_calls[3][1][1]["status"], "finished") self.assertEqual(mock_session.method_calls[3][1][1]["id"], "examplefile.txt") self.assertEqual(mock_session.method_calls[3][1][1]["chunk"], 2) # No item in the temp dir which we made earlier, one item in the # upload dir. Otherjunk is removed because it belongs to no upload. self.assertEqual(len(temp_dir.listdir()), 0) self.assertEqual(len(upload_dir.listdir()), 1) with upload_dir.child("examplefile.txt").open("rb") as f: self.assertEqual(f.read(), b"hello Crossbar!\n")
class DeckStore(object): def __init__(self, enabled_directory=config.decks_enabled_directory, available_directory=config.decks_available_directory): self.enabled_directory = FilePath(enabled_directory) self.available_directory = FilePath(available_directory) self._cache = {} self._cache_stale = True def _list(self): if self._cache_stale: self._update_cache() for deck_id, deck in self._cache.iteritems(): yield (deck_id, deck) def list(self): decks = [] for deck_id, deck in self._list(): decks.append((deck_id, deck)) return decks def list_enabled(self): decks = [] for deck_id, deck in self._list(): if not self.is_enabled(deck_id): continue decks.append((deck_id, deck)) return decks def is_enabled(self, deck_id): return self.enabled_directory.child(deck_id + '.yaml').exists() def enable(self, deck_id): deck_path = self.available_directory.child(deck_id + '.yaml') if not deck_path.exists(): raise DeckNotFound(deck_id) deck_enabled_path = self.enabled_directory.child(deck_id + '.yaml') try: deck_path.linkTo(deck_enabled_path) except OSError as ose: if ose.errno != errno.EEXIST: raise def disable(self, deck_id): deck_enabled_path = self.enabled_directory.child(deck_id + '.yaml') if not deck_enabled_path.exists(): raise DeckNotFound(deck_id) deck_enabled_path.remove() def _update_cache(self): new_cache = {} for deck_path in self.available_directory.listdir(): if not deck_path.endswith('.yaml'): continue deck = NGDeck( deck_path=self.available_directory.child(deck_path).path) new_cache[deck.id] = deck self._cache = new_cache self._cache_stale = False def get(self, deck_id): if self._cache_stale: self._update_cache() try: return deepcopy(self._cache[deck_id]) except KeyError: raise DeckNotFound(deck_id)
def test_multichunk(self): """ Uploading files that are in multiple chunks works. """ upload_dir = FilePath(self.mktemp()) upload_dir.makedirs() temp_dir = FilePath(self.mktemp()) temp_dir.makedirs() fields = { "file_name": "resumableFilename", "mime_type": "resumableType", "total_size": "resumableTotalSize", "chunk_number": "resumableChunkNumber", "chunk_size": "resumableChunkSize", "total_chunks": "resumableTotalChunks", "content": "file", "on_progress": "on_progress", "session": "session" } mock_session = Mock() resource = FileUploadResource(upload_dir.path, temp_dir.path, fields, mock_session) # # Chunk 1 mp = Multipart() mp.add_part(b"resumableChunkNumber", b"1") mp.add_part(b"resumableChunkSize", b"10") mp.add_part(b"resumableCurrentChunkSize", b"10") mp.add_part(b"resumableTotalSize", b"16") mp.add_part(b"resumableType", b"text/plain") mp.add_part(b"resumableIdentifier", b"16-examplefiletxt") mp.add_part(b"resumableFilename", b"examplefile.txt") mp.add_part(b"resumableRelativePath", b"examplefile.txt") mp.add_part(b"resumableTotalChunks", b"2") mp.add_part(b"on_progress", b"com.example.upload.on_progress") mp.add_part(b"session", b"6891276359801283") mp.add_part(b"file", b"hello Cros", content_type=b"application/octet-stream", filename=b"blob") body, headers = mp.render() d = renderResource( resource, b"/", method="POST", headers=headers, body=body ) res = self.successResultOf(d) res.setResponseCode.assert_called_once_with(200) # One directory in the temp dir, nothing in the upload dir, temp dir # contains one chunk self.assertEqual(len(temp_dir.listdir()), 1) self.assertEqual(len(temp_dir.child("examplefile.txt").listdir()), 1) with temp_dir.child("examplefile.txt").child("chunk_1").open("rb") as f: self.assertEqual(f.read(), b"hello Cros") self.assertEqual(len(upload_dir.listdir()), 0) # # Chunk 2 mp = Multipart() mp.add_part(b"resumableChunkNumber", b"2") mp.add_part(b"resumableChunkSize", b"10") mp.add_part(b"resumableCurrentChunkSize", b"6") mp.add_part(b"resumableTotalSize", b"16") mp.add_part(b"resumableType", b"text/plain") mp.add_part(b"resumableIdentifier", b"16-examplefiletxt") mp.add_part(b"resumableFilename", b"examplefile.txt") mp.add_part(b"resumableRelativePath", b"examplefile.txt") mp.add_part(b"resumableTotalChunks", b"2") mp.add_part(b"on_progress", b"com.example.upload.on_progress") mp.add_part(b"session", b"6891276359801283") mp.add_part(b"file", b"sbar!\n", content_type=b"application/octet-stream", filename=b"blob") body, headers = mp.render() d = renderResource( resource, b"/", method="POST", headers=headers, body=body ) res = self.successResultOf(d) res.setResponseCode.assert_called_once_with(200) self.assertEqual(len(mock_session.method_calls), 4) # Starting the upload self.assertEqual(mock_session.method_calls[0][1][0], u"com.example.upload.on_progress") self.assertEqual(mock_session.method_calls[0][1][1]["status"], "started") self.assertEqual(mock_session.method_calls[0][1][1]["id"], "examplefile.txt") self.assertEqual(mock_session.method_calls[0][1][1]["chunk"], 1) # Progress, first chunk done self.assertEqual(mock_session.method_calls[1][1][0], u"com.example.upload.on_progress") self.assertEqual(mock_session.method_calls[1][1][1]["status"], "progress") self.assertEqual(mock_session.method_calls[1][1][1]["id"], "examplefile.txt") self.assertEqual(mock_session.method_calls[1][1][1]["chunk"], 1) # Progress, second chunk done self.assertEqual(mock_session.method_calls[2][1][0], u"com.example.upload.on_progress") self.assertEqual(mock_session.method_calls[2][1][1]["status"], "progress") self.assertEqual(mock_session.method_calls[2][1][1]["id"], "examplefile.txt") self.assertEqual(mock_session.method_calls[2][1][1]["chunk"], 2) # Upload complete self.assertEqual(mock_session.method_calls[3][1][0], u"com.example.upload.on_progress") self.assertEqual(mock_session.method_calls[3][1][1]["status"], "finished") self.assertEqual(mock_session.method_calls[3][1][1]["id"], "examplefile.txt") self.assertEqual(mock_session.method_calls[3][1][1]["chunk"], 2) # Nothing in the temp dir, one file in the upload self.assertEqual(len(temp_dir.listdir()), 0) self.assertEqual(len(upload_dir.listdir()), 1) with upload_dir.child("examplefile.txt").open("rb") as f: self.assertEqual(f.read(), b"hello Crossbar!\n")
class DeckStore(object): def __init__(self, enabled_directory=config.decks_enabled_directory, available_directory=config.decks_available_directory): self.enabled_directory = FilePath(enabled_directory) self.available_directory = FilePath(available_directory) self._cache = {} self._cache_stale = True def _list(self): if self._cache_stale: self._update_cache() for deck_id, deck in self._cache.iteritems(): yield (deck_id, deck) def list(self): decks = [] for deck_id, deck in self._list(): decks.append((deck_id, deck)) return decks def list_enabled(self): decks = [] for deck_id, deck in self._list(): if not self.is_enabled(deck_id): continue decks.append((deck_id, deck)) return decks def is_enabled(self, deck_id): return self.enabled_directory.child(deck_id + '.yaml').exists() def enable(self, deck_id): deck_path = self.available_directory.child(deck_id + '.yaml') if not deck_path.exists(): raise DeckNotFound(deck_id) deck_enabled_path = self.enabled_directory.child(deck_id + '.yaml') try: deck_path.linkTo(deck_enabled_path) except OSError as ose: if ose.errno != errno.EEXIST: raise def disable(self, deck_id): deck_enabled_path = self.enabled_directory.child(deck_id + '.yaml') if not deck_enabled_path.exists(): raise DeckNotFound(deck_id) deck_enabled_path.remove() def _update_cache(self): new_cache = {} for deck_path in self.available_directory.listdir(): if not deck_path.endswith('.yaml'): continue deck = NGDeck( deck_path=self.available_directory.child(deck_path).path ) new_cache[deck.id] = deck self._cache = new_cache self._cache_stale = False def get(self, deck_id): if self._cache_stale: self._update_cache() try: return deepcopy(self._cache[deck_id]) except KeyError: raise DeckNotFound(deck_id)
class BuildScriptsTests(TestCase): """ Tests for L{dist.build_scripts_twisted}. """ def setUp(self): self.source = FilePath(self.mktemp()) self.target = FilePath(self.mktemp()) self.source.makedirs() self.addCleanup(os.chdir, os.getcwd()) os.chdir(self.source.path) def buildScripts(self): """ Write 3 types of scripts and run the L{build_scripts_twisted} command. """ self.writeScript(self.source, "script1", ("#! /usr/bin/env python2.7\n" "# bogus script w/ Python sh-bang\n" "pass\n")) self.writeScript(self.source, "script2.py", ("#!/usr/bin/python\n" "# bogus script w/ Python sh-bang\n" "pass\n")) self.writeScript(self.source, "shell.sh", ("#!/bin/sh\n" "# bogus shell script w/ sh-bang\n" "exit 0\n")) expected = ['script1', 'script2.py', 'shell.sh'] cmd = self.getBuildScriptsCmd(self.target, [self.source.child(fn).path for fn in expected]) cmd.finalize_options() cmd.run() return self.target.listdir() def getBuildScriptsCmd(self, target, scripts): """ Create a distutils L{Distribution} with a L{DummyCommand} and wrap it in L{build_scripts_twisted}. @type target: L{FilePath} """ dist = Distribution() dist.scripts = scripts dist.command_obj["build"] = DummyCommand( build_scripts = target.path, force = 1, executable = sys.executable ) return build_scripts_twisted(dist) def writeScript(self, dir, name, text): """ Write the script to disk. """ with open(dir.child(name).path, "w") as f: f.write(text) def test_notWindows(self): """ L{build_scripts_twisted} does not rename scripts on non-Windows platforms. """ self.patch(os, "name", "twisted") built = self.buildScripts() for name in ['script1', 'script2.py', 'shell.sh']: self.assertTrue(name in built) def test_windows(self): """ L{build_scripts_twisted} renames scripts so they end with '.py' on the Windows platform. """ self.patch(os, "name", "nt") built = self.buildScripts() for name in ['script1.py', 'script2.py', 'shell.sh.py']: self.assertTrue(name in built)
def test_resumed_upload(self): """ Uploading part of a file, simulating a Crossbar restart, and continuing to upload works. """ upload_dir = FilePath(self.mktemp()) upload_dir.makedirs() temp_dir = FilePath(self.mktemp()) temp_dir.makedirs() fields = { "file_name": "resumableFilename", "mime_type": "resumableType", "total_size": "resumableTotalSize", "chunk_number": "resumableChunkNumber", "chunk_size": "resumableChunkSize", "total_chunks": "resumableTotalChunks", "content": "file", "on_progress": "on_progress", "session": "session", } mock_session = Mock() resource = FileUploadResource(upload_dir.path, temp_dir.path, fields, mock_session) # Make some stuff in the temp dir that wasn't there when it started but # is put there before a file upload temp_dir.child("otherjunk").makedirs() # # Chunk 1 mp = Multipart() mp.add_part(b"resumableChunkNumber", b"1") mp.add_part(b"resumableChunkSize", b"10") mp.add_part(b"resumableCurrentChunkSize", b"10") mp.add_part(b"resumableTotalSize", b"16") mp.add_part(b"resumableType", b"text/plain") mp.add_part(b"resumableIdentifier", b"16-examplefiletxt") mp.add_part(b"resumableFilename", b"examplefile.txt") mp.add_part(b"resumableRelativePath", b"examplefile.txt") mp.add_part(b"resumableTotalChunks", b"2") mp.add_part(b"on_progress", b"com.example.upload.on_progress") mp.add_part(b"session", b"6891276359801283") mp.add_part(b"file", b"hello Cros", content_type=b"application/octet-stream", filename=b"blob") body, headers = mp.render() d = renderResource(resource, b"/", method="POST", headers=headers, body=body) res = self.successResultOf(d) self.assertEqual(res.code, 200) # One directory in the temp dir, nothing in the upload dir, temp dir # contains one chunk self.assertEqual(len(temp_dir.listdir()), 1) self.assertEqual(len(temp_dir.child("examplefile.txt").listdir()), 1) with temp_dir.child("examplefile.txt").child("chunk_1").open("rb") as f: self.assertEqual(f.read(), b"hello Cros") self.assertEqual(len(upload_dir.listdir()), 0) del resource # Add some random junk in there that Crossbar isn't expecting temp_dir.child("junk").setContent(b"just some junk") temp_dir.child("examplefile.txt").child("hi").setContent(b"what") # Simulate restarting Crossbar by reinitialising the FileUploadResource resource = FileUploadResource(upload_dir.path, temp_dir.path, fields, mock_session) # # Chunk 2 mp = Multipart() mp.add_part(b"resumableChunkNumber", b"2") mp.add_part(b"resumableChunkSize", b"10") mp.add_part(b"resumableCurrentChunkSize", b"6") mp.add_part(b"resumableTotalSize", b"16") mp.add_part(b"resumableType", b"text/plain") mp.add_part(b"resumableIdentifier", b"16-examplefiletxt") mp.add_part(b"resumableFilename", b"examplefile.txt") mp.add_part(b"resumableRelativePath", b"examplefile.txt") mp.add_part(b"resumableTotalChunks", b"2") mp.add_part(b"on_progress", b"com.example.upload.on_progress") mp.add_part(b"session", b"6891276359801283") mp.add_part(b"file", b"sbar!\n", content_type=b"application/octet-stream", filename=b"blob") body, headers = mp.render() d = renderResource(resource, b"/", method="POST", headers=headers, body=body) res = self.successResultOf(d) self.assertEqual(res.code, 200) self.assertEqual(len(mock_session.method_calls), 4) # Starting the upload self.assertEqual(mock_session.method_calls[0][1][0], u"com.example.upload.on_progress") self.assertEqual(mock_session.method_calls[0][1][1]["status"], "started") self.assertEqual(mock_session.method_calls[0][1][1]["id"], "examplefile.txt") self.assertEqual(mock_session.method_calls[0][1][1]["chunk"], 1) # Progress, first chunk done self.assertEqual(mock_session.method_calls[1][1][0], u"com.example.upload.on_progress") self.assertEqual(mock_session.method_calls[1][1][1]["status"], "progress") self.assertEqual(mock_session.method_calls[1][1][1]["id"], "examplefile.txt") self.assertEqual(mock_session.method_calls[1][1][1]["chunk"], 1) # Progress, second chunk done self.assertEqual(mock_session.method_calls[2][1][0], u"com.example.upload.on_progress") self.assertEqual(mock_session.method_calls[2][1][1]["status"], "progress") self.assertEqual(mock_session.method_calls[2][1][1]["id"], "examplefile.txt") self.assertEqual(mock_session.method_calls[2][1][1]["chunk"], 2) # Upload complete self.assertEqual(mock_session.method_calls[3][1][0], u"com.example.upload.on_progress") self.assertEqual(mock_session.method_calls[3][1][1]["status"], "finished") self.assertEqual(mock_session.method_calls[3][1][1]["id"], "examplefile.txt") self.assertEqual(mock_session.method_calls[3][1][1]["chunk"], 2) # No item in the temp dir which we made earlier, one item in the # upload dir. Otherjunk is removed because it belongs to no upload. self.assertEqual(len(temp_dir.listdir()), 0) self.assertEqual(len(upload_dir.listdir()), 1) with upload_dir.child("examplefile.txt").open("rb") as f: self.assertEqual(f.read(), b"hello Crossbar!\n")
def test_multichunk_shuffle(self): """ Uploading files that are in multiple chunks and are uploaded in different order works. """ upload_dir = FilePath(self.mktemp()) upload_dir.makedirs() temp_dir = FilePath(self.mktemp()) temp_dir.makedirs() fields = { "file_name": "resumableFilename", "mime_type": "resumableType", "total_size": "resumableTotalSize", "chunk_number": "resumableChunkNumber", "chunk_size": "resumableChunkSize", "total_chunks": "resumableTotalChunks", "content": "file", "on_progress": "on_progress", "session": "session", } mock_session = Mock() resource = FileUploadResource(upload_dir.path, temp_dir.path, fields, mock_session) # # Chunk 2 multipart_body = b"""-----------------------------42560029919436807832069165364\r\nContent-Disposition: form-data; name="resumableChunkNumber"\r\n\r\n2\r\n-----------------------------42560029919436807832069165364\r\nContent-Disposition: form-data; name="resumableChunkSize"\r\n\r\n10\r\n-----------------------------42560029919436807832069165364\r\nContent-Disposition: form-data; name="resumableCurrentChunkSize"\r\n\r\n6\r\n-----------------------------42560029919436807832069165364\r\nContent-Disposition: form-data; name="resumableTotalSize"\r\n\r\n16\r\n-----------------------------42560029919436807832069165364\r\nContent-Disposition: form-data; name="resumableType"\r\n\r\ntext/plain\r\n-----------------------------42560029919436807832069165364\r\nContent-Disposition: form-data; name="resumableIdentifier"\r\n\r\n16-examplefiletxt\r\n-----------------------------42560029919436807832069165364\r\nContent-Disposition: form-data; name="resumableFilename"\r\n\r\nexamplefile.txt\r\n-----------------------------42560029919436807832069165364\r\nContent-Disposition: form-data; name="resumableRelativePath"\r\n\r\nexamplefile.txt\r\n-----------------------------42560029919436807832069165364\r\nContent-Disposition: form-data; name="resumableTotalChunks"\r\n\r\n2\r\n-----------------------------42560029919436807832069165364\r\nContent-Disposition: form-data; name="on_progress"\r\n\r\ncom.example.upload.on_progress\r\n-----------------------------42560029919436807832069165364\r\nContent-Disposition: form-data; name="session"\r\n\r\n8887465641628580\r\n-----------------------------42560029919436807832069165364\r\nContent-Disposition: form-data; name="file"; filename="blob"\r\nContent-Type: application/octet-stream\r\n\r\nsbar!\n\r\n-----------------------------42560029919436807832069165364--\r\n""" d = renderResource( resource, b"/", method="POST", headers={ b"content-type": [ b"multipart/form-data; boundary=---------------------------42560029919436807832069165364" ], b"Content-Length": [b"1688"], }, body=multipart_body, ) res = self.successResultOf(d) self.assertEqual(res.code, 200) # One directory in the temp dir, nothing in the upload dir, temp dir # contains one chunk self.assertEqual(len(temp_dir.listdir()), 1) self.assertEqual(len(temp_dir.child("examplefile.txt").listdir()), 1) with temp_dir.child("examplefile.txt").child("chunk_2").open("rb") as f: self.assertEqual(f.read(), b"sbar!\n") self.assertEqual(len(upload_dir.listdir()), 0) # # Chunk 1 multipart_body = b"""-----------------------------1311987731215707521443909311\r\nContent-Disposition: form-data; name="resumableChunkNumber"\r\n\r\n1\r\n-----------------------------1311987731215707521443909311\r\nContent-Disposition: form-data; name="resumableChunkSize"\r\n\r\n10\r\n-----------------------------1311987731215707521443909311\r\nContent-Disposition: form-data; name="resumableCurrentChunkSize"\r\n\r\n10\r\n-----------------------------1311987731215707521443909311\r\nContent-Disposition: form-data; name="resumableTotalSize"\r\n\r\n16\r\n-----------------------------1311987731215707521443909311\r\nContent-Disposition: form-data; name="resumableType"\r\n\r\ntext/plain\r\n-----------------------------1311987731215707521443909311\r\nContent-Disposition: form-data; name="resumableIdentifier"\r\n\r\n16-examplefiletxt\r\n-----------------------------1311987731215707521443909311\r\nContent-Disposition: form-data; name="resumableFilename"\r\n\r\nexamplefile.txt\r\n-----------------------------1311987731215707521443909311\r\nContent-Disposition: form-data; name="resumableRelativePath"\r\n\r\nexamplefile.txt\r\n-----------------------------1311987731215707521443909311\r\nContent-Disposition: form-data; name="resumableTotalChunks"\r\n\r\n2\r\n-----------------------------1311987731215707521443909311\r\nContent-Disposition: form-data; name="on_progress"\r\n\r\ncom.example.upload.on_progress\r\n-----------------------------1311987731215707521443909311\r\nContent-Disposition: form-data; name="session"\r\n\r\n8887465641628580\r\n-----------------------------1311987731215707521443909311\r\nContent-Disposition: form-data; name="file"; filename="blob"\r\nContent-Type: application/octet-stream\r\n\r\nhello Cros\r\n-----------------------------1311987731215707521443909311--\r\n""" d = renderResource( resource, b"/", method="POST", headers={ b"content-type": [ b"multipart/form-data; boundary=---------------------------1311987731215707521443909311" ], b"Content-Length": [b"1680"], }, body=multipart_body, ) res = self.successResultOf(d) self.assertEqual(res.code, 200) self.assertEqual(len(mock_session.method_calls), 4) # Starting the upload self.assertEqual(mock_session.method_calls[0][1][0], u"com.example.upload.on_progress") self.assertEqual(mock_session.method_calls[0][1][1]["status"], "started") self.assertEqual(mock_session.method_calls[0][1][1]["id"], "examplefile.txt") self.assertEqual(mock_session.method_calls[0][1][1]["chunk"], 2) # Progress, first chunk done self.assertEqual(mock_session.method_calls[1][1][0], u"com.example.upload.on_progress") self.assertEqual(mock_session.method_calls[1][1][1]["status"], "progress") self.assertEqual(mock_session.method_calls[1][1][1]["id"], "examplefile.txt") self.assertEqual(mock_session.method_calls[1][1][1]["chunk"], 2) # Progress, second chunk done self.assertEqual(mock_session.method_calls[2][1][0], u"com.example.upload.on_progress") self.assertEqual(mock_session.method_calls[2][1][1]["status"], "progress") self.assertEqual(mock_session.method_calls[2][1][1]["id"], "examplefile.txt") self.assertEqual(mock_session.method_calls[2][1][1]["chunk"], 1) # Upload complete self.assertEqual(mock_session.method_calls[3][1][0], u"com.example.upload.on_progress") self.assertEqual(mock_session.method_calls[3][1][1]["status"], "finished") self.assertEqual(mock_session.method_calls[3][1][1]["id"], "examplefile.txt") self.assertEqual(mock_session.method_calls[3][1][1]["chunk"], 1) # Nothing in the temp dir, one file in the upload self.assertEqual(len(temp_dir.listdir()), 0) self.assertEqual(len(upload_dir.listdir()), 1) with upload_dir.child("examplefile.txt").open("rb") as f: self.assertEqual(f.read(), b"hello Crossbar!\n")
def test_basic(self): """ Upload a basic file using the FileUploadResource, in just a single chunk. """ upload_dir = FilePath(self.mktemp()) upload_dir.makedirs() temp_dir = FilePath(self.mktemp()) temp_dir.makedirs() fields = { "file_name": "resumableFilename", "mime_type": "resumableType", "total_size": "resumableTotalSize", "chunk_number": "resumableChunkNumber", "chunk_size": "resumableChunkSize", "total_chunks": "resumableTotalChunks", "content": "file", "on_progress": "on_progress", "session": "session", } mock_session = Mock() resource = FileUploadResource(upload_dir.path, temp_dir.path, fields, mock_session) mp = Multipart() mp.add_part(b"resumableChunkNumber", b"1") mp.add_part(b"resumableChunkSize", b"1048576") mp.add_part(b"resumableCurrentChunkSize", b"16") mp.add_part(b"resumableTotalSize", b"16") mp.add_part(b"resumableType", b"text/plain") mp.add_part(b"resumableIdentifier", b"16-examplefiletxt") mp.add_part(b"resumableFilename", b"examplefile.txt") mp.add_part(b"resumableRelativePath", b"examplefile.txt") mp.add_part(b"resumableTotalChunks", b"1") mp.add_part(b"on_progress", b"com.example.upload.on_progress") mp.add_part(b"session", b"6891276359801283") mp.add_part(b"file", b"hello Crossbar!\n", content_type=b"application/octet-stream", filename=b"blob") body, headers = mp.render() d = renderResource(resource, b"/", method="POST", headers=headers, body=body) res = self.successResultOf(d) self.assertEqual(res.code, 200) self.assertEqual(len(mock_session.method_calls), 2) # Starting the upload self.assertEqual(mock_session.method_calls[0][1][0], u"com.example.upload.on_progress") self.assertEqual(mock_session.method_calls[0][1][1]["status"], "started") self.assertEqual(mock_session.method_calls[0][1][1]["id"], "examplefile.txt") # Upload complete self.assertEqual(mock_session.method_calls[1][1][0], u"com.example.upload.on_progress") self.assertEqual(mock_session.method_calls[1][1][1]["status"], "finished") self.assertEqual(mock_session.method_calls[1][1][1]["id"], "examplefile.txt") # Nothing in the temp dir, one file in the upload self.assertEqual(len(temp_dir.listdir()), 0) self.assertEqual(len(upload_dir.listdir()), 1) with upload_dir.child("examplefile.txt").open("rb") as f: self.assertEqual(f.read(), b"hello Crossbar!\n")
def test_multichunk_shuffle(self): """ Uploading files that are in multiple chunks and are uploaded in different order works. """ upload_dir = FilePath(self.mktemp()) upload_dir.makedirs() temp_dir = FilePath(self.mktemp()) temp_dir.makedirs() fields = { "file_name": "resumableFilename", "mime_type": "resumableType", "total_size": "resumableTotalSize", "chunk_number": "resumableChunkNumber", "chunk_size": "resumableChunkSize", "total_chunks": "resumableTotalChunks", "content": "file", "on_progress": "on_progress", "session": "session" } mock_session = Mock() resource = FileUploadResource(upload_dir.path, temp_dir.path, fields, mock_session) # # Chunk 2 multipart_body = b"""-----------------------------42560029919436807832069165364\r\nContent-Disposition: form-data; name="resumableChunkNumber"\r\n\r\n2\r\n-----------------------------42560029919436807832069165364\r\nContent-Disposition: form-data; name="resumableChunkSize"\r\n\r\n10\r\n-----------------------------42560029919436807832069165364\r\nContent-Disposition: form-data; name="resumableCurrentChunkSize"\r\n\r\n6\r\n-----------------------------42560029919436807832069165364\r\nContent-Disposition: form-data; name="resumableTotalSize"\r\n\r\n16\r\n-----------------------------42560029919436807832069165364\r\nContent-Disposition: form-data; name="resumableType"\r\n\r\ntext/plain\r\n-----------------------------42560029919436807832069165364\r\nContent-Disposition: form-data; name="resumableIdentifier"\r\n\r\n16-examplefiletxt\r\n-----------------------------42560029919436807832069165364\r\nContent-Disposition: form-data; name="resumableFilename"\r\n\r\nexamplefile.txt\r\n-----------------------------42560029919436807832069165364\r\nContent-Disposition: form-data; name="resumableRelativePath"\r\n\r\nexamplefile.txt\r\n-----------------------------42560029919436807832069165364\r\nContent-Disposition: form-data; name="resumableTotalChunks"\r\n\r\n2\r\n-----------------------------42560029919436807832069165364\r\nContent-Disposition: form-data; name="on_progress"\r\n\r\ncom.example.upload.on_progress\r\n-----------------------------42560029919436807832069165364\r\nContent-Disposition: form-data; name="session"\r\n\r\n8887465641628580\r\n-----------------------------42560029919436807832069165364\r\nContent-Disposition: form-data; name="file"; filename="blob"\r\nContent-Type: application/octet-stream\r\n\r\nsbar!\n\r\n-----------------------------42560029919436807832069165364--\r\n""" d = renderResource( resource, b"/", method="POST", headers={ b"content-type": [b"multipart/form-data; boundary=---------------------------42560029919436807832069165364"], b"Content-Length": [b"1688"] }, body=multipart_body ) res = self.successResultOf(d) self.assertEqual(res.code, 200) # One directory in the temp dir, nothing in the upload dir, temp dir # contains one chunk self.assertEqual(len(temp_dir.listdir()), 1) self.assertEqual(len(temp_dir.child("examplefile.txt").listdir()), 1) with temp_dir.child("examplefile.txt").child("chunk_2").open("rb") as f: self.assertEqual(f.read(), b"sbar!\n") self.assertEqual(len(upload_dir.listdir()), 0) # # Chunk 1 multipart_body = b"""-----------------------------1311987731215707521443909311\r\nContent-Disposition: form-data; name="resumableChunkNumber"\r\n\r\n1\r\n-----------------------------1311987731215707521443909311\r\nContent-Disposition: form-data; name="resumableChunkSize"\r\n\r\n10\r\n-----------------------------1311987731215707521443909311\r\nContent-Disposition: form-data; name="resumableCurrentChunkSize"\r\n\r\n10\r\n-----------------------------1311987731215707521443909311\r\nContent-Disposition: form-data; name="resumableTotalSize"\r\n\r\n16\r\n-----------------------------1311987731215707521443909311\r\nContent-Disposition: form-data; name="resumableType"\r\n\r\ntext/plain\r\n-----------------------------1311987731215707521443909311\r\nContent-Disposition: form-data; name="resumableIdentifier"\r\n\r\n16-examplefiletxt\r\n-----------------------------1311987731215707521443909311\r\nContent-Disposition: form-data; name="resumableFilename"\r\n\r\nexamplefile.txt\r\n-----------------------------1311987731215707521443909311\r\nContent-Disposition: form-data; name="resumableRelativePath"\r\n\r\nexamplefile.txt\r\n-----------------------------1311987731215707521443909311\r\nContent-Disposition: form-data; name="resumableTotalChunks"\r\n\r\n2\r\n-----------------------------1311987731215707521443909311\r\nContent-Disposition: form-data; name="on_progress"\r\n\r\ncom.example.upload.on_progress\r\n-----------------------------1311987731215707521443909311\r\nContent-Disposition: form-data; name="session"\r\n\r\n8887465641628580\r\n-----------------------------1311987731215707521443909311\r\nContent-Disposition: form-data; name="file"; filename="blob"\r\nContent-Type: application/octet-stream\r\n\r\nhello Cros\r\n-----------------------------1311987731215707521443909311--\r\n""" d = renderResource( resource, b"/", method="POST", headers={ b"content-type": [b"multipart/form-data; boundary=---------------------------1311987731215707521443909311"], b"Content-Length": [b"1680"] }, body=multipart_body ) res = self.successResultOf(d) self.assertEqual(res.code, 200) self.assertEqual(len(mock_session.method_calls), 4) # Starting the upload self.assertEqual(mock_session.method_calls[0][1][0], u"com.example.upload.on_progress") self.assertEqual(mock_session.method_calls[0][1][1]["status"], "started") self.assertEqual(mock_session.method_calls[0][1][1]["id"], "examplefile.txt") self.assertEqual(mock_session.method_calls[0][1][1]["chunk"], 2) # Progress, first chunk done self.assertEqual(mock_session.method_calls[1][1][0], u"com.example.upload.on_progress") self.assertEqual(mock_session.method_calls[1][1][1]["status"], "progress") self.assertEqual(mock_session.method_calls[1][1][1]["id"], "examplefile.txt") self.assertEqual(mock_session.method_calls[1][1][1]["chunk"], 2) # Progress, second chunk done self.assertEqual(mock_session.method_calls[2][1][0], u"com.example.upload.on_progress") self.assertEqual(mock_session.method_calls[2][1][1]["status"], "progress") self.assertEqual(mock_session.method_calls[2][1][1]["id"], "examplefile.txt") self.assertEqual(mock_session.method_calls[2][1][1]["chunk"], 1) # Upload complete self.assertEqual(mock_session.method_calls[3][1][0], u"com.example.upload.on_progress") self.assertEqual(mock_session.method_calls[3][1][1]["status"], "finished") self.assertEqual(mock_session.method_calls[3][1][1]["id"], "examplefile.txt") self.assertEqual(mock_session.method_calls[3][1][1]["chunk"], 1) # Nothing in the temp dir, one file in the upload self.assertEqual(len(temp_dir.listdir()), 0) self.assertEqual(len(upload_dir.listdir()), 1) with upload_dir.child("examplefile.txt").open("rb") as f: self.assertEqual(f.read(), b"hello Crossbar!\n")
def test_multichunk(self): """ Uploading files that are in multiple chunks works. """ upload_dir = FilePath(self.mktemp()) upload_dir.makedirs() temp_dir = FilePath(self.mktemp()) temp_dir.makedirs() fields = { "file_name": "resumableFilename", "mime_type": "resumableType", "total_size": "resumableTotalSize", "chunk_number": "resumableChunkNumber", "chunk_size": "resumableChunkSize", "total_chunks": "resumableTotalChunks", "content": "file", "on_progress": "on_progress", "session": "session" } mock_session = Mock() resource = FileUploadResource(upload_dir.path, temp_dir.path, fields, mock_session) # # Chunk 1 mp = Multipart() mp.add_part(b"resumableChunkNumber", b"1") mp.add_part(b"resumableChunkSize", b"10") mp.add_part(b"resumableCurrentChunkSize", b"10") mp.add_part(b"resumableTotalSize", b"16") mp.add_part(b"resumableType", b"text/plain") mp.add_part(b"resumableIdentifier", b"16-examplefiletxt") mp.add_part(b"resumableFilename", b"examplefile.txt") mp.add_part(b"resumableRelativePath", b"examplefile.txt") mp.add_part(b"resumableTotalChunks", b"2") mp.add_part(b"on_progress", b"com.example.upload.on_progress") mp.add_part(b"session", b"6891276359801283") mp.add_part(b"file", b"hello Cros", content_type=b"application/octet-stream", filename=b"blob") body, headers = mp.render() d = renderResource(resource, b"/", method="POST", headers=headers, body=body) res = self.successResultOf(d) res.setResponseCode.assert_called_once_with(200) # One directory in the temp dir, nothing in the upload dir, temp dir # contains one chunk self.assertEqual(len(temp_dir.listdir()), 1) self.assertEqual(len(temp_dir.child("examplefile.txt").listdir()), 1) with temp_dir.child("examplefile.txt").child("chunk_1").open( "rb") as f: self.assertEqual(f.read(), b"hello Cros") self.assertEqual(len(upload_dir.listdir()), 0) # # Chunk 2 mp = Multipart() mp.add_part(b"resumableChunkNumber", b"2") mp.add_part(b"resumableChunkSize", b"10") mp.add_part(b"resumableCurrentChunkSize", b"6") mp.add_part(b"resumableTotalSize", b"16") mp.add_part(b"resumableType", b"text/plain") mp.add_part(b"resumableIdentifier", b"16-examplefiletxt") mp.add_part(b"resumableFilename", b"examplefile.txt") mp.add_part(b"resumableRelativePath", b"examplefile.txt") mp.add_part(b"resumableTotalChunks", b"2") mp.add_part(b"on_progress", b"com.example.upload.on_progress") mp.add_part(b"session", b"6891276359801283") mp.add_part(b"file", b"sbar!\n", content_type=b"application/octet-stream", filename=b"blob") body, headers = mp.render() d = renderResource(resource, b"/", method="POST", headers=headers, body=body) res = self.successResultOf(d) res.setResponseCode.assert_called_once_with(200) self.assertEqual(len(mock_session.method_calls), 4) # Starting the upload self.assertEqual(mock_session.method_calls[0][1][0], u"com.example.upload.on_progress") self.assertEqual(mock_session.method_calls[0][1][1]["status"], "started") self.assertEqual(mock_session.method_calls[0][1][1]["id"], "examplefile.txt") self.assertEqual(mock_session.method_calls[0][1][1]["chunk"], 1) # Progress, first chunk done self.assertEqual(mock_session.method_calls[1][1][0], u"com.example.upload.on_progress") self.assertEqual(mock_session.method_calls[1][1][1]["status"], "progress") self.assertEqual(mock_session.method_calls[1][1][1]["id"], "examplefile.txt") self.assertEqual(mock_session.method_calls[1][1][1]["chunk"], 1) # Progress, second chunk done self.assertEqual(mock_session.method_calls[2][1][0], u"com.example.upload.on_progress") self.assertEqual(mock_session.method_calls[2][1][1]["status"], "progress") self.assertEqual(mock_session.method_calls[2][1][1]["id"], "examplefile.txt") self.assertEqual(mock_session.method_calls[2][1][1]["chunk"], 2) # Upload complete self.assertEqual(mock_session.method_calls[3][1][0], u"com.example.upload.on_progress") self.assertEqual(mock_session.method_calls[3][1][1]["status"], "finished") self.assertEqual(mock_session.method_calls[3][1][1]["id"], "examplefile.txt") self.assertEqual(mock_session.method_calls[3][1][1]["chunk"], 2) # Nothing in the temp dir, one file in the upload self.assertEqual(len(temp_dir.listdir()), 0) self.assertEqual(len(upload_dir.listdir()), 1) with upload_dir.child("examplefile.txt").open("rb") as f: self.assertEqual(f.read(), b"hello Crossbar!\n")
class DirDBM: """ A directory with a DBM interface. This class presents a hash-like interface to a directory of small, flat files. It can only use strings as keys or values. """ def __init__(self, name): """ @type name: str @param name: Base path to use for the directory storage. """ self.dname = os.path.abspath(name) self._dnamePath = FilePath(name) if not self._dnamePath.isdir(): self._dnamePath.createDirectory() else: # Run recovery, in case we crashed. we delete all files ending # with ".new". Then we find all files who end with ".rpl". If a # corresponding file exists without ".rpl", we assume the write # failed and delete the ".rpl" file. If only a ".rpl" exist we # assume the program crashed right after deleting the old entry # but before renaming the replacement entry. # # NOTE: '.' is NOT in the base64 alphabet! for f in glob.glob(self._dnamePath.child("*.new").path): os.remove(f) replacements = glob.glob(self._dnamePath.child("*.rpl").path) for f in replacements: old = f[:-4] if os.path.exists(old): os.remove(f) else: os.rename(f, old) def _encode(self, k): """ Encode a key so it can be used as a filename. """ # NOTE: '_' is NOT in the base64 alphabet! return base64.encodestring(k).replace(b'\n', b'_').replace(b"/", b"-") def _decode(self, k): """ Decode a filename to get the key. """ return base64.decodestring(k.replace(b'_', b'\n').replace(b"-", b"/")) def _readFile(self, path): """ Read in the contents of a file. Override in subclasses to e.g. provide transparently encrypted dirdbm. """ with _open(path.path, "rb") as f: s = f.read() return s def _writeFile(self, path, data): """ Write data to a file. Override in subclasses to e.g. provide transparently encrypted dirdbm. """ with _open(path.path, "wb") as f: f.write(data) f.flush() def __len__(self): """ @return: The number of key/value pairs in this Shelf """ return len(self._dnamePath.listdir()) def __setitem__(self, k, v): """ C{dirdbm[k] = v} Create or modify a textfile in this directory @type k: bytes @param k: key to set @type v: bytes @param v: value to associate with C{k} """ if not type(k) == bytes: raise TypeError("DirDBM key must be bytes") if not type(v) == bytes: raise TypeError("DirDBM value must be bytes") k = self._encode(k) # We create a new file with extension .new, write the data to it, and # if the write succeeds delete the old file and rename the new one. old = self._dnamePath.child(k) if old.exists(): new = old.siblingExtension(".rpl") # Replacement entry else: new = old.siblingExtension(".new") # New entry try: self._writeFile(new, v) except: new.remove() raise else: if (old.exists()): old.remove() new.moveTo(old) def __getitem__(self, k): """ C{dirdbm[k]} Get the contents of a file in this directory as a string. @type k: bytes @param k: key to lookup @return: The value associated with C{k} @raise KeyError: Raised when there is no such key """ if not type(k) == bytes: raise TypeError("DirDBM key must be bytes") path = self._dnamePath.child(self._encode(k)) try: return self._readFile(path) except (EnvironmentError): raise KeyError(k) def __delitem__(self, k): """ C{del dirdbm[foo]} Delete a file in this directory. @type k: bytes @param k: key to delete @raise KeyError: Raised when there is no such key """ if not type(k) == bytes: raise TypeError("DirDBM key must be bytes") k = self._encode(k) try: self._dnamePath.child(k).remove() except (EnvironmentError): raise KeyError(self._decode(k)) def keys(self): """ @return: a L{list} of filenames (keys). """ return list(map(self._decode, self._dnamePath.asBytesMode().listdir())) def values(self): """ @return: a L{list} of file-contents (values). """ vals = [] keys = self.keys() for key in keys: vals.append(self[key]) return vals def items(self): """ @return: a L{list} of 2-tuples containing key/value pairs. """ items = [] keys = self.keys() for key in keys: items.append((key, self[key])) return items def has_key(self, key): """ @type key: bytes @param key: The key to test @return: A true value if this dirdbm has the specified key, a false value otherwise. """ if not type(key) == bytes: raise TypeError("DirDBM key must be bytes") key = self._encode(key) return self._dnamePath.child(key).isfile() def setdefault(self, key, value): """ @type key: bytes @param key: The key to lookup @param value: The value to associate with key if key is not already associated with a value. """ if key not in self: self[key] = value return value return self[key] def get(self, key, default=None): """ @type key: bytes @param key: The key to lookup @param default: The value to return if the given key does not exist @return: The value associated with C{key} or C{default} if not L{DirDBM.has_key(key)} """ if key in self: return self[key] else: return default def __contains__(self, key): """ @see: L{DirDBM.has_key} """ return self.has_key(key) def update(self, dict): """ Add all the key/value pairs in L{dict} to this dirdbm. Any conflicting keys will be overwritten with the values from L{dict}. @type dict: mapping @param dict: A mapping of key/value pairs to add to this dirdbm. """ for key, val in dict.items(): self[key] = val def copyTo(self, path): """ Copy the contents of this dirdbm to the dirdbm at C{path}. @type path: L{str} @param path: The path of the dirdbm to copy to. If a dirdbm exists at the destination path, it is cleared first. @rtype: C{DirDBM} @return: The dirdbm this dirdbm was copied to. """ path = FilePath(path) assert path != self._dnamePath d = self.__class__(path.path) d.clear() for k in self.keys(): d[k] = self[k] return d def clear(self): """ Delete all key/value pairs in this dirdbm. """ for k in self.keys(): del self[k] def close(self): """ Close this dbm: no-op, for dbm-style interface compliance. """ def getModificationTime(self, key): """ Returns modification time of an entry. @return: Last modification date (seconds since epoch) of entry C{key} @raise KeyError: Raised when there is no such key """ if not type(key) == bytes: raise TypeError("DirDBM key must be bytes") path = self._dnamePath.child(self._encode(key)) if path.isfile(): return path.getModificationTime() else: raise KeyError(key)
class DirDBM: """ A directory with a DBM interface. This class presents a hash-like interface to a directory of small, flat files. It can only use strings as keys or values. """ def __init__(self, name): """ @type name: str @param name: Base path to use for the directory storage. """ self.dname = os.path.abspath(name) self._dnamePath = FilePath(name) if not self._dnamePath.isdir(): self._dnamePath.createDirectory() else: # Run recovery, in case we crashed. we delete all files ending # with ".new". Then we find all files who end with ".rpl". If a # corresponding file exists without ".rpl", we assume the write # failed and delete the ".rpl" file. If only a ".rpl" exist we # assume the program crashed right after deleting the old entry # but before renaming the replacement entry. # # NOTE: '.' is NOT in the base64 alphabet! for f in glob.glob(self._dnamePath.child("*.new").path): os.remove(f) replacements = glob.glob(self._dnamePath.child("*.rpl").path) for f in replacements: old = f[:-4] if os.path.exists(old): os.remove(f) else: os.rename(f, old) def _encode(self, k): """ Encode a key so it can be used as a filename. """ # NOTE: '_' is NOT in the base64 alphabet! return base64.encodestring(k).replace(b'\n', b'_').replace(b"/", b"-") def _decode(self, k): """ Decode a filename to get the key. """ return base64.decodestring(k.replace(b'_', b'\n').replace(b"-", b"/")) def _readFile(self, path): """ Read in the contents of a file. Override in subclasses to e.g. provide transparently encrypted dirdbm. """ with _open(path.path, "rb") as f: s = f.read() return s def _writeFile(self, path, data): """ Write data to a file. Override in subclasses to e.g. provide transparently encrypted dirdbm. """ with _open(path.path, "wb") as f: f.write(data) f.flush() def __len__(self): """ @return: The number of key/value pairs in this Shelf """ return len(self._dnamePath.listdir()) def __setitem__(self, k, v): """ C{dirdbm[k] = v} Create or modify a textfile in this directory @type k: bytes @param k: key to set @type v: bytes @param v: value to associate with C{k} """ if not type(k) == bytes: raise TypeError("DirDBM key must be bytes") if not type(v) == bytes: raise TypeError("DirDBM value must be bytes") k = self._encode(k) # We create a new file with extension .new, write the data to it, and # if the write succeeds delete the old file and rename the new one. old = self._dnamePath.child(k) if old.exists(): new = old.siblingExtension(".rpl") # Replacement entry else: new = old.siblingExtension(".new") # New entry try: self._writeFile(new, v) except: new.remove() raise else: if (old.exists()): old.remove() new.moveTo(old) def __getitem__(self, k): """ C{dirdbm[k]} Get the contents of a file in this directory as a string. @type k: bytes @param k: key to lookup @return: The value associated with C{k} @raise KeyError: Raised when there is no such key """ if not type(k) == bytes: raise TypeError("DirDBM key must be bytes") path = self._dnamePath.child(self._encode(k)) try: return self._readFile(path) except (EnvironmentError): raise KeyError(k) def __delitem__(self, k): """ C{del dirdbm[foo]} Delete a file in this directory. @type k: bytes @param k: key to delete @raise KeyError: Raised when there is no such key """ if not type(k) == bytes: raise TypeError("DirDBM key must be bytes") k = self._encode(k) try: self._dnamePath.child(k).remove() except (EnvironmentError): raise KeyError(self._decode(k)) def keys(self): """ @return: a L{list} of filenames (keys). """ return list(map(self._decode, self._dnamePath.asBytesMode().listdir())) def values(self): """ @return: a L{list} of file-contents (values). """ vals = [] keys = self.keys() for key in keys: vals.append(self[key]) return vals def items(self): """ @return: a L{list} of 2-tuples containing key/value pairs. """ items = [] keys = self.keys() for key in keys: items.append((key, self[key])) return items def has_key(self, key): """ @type key: bytes @param key: The key to test @return: A true value if this dirdbm has the specified key, a false value otherwise. """ if not type(key) == bytes: raise TypeError("DirDBM key must be bytes") key = self._encode(key) return self._dnamePath.child(key).isfile() def setdefault(self, key, value): """ @type key: bytes @param key: The key to lookup @param value: The value to associate with key if key is not already associated with a value. """ if key not in self: self[key] = value return value return self[key] def get(self, key, default = None): """ @type key: bytes @param key: The key to lookup @param default: The value to return if the given key does not exist @return: The value associated with C{key} or C{default} if not L{DirDBM.has_key(key)} """ if key in self: return self[key] else: return default def __contains__(self, key): """ @see: L{DirDBM.has_key} """ return self.has_key(key) def update(self, dict): """ Add all the key/value pairs in L{dict} to this dirdbm. Any conflicting keys will be overwritten with the values from L{dict}. @type dict: mapping @param dict: A mapping of key/value pairs to add to this dirdbm. """ for key, val in dict.items(): self[key]=val def copyTo(self, path): """ Copy the contents of this dirdbm to the dirdbm at C{path}. @type path: L{str} @param path: The path of the dirdbm to copy to. If a dirdbm exists at the destination path, it is cleared first. @rtype: C{DirDBM} @return: The dirdbm this dirdbm was copied to. """ path = FilePath(path) assert path != self._dnamePath d = self.__class__(path.path) d.clear() for k in self.keys(): d[k] = self[k] return d def clear(self): """ Delete all key/value pairs in this dirdbm. """ for k in self.keys(): del self[k] def close(self): """ Close this dbm: no-op, for dbm-style interface compliance. """ def getModificationTime(self, key): """ Returns modification time of an entry. @return: Last modification date (seconds since epoch) of entry C{key} @raise KeyError: Raised when there is no such key """ if not type(key) == bytes: raise TypeError("DirDBM key must be bytes") path = self._dnamePath.child(self._encode(key)) if path.isfile(): return path.getModificationTime() else: raise KeyError(key)