def test_basic(self):
"""
Upload a basic file using the FileUploadResource, in just a single chunk.
"""
upload_dir = FilePath(self.mktemp())
upload_dir.makedirs()
temp_dir = FilePath(self.mktemp())
temp_dir.makedirs()
fields = {
"file_name": "resumableFilename",
"mime_type": "resumableType",
"total_size": "resumableTotalSize",
"chunk_number": "resumableChunkNumber",
"chunk_size": "resumableChunkSize",
"total_chunks": "resumableTotalChunks",
"content": "file",
"on_progress": "on_progress",
"session": "session"
}
mock_session = Mock()
resource = FileUploadResource(upload_dir.path, temp_dir.path, fields, mock_session)
multipart_body = b"""-----------------------------478904261175205671481632460\r\nContent-Disposition: form-data; name="resumableChunkNumber"\r\n\r\n1\r\n-----------------------------478904261175205671481632460\r\nContent-Disposition: form-data; name="resumableChunkSize"\r\n\r\n1048576\r\n-----------------------------478904261175205671481632460\r\nContent-Disposition: form-data; name="resumableCurrentChunkSize"\r\n\r\n16\r\n-----------------------------478904261175205671481632460\r\nContent-Disposition: form-data; name="resumableTotalSize"\r\n\r\n16\r\n-----------------------------478904261175205671481632460\r\nContent-Disposition: form-data; name="resumableType"\r\n\r\ntext/plain\r\n-----------------------------478904261175205671481632460\r\nContent-Disposition: form-data; name="resumableIdentifier"\r\n\r\n16-examplefiletxt\r\n-----------------------------478904261175205671481632460\r\nContent-Disposition: form-data; name="resumableFilename"\r\n\r\nexamplefile.txt\r\n-----------------------------478904261175205671481632460\r\nContent-Disposition: form-data; name="resumableRelativePath"\r\n\r\nexamplefile.txt\r\n-----------------------------478904261175205671481632460\r\nContent-Disposition: form-data; name="resumableTotalChunks"\r\n\r\n1\r\n-----------------------------478904261175205671481632460\r\nContent-Disposition: form-data; name="on_progress"\r\n\r\ncom.example.upload.on_progress\r\n-----------------------------478904261175205671481632460\r\nContent-Disposition: form-data; name="session"\r\n\r\n6891276359801283\r\n-----------------------------478904261175205671481632460\r\nContent-Disposition: form-data; name="file"; filename="blob"\r\nContent-Type: application/octet-stream\r\n\r\nhello Crossbar!\n\r\n-----------------------------478904261175205671481632460--\r\n"""
d = renderResource(
resource, b"/", method="POST",
headers={
b"content-type": [b"multipart/form-data; boundary=---------------------------478904261175205671481632460"],
b"Content-Length": [b"1678"]
},
body=multipart_body
)
res = self.successResultOf(d)
res.setResponseCode.assert_called_once_with(200)
self.assertEqual(len(mock_session.method_calls), 2)
# Starting the upload
self.assertEqual(mock_session.method_calls[0][1][0], u"com.example.upload.on_progress")
self.assertEqual(mock_session.method_calls[0][1][1]["status"], "started")
self.assertEqual(mock_session.method_calls[0][1][1]["id"], "examplefile.txt")
# Upload complete
self.assertEqual(mock_session.method_calls[1][1][0], u"com.example.upload.on_progress")
self.assertEqual(mock_session.method_calls[1][1][1]["status"], "finished")
self.assertEqual(mock_session.method_calls[1][1][1]["id"], "examplefile.txt")
# Nothing in the temp dir, one file in the upload
self.assertEqual(len(temp_dir.listdir()), 0)
self.assertEqual(len(upload_dir.listdir()), 1)
with upload_dir.child("examplefile.txt").open("rb") as f:
self.assertEqual(f.read(), b"hello Crossbar!\n")
class Fs(CrawlJob):
def __init__(self, cfg, db_pool, data):
super(Fs, self).__init__(cfg, db_pool, data)
self.cfg = cfg
self.data = data
self.protocol = 'filesystem'
self.creator = None
self.fs = None
self.connection_closed = False
def create(self, path=None):
self.fs = FilePath(path)
try:
return self.fs.getPermissions()
except:
return False
@defer.inlineCallbacks
def getDirectory(self, path='/'):
self.fs = yield FilePath(path)
if not self.fs.getPermissions():
defer.returnValue(False)
files = []
for f in self.fs.listdir():
if f == '/':
continue
fp = path+f
fs = FilePath(fp)
# dont follow symlinks
if fs.realpath().path != fp:
continue
perm = None
isdir = fs.isdir()
size = fs.getsize()
modified = datetime.utcfromtimestamp(fs.getModificationTime())
df = DiscoveredFile(
resource_id=self.data['resource_id'],
file_path=path,
file_name=f,
file_isdir=isdir,
file_size=size,
file_modified=modified,
file_perm=perm
)
print '[%s] LIST %s.' % (self.data['resource_name'], fp if not fp.endswith('.') else fp)
files.append(df)
defer.returnValue(files)
def _get_capinfos(self, username, ip_addr):
"""
Get capinfos of user's pcap. This should be run afte the account
has expired and after executing pcapsummarizer on the user's
directory.
:param username (str): account username
:param ip_addr (IPv4Address): IP address allocated for the account.
:return: String with capinfos content.
"""
user_dir = "{}_{}".format(username, str(ip_addr))
user_dir = os.path.join(self.path['pcaps'], user_dir)
user_dir_fp = FilePath(user_dir)
capinfos = None
for f in user_dir_fp.listdir():
filename = os.path.join(user_dir, f)
fp = FilePath(filename)
if re.match(".*\.capinfos$", fp.basename()):
if capinfos is None:
capinfos = fp.getContent()
else:
capinfos = "{}\n\n{}".format(capinfos, fp.getContent())
return capinfos
def _setupTheme(self):
"""
Download and setup the theme.
"""
logging.info("\tBuilding theme...")
try:
tarball = urlopen(self.theme_url).read()
except HTTPError:
print("Error downloading theme from %s" % self.theme_url)
sys.exit(1)
workPath = FilePath(mkdtemp())
sourceFile = workPath.child("theme.tar.gz")
# change dir to fix issue with sphinx & themes
os.chdir(self.docPath.path)
o = open(sourceFile.path , "w")
o.write(tarball)
o.close()
tar = opentar(sourceFile.path, mode='r:*')
tar.extractall(workPath.path)
theme = None
for d in workPath.listdir():
theme = workPath.child(d)
if theme.isdir():
theme = theme.child("source").child("themes")
dest = self.docPath.child("themes")
theme.moveTo(dest)
break
def __init__(self, static_path, templates_path):
resource.Resource.__init__(self)
self.log = logging.getLogger('webui.core')
for action in 'online', 'offline', 'scan':
self.putChild(action, WebUIAction(self, 'nm_{}'.format(action)))
if not isinstance(static_path, FilePath):
static_path = FilePath(static_path)
for p in static_path.listdir():
self.putChild(p, File(static_path.child(p).path))
if not isinstance(templates_path, FilePath):
templates_path = FilePath(templates_path)
self.templates = jinja2.Environment(
loader=jinja2.FileSystemLoader(templates_path.path) )
self.templates.filters['json'] = self.jinja2_json
self.templates.filters['unless_false'] = self.jinja2_unless_false
self.events = ClientEvents(self.handle_command)
self.putChild('events', SockJSResource(self.events))
def list_measurements(compute_size=False, order=None):
measurements = []
measurement_path = FilePath(config.measurements_directory)
if not measurement_path.exists():
return measurements
for measurement_id in measurement_path.listdir():
try:
measurements.append(get_measurement(measurement_id, compute_size))
except:
log.err("Failed to get metadata for measurement {0}".format(measurement_id))
if order is None:
return measurements
if order.lower() in ['asc', 'desc']:
reverse = {'asc': False, 'desc': True}[order.lower()]
measurements.sort(key=operator.itemgetter('test_start_time'),
reverse=reverse)
return measurements
else:
raise ValueError("order must be either 'asc' 'desc' or None")
def __init__(self, static_path, templates_path):
resource.Resource.__init__(self)
self.log = logging.getLogger('webui.core')
for action in 'online', 'offline', 'scan':
self.putChild(action, WebUIAction(self, 'nm_{}'.format(action)))
if not isinstance(static_path, FilePath):
static_path = FilePath(static_path)
for p in static_path.listdir():
self.putChild(p, File(static_path.child(p).path))
if not isinstance(templates_path, FilePath):
templates_path = FilePath(templates_path)
self.templates = jinja2.Environment(
loader=jinja2.FileSystemLoader(templates_path.path))
self.templates.filters['json'] = self.jinja2_json
self.templates.filters['unless_false'] = self.jinja2_unless_false
self.events = ClientEvents(self.handle_command)
self.putChild('events', SockJSResource(self.events))
def __init__(self, static_path, templates_path):
resource.Resource.__init__(self)
self.log = logging.getLogger("webui.core")
for action in b"online", b"offline", b"scan":
self.putChild(action, WebUIAction(self, "nm_{}".format(action)))
if not isinstance(static_path, FilePath):
static_path = FilePath(static_path)
for p in static_path.listdir():
self.putChild(bytes(p), File(static_path.child(p).path))
if not isinstance(templates_path, FilePath):
templates_path = FilePath(templates_path)
self.templates = jinja2.Environment(
loader=jinja2.FileSystemLoader(templates_path.path)
)
self.templates.filters["json"] = self.jinja2_json
self.templates.filters["unless_false"] = self.jinja2_unless_false
self.events = ClientEvents(self.handle_command)
self.putChild("events", SockJSResource(self.events))
def list_measurements(compute_size=False, order=None):
measurements = []
measurement_path = FilePath(config.measurements_directory)
if not measurement_path.exists():
return measurements
for measurement_id in measurement_path.listdir():
try:
measurements.append(get_measurement(measurement_id, compute_size))
except Exception as exc:
log.err("Failed to get metadata for measurement {0}".format(measurement_id))
log.exception(exc)
if order is None:
return measurements
if order.lower() in ['asc', 'desc']:
reverse = {'asc': False, 'desc': True}[order.lower()]
measurements.sort(key=operator.itemgetter('test_start_time'),
reverse=reverse)
return measurements
else:
raise ValueError("order must be either 'asc' 'desc' or None")
class FileUploadResource(Resource):
"""
Twisted Web resource that handles file uploads over `HTTP/POST` requests.
"""
log = make_logger()
def __init__(self,
upload_directory,
temp_directory,
form_fields,
upload_session,
options=None):
"""
:param upload_directory: The target directory where uploaded files will be stored.
:type upload_directory: str
:param temp_directory: A temporary directory where chunks of a file being uploaded are stored.
:type temp_directory: str
:param form_fields: Names of HTML form fields used for uploading.
:type form_fields: dict
:param upload_session: An instance of `ApplicationSession` used for publishing progress events.
:type upload_session: obj
:param options: Options for file upload.
:type options: dict or None
"""
Resource.__init__(self)
self._dir = FilePath(upload_directory)
self._tempDir = FilePath(temp_directory)
self._form_fields = form_fields
self._fileupload_session = upload_session
self._options = options or {}
self._max_file_size = self._options.get('max_file_size',
10 * 1024 * 1024)
self._fileTypes = self._options.get('file_types', None)
self._file_permissions = self._options.get('file_permissions', None)
# track uploaded files / chunks
self._uploads = {}
# scan the temp dir for uploaded chunks and fill the _uploads dict with it
# so existing uploads can be resumed
for fileTempDir in self._tempDir.listdir():
ft = self._tempDir.child(fileTempDir)
if ft.isdir():
self._uploads[fileTempDir] = {
'chunk_list': {},
'origin': 'startup'
}
for chunk in ft.listdir():
if chunk[:6] == 'chunk_':
self._uploads[fileTempDir]['chunk_list'][int(
chunk[6:])] = True
self.log.debug("Scanned pending uploads: {uploads}",
uploads=self._uploads)
def render_POST(self, request):
headers = {
x.decode('iso-8859-1'): y.decode('iso-8859-1')
for x, y in request.getAllHeaders().items()
}
origin = headers['host']
content = cgi.FieldStorage(fp=request.content,
headers=headers,
environ={"REQUEST_METHOD": "POST"})
f = self._form_fields
filename = content[f['file_name']].value
totalSize = int(content[f['total_size']].value)
totalChunks = int(content[f['total_chunks']].value)
chunkSize = int(content[f['chunk_size']].value)
chunkNumber = int(content[f['chunk_number']].value)
fileContent = content[f['content']].value
fileId = filename
# # prepare user specific upload areas
# # NOT YET IMPLEMENTED
# #
# if 'auth_id' in f and f['auth_id'] in content:
# auth_id = content[f['auth_id']].value
# mydir = os.path.join(self._dir, auth_id)
# my_temp_dir = os.path.join(self._tempDir, auth_id)
#
# # check if auth_id is a valid directory_name
# #
# if auth_id != auth_id.encode('ascii', 'ignore'):
# msg = "The requestor auth_id must be an ascii string."
# if self._debug:
# log.msg(msg)
# # 415 Unsupported Media Type
# request.setResponseCode(415, msg)
# return msg
# else:
# auth_id = 'anonymous'
# create user specific folder
# mydir = self._dir
# my_temp_dir = self._tempDir
# if not os.path.exists(mydir):
# os.makedirs(mydir)
# if not os.path.exists(my_temp_dir):
# os.makedirs(my_temp_dir)
if 'on_progress' in f and f[
'on_progress'] in content and self._fileupload_session != {}:
topic = content[f['on_progress']].value
if 'session' in f and f['session'] in content:
session = int(content[f['session']].value)
publish_options = PublishOptions(eligible=[session])
else:
publish_options = None
def fileupload_publish(payload):
self._fileupload_session.publish(topic,
payload,
options=publish_options)
else:
def fileupload_publish(payload):
pass
# Register upload right at the start to avoid overlapping upload conflicts
if fileId not in self._uploads:
self._uploads[fileId] = {'chunk_list': {}, 'origin': origin}
chunk_is_first = True
else:
chunk_is_first = False
self.log.debug(
'Started upload of file: file_name={file_name}, total_size={total_size}, total_chunks={total_chunks}, chunk_size={chunk_size}, chunk_number={chunk_number}',
file_name=fileId,
total_size=totalSize,
total_chunks=totalChunks,
chunk_size=chunkSize,
chunk_number=chunkNumber)
# check file size
#
if totalSize > self._max_file_size:
msg = "Size {} of file to be uploaded exceeds maximum {}".format(
totalSize, self._max_file_size)
self.log.debug(msg)
# 413 Request Entity Too Large
request.setResponseCode(413, msg.encode('utf8'))
return msg.encode('utf8')
# check file extensions
#
extension = os.path.splitext(filename)[1]
if self._fileTypes and extension not in self._fileTypes:
msg = "Type '{}' of file to be uploaded is in allowed types {}".format(
extension, self._fileTypes)
self.log.debug(msg)
# 415 Unsupported Media Type
request.setResponseCode(415, msg.encode('utf8'))
return msg.encode('utf8')
# check if another session is uploading this file already
# If the chunks are read at startup of crossbar any client may resume the pending upload !
#
try:
upl = self._uploads[fileId]
if upl['origin'] != origin and upl['origin'] != 'startup':
msg = "File being uploaded is already uploaded in a different session"
self.log.debug(msg)
# 409 Conflict
request.setResponseCode(409, msg.encode('utf8'))
return msg.encode('utf8')
except Exception:
pass
# TODO: check mime type
fileTempDir = self._tempDir.child(fileId)
chunkName = fileTempDir.child('chunk_' + str(chunkNumber))
_chunkName = fileTempDir.child('#kfhfkzuru578e38viokbjhfvz4w__' +
'chunk_' + str(chunkNumber))
if chunk_is_first:
# first chunk of file
# clean the temp dir once per file upload
self._remove_stale_uploads()
# publish file upload start
#
fileupload_publish({
"id": fileId,
"chunk": chunkNumber,
"name": filename,
"total": totalSize,
"remaining": totalSize,
"status": "started",
"progress": 0.
})
if totalChunks == 1:
# only one chunk overall -> write file directly
finalFileName = self._dir.child(fileId)
_finalFileName = self._dir.child(
'#kfhfkzuru578e38viokbjhfvz4w__' + fileId)
with open(_finalFileName.path, 'wb') as finalFile:
finalFile.write(fileContent)
_finalFileName.moveTo(finalFileName)
self._uploads[fileId]['chunk_list'][chunkNumber] = True
if self._file_permissions:
perm = int(self._file_permissions, 8)
try:
finalFileName.chmod(perm)
except Exception as e:
finalFileName.remove()
msg = "Could not change file permissions of uploaded file"
self.log.debug(msg)
self.log.debug(e)
request.setResponseCode(500, msg.encode('utf8'))
return msg.encode('utf8')
else:
self.log.debug(
"Changed permissions on {file_name} to {permissions}",
file_name=finalFileName,
permissions=self._file_permissions)
self._uploads.pop(fileId, None)
# publish file upload progress to file_progress_URI
fileupload_publish({
"id": fileId,
"chunk": chunkNumber,
"name": filename,
"total": totalSize,
"remaining": 0,
"status": "finished",
"progress": 1.
})
else:
# first of more chunks
fileTempDir.makedirs()
with open(_chunkName.path, 'wb') as chunk:
chunk.write(fileContent)
_chunkName.moveTo(chunkName)
self._uploads[fileId]['chunk_list'][chunkNumber] = True
# publish file upload progress
#
fileupload_publish({
"id":
fileId,
"chunk":
chunkNumber,
"name":
filename,
"total":
totalSize,
"remaining":
totalSize - chunkSize,
"status":
"progress",
"progress":
round(float(chunkSize) / float(totalSize), 3)
})
else:
# intermediate chunk
with open(_chunkName.path, 'wb') as chunk:
chunk.write(fileContent)
_chunkName.moveTo(chunkName)
self._uploads[fileId]['chunk_list'][chunkNumber] = True
received = sum(
fileTempDir.child(f).getsize() for f in fileTempDir.listdir())
fileupload_publish({
"id":
fileId,
"chunk":
chunkNumber,
"name":
filename,
"total":
totalSize,
"remaining":
totalSize - received,
"status":
"progress",
"progress":
round(float(received) / float(totalSize), 3)
})
# every chunk has to check if it is the last chunk written, except in a single chunk scenario
if totalChunks > 1 and len(
self._uploads[fileId]['chunk_list']) == totalChunks:
# last chunk
self.log.debug('Finished file upload after chunk {chunk_number}',
chunk_number=chunkNumber)
# Merge all files into one file and remove the temp files
# TODO: How to avoid the extra file IO ?
finalFileName = self._dir.child(fileId)
_finalFileName = self._dir.child(
'#kfhf3kz412uru578e38viokbjhfvz4w__' + fileId)
with open(_finalFileName.path, 'wb') as finalFile:
for tfileName in fileTempDir.listdir():
with open(fileTempDir.child(tfileName).path,
'rb') as tfile:
finalFile.write(tfile.read())
_finalFileName.moveTo(finalFileName)
if self._file_permissions:
perm = int(self._file_permissions, 8)
try:
finalFileName.chmod(perm)
except Exception as e:
msg = "file upload resource - could not change file permissions of uploaded file"
self.log.debug(msg)
self.log.debug(e)
request.setResponseCode(500, msg.encode('utf8'))
return msg.encode('utf8')
else:
self.log.debug(
"Changed permissions on {file_name} to {permissions}",
file_name=finalFileName,
permissions=self._file_permissions)
# publish file upload progress to file_progress_URI
fileupload_publish({
"id": fileId,
"chunk": chunkNumber,
"name": filename,
"total": totalSize,
"remaining": 0,
"status": "finished",
"progress": 1.
})
# remove the file temp folder
self._remove_temp_dir(fileTempDir)
self._uploads.pop(fileId, None)
request.setResponseCode(200)
return b''
def _remove_temp_dir(self, fileTempDir):
fileTempDir.remove()
def _remove_stale_uploads(self):
"""
This only works if there is a temp folder exclusive for crossbar file uploads
if the system temp folder is used then crossbar creates a "crossbar-uploads" there and
uses that as the temp folder for uploads
If you don't clean up regularly an attacker could fill up the OS file system
"""
for _dir in self._tempDir.listdir():
fileTempDir = self._tempDir.child(_dir)
if fileTempDir.isdir() and _dir not in self._uploads:
self._remove_temp_dir(fileTempDir)
def render_GET(self, request):
"""
This method can be used to check whether a chunk has been uploaded already.
It returns with HTTP status code `200` if yes and `404` if not.
The request needs to contain the file identifier and the chunk number to check for.
"""
for param in ['file_name', 'chunk_number']:
if not self._form_fields[param].encode(
'iso-8859-1') in request.args:
msg = "file upload resource - missing request query parameter '{}', configured from '{}'".format(
self._form_fields[param], param)
self.log.debug(msg)
# 400 Bad Request
request.setResponseCode(400, msg.encode('utf8'))
return msg.encode('utf8')
file_name = request.args[self._form_fields['file_name'].encode(
'iso-8859-1')][0].decode('utf8')
chunk_number = int(
request.args[self._form_fields['chunk_number'].encode(
'iso-8859-1')][0].decode('utf8'))
# a complete upload will be repeated an incomplete upload will be resumed
if file_name in self._uploads and chunk_number in self._uploads[
file_name]['chunk_list']:
self.log.debug(
"Skipping chunk upload {file_name} of chunk {chunk_number}",
file_name=file_name,
chunk_number=chunk_number)
msg = b"chunk of file already uploaded"
request.setResponseCode(200, msg)
return msg
else:
msg = b"chunk of file not yet uploaded"
request.setResponseCode(404, msg)
return msg
class FileUploadResource(Resource):
"""
Twisted Web resource that handles file uploads over `HTTP/POST` requests.
"""
log = make_logger()
def __init__(self,
upload_directory,
temp_directory,
form_fields,
upload_session,
options=None):
"""
:param upload_directory: The target directory where uploaded files will be stored.
:type upload_directory: str
:param temp_directory: A temporary directory where chunks of a file being uploaded are stored.
:type temp_directory: str
:param form_fields: Names of HTML form fields used for uploading.
:type form_fields: dict
:param upload_session: An instance of `ApplicationSession` used for publishing progress events.
:type upload_session: obj
:param options: Options for file upload.
:type options: dict or None
"""
Resource.__init__(self)
self._dir = FilePath(upload_directory)
self._tempDir = FilePath(temp_directory)
self._form_fields = form_fields
self._fileupload_session = upload_session
self._options = options or {}
self._max_file_size = self._options.get('max_file_size', 10 * 1024 * 1024)
self._fileTypes = self._options.get('file_types', None)
self._file_permissions = self._options.get('file_permissions', None)
# track uploaded files / chunks
self._uploads = {}
# scan the temp dir for uploaded chunks and fill the _uploads dict with it
# so existing uploads can be resumed
for fileTempDir in self._tempDir.listdir():
ft = self._tempDir.child(fileTempDir)
if ft.isdir():
self._uploads[fileTempDir] = {'chunk_list': {}, 'origin': 'startup'}
for chunk in ft.listdir():
if chunk[:6] == 'chunk_':
self._uploads[fileTempDir]['chunk_list'][int(chunk[6:])] = True
self.log.debug("Scanned pending uploads: {uploads}", uploads=self._uploads)
def render_POST(self, request):
headers = {x.decode('iso-8859-1'): y.decode('iso-8859-1')
for x, y in request.getAllHeaders().items()}
origin = headers['host']
content = cgi.FieldStorage(
fp=request.content,
headers=headers,
environ={"REQUEST_METHOD": "POST"})
f = self._form_fields
filename = content[f['file_name']].value
totalSize = int(content[f['total_size']].value)
totalChunks = int(content[f['total_chunks']].value)
chunkSize = int(content[f['chunk_size']].value)
chunkNumber = int(content[f['chunk_number']].value)
fileContent = content[f['content']].value
fileId = filename
# # prepare user specific upload areas
# # NOT YET IMPLEMENTED
# #
# if 'auth_id' in f and f['auth_id'] in content:
# auth_id = content[f['auth_id']].value
# mydir = os.path.join(self._dir, auth_id)
# my_temp_dir = os.path.join(self._tempDir, auth_id)
#
# # check if auth_id is a valid directory_name
# #
# if auth_id != auth_id.encode('ascii', 'ignore'):
# msg = "The requestor auth_id must be an ascii string."
# if self._debug:
# log.msg(msg)
# # 415 Unsupported Media Type
# request.setResponseCode(415, msg)
# return msg
# else:
# auth_id = 'anonymous'
# create user specific folder
# mydir = self._dir
# my_temp_dir = self._tempDir
# if not os.path.exists(mydir):
# os.makedirs(mydir)
# if not os.path.exists(my_temp_dir):
# os.makedirs(my_temp_dir)
if 'on_progress' in f and f['on_progress'] in content and self._fileupload_session != {}:
topic = content[f['on_progress']].value
if 'session' in f and f['session'] in content:
session = int(content[f['session']].value)
publish_options = PublishOptions(eligible=[session])
else:
publish_options = None
def fileupload_publish(payload):
self._fileupload_session.publish(topic, payload, options=publish_options)
else:
def fileupload_publish(payload):
pass
# Register upload right at the start to avoid overlapping upload conflicts
if fileId not in self._uploads:
self._uploads[fileId] = {'chunk_list': {}, 'origin': origin}
chunk_is_first = True
else:
chunk_is_first = False
self.log.debug('Started upload of file: file_name={file_name}, total_size={total_size}, total_chunks={total_chunks}, chunk_size={chunk_size}, chunk_number={chunk_number}',
file_name=fileId, total_size=totalSize, total_chunks=totalChunks, chunk_size=chunkSize, chunk_number=chunkNumber)
# check file size
#
if totalSize > self._max_file_size:
msg = "Size {} of file to be uploaded exceeds maximum {}".format(totalSize, self._max_file_size)
self.log.debug(msg)
# 413 Request Entity Too Large
request.setResponseCode(413, msg.encode('utf8'))
return msg.encode('utf8')
# check file extensions
#
extension = os.path.splitext(filename)[1]
if self._fileTypes and extension not in self._fileTypes:
msg = "Type '{}' of file to be uploaded is in allowed types {}".format(extension, self._fileTypes)
self.log.debug(msg)
# 415 Unsupported Media Type
request.setResponseCode(415, msg.encode('utf8'))
return msg.encode('utf8')
# check if another session is uploading this file already
# If the chunks are read at startup of crossbar any client may resume the pending upload !
#
try:
upl = self._uploads[fileId]
if upl['origin'] != origin and upl['origin'] != 'startup':
msg = "File being uploaded is already uploaded in a different session"
self.log.debug(msg)
# 409 Conflict
request.setResponseCode(409, msg.encode('utf8'))
return msg.encode('utf8')
except Exception:
pass
# TODO: check mime type
fileTempDir = self._tempDir.child(fileId)
chunkName = fileTempDir.child('chunk_' + str(chunkNumber))
_chunkName = fileTempDir.child('#kfhfkzuru578e38viokbjhfvz4w__' + 'chunk_' + str(chunkNumber))
if chunk_is_first:
# first chunk of file
# clean the temp dir once per file upload
self._remove_stale_uploads()
# publish file upload start
#
fileupload_publish({
"id": fileId,
"chunk": chunkNumber,
"name": filename,
"total": totalSize,
"remaining": totalSize,
"status": "started",
"progress": 0.
})
if totalChunks == 1:
# only one chunk overall -> write file directly
finalFileName = self._dir.child(fileId)
_finalFileName = self._dir.child('#kfhfkzuru578e38viokbjhfvz4w__' + fileId)
with open(_finalFileName.path, 'wb') as finalFile:
finalFile.write(fileContent)
_finalFileName.moveTo(finalFileName)
self._uploads[fileId]['chunk_list'][chunkNumber] = True
if self._file_permissions:
perm = int(self._file_permissions, 8)
try:
finalFileName.chmod(perm)
except Exception as e:
finalFileName.remove()
msg = "Could not change file permissions of uploaded file"
self.log.debug(msg)
self.log.debug(e)
request.setResponseCode(500, msg.encode('utf8'))
return msg.encode('utf8')
else:
self.log.debug("Changed permissions on {file_name} to {permissions}", file_name=finalFileName, permissions=self._file_permissions)
self._uploads.pop(fileId, None)
# publish file upload progress to file_progress_URI
fileupload_publish({
"id": fileId,
"chunk": chunkNumber,
"name": filename,
"total": totalSize,
"remaining": 0,
"status": "finished",
"progress": 1.
})
else:
# first of more chunks
fileTempDir.makedirs()
with open(_chunkName.path, 'wb') as chunk:
chunk.write(fileContent)
_chunkName.moveTo(chunkName)
self._uploads[fileId]['chunk_list'][chunkNumber] = True
# publish file upload progress
#
fileupload_publish({
"id": fileId,
"chunk": chunkNumber,
"name": filename,
"total": totalSize,
"remaining": totalSize - chunkSize,
"status": "progress",
"progress": round(float(chunkSize) / float(totalSize), 3)
})
else:
# intermediate chunk
with open(_chunkName.path, 'wb') as chunk:
chunk.write(fileContent)
_chunkName.moveTo(chunkName)
self._uploads[fileId]['chunk_list'][chunkNumber] = True
received = sum(fileTempDir.child(f).getsize() for f in fileTempDir.listdir())
fileupload_publish({
"id": fileId,
"chunk": chunkNumber,
"name": filename,
"total": totalSize,
"remaining": totalSize - received,
"status": "progress",
"progress": round(float(received) / float(totalSize), 3)
})
# every chunk has to check if it is the last chunk written, except in a single chunk scenario
if totalChunks > 1 and len(self._uploads[fileId]['chunk_list']) == totalChunks:
# last chunk
self.log.debug('Finished file upload after chunk {chunk_number}', chunk_number=chunkNumber)
# Merge all files into one file and remove the temp files
# TODO: How to avoid the extra file IO ?
finalFileName = self._dir.child(fileId)
_finalFileName = self._dir.child('#kfhf3kz412uru578e38viokbjhfvz4w__' + fileId)
with open(_finalFileName.path, 'wb') as finalFile:
for tfileName in fileTempDir.listdir():
with open(fileTempDir.child(tfileName).path, 'rb') as tfile:
finalFile.write(tfile.read())
_finalFileName.moveTo(finalFileName)
if self._file_permissions:
perm = int(self._file_permissions, 8)
try:
finalFileName.chmod(perm)
except Exception as e:
msg = "file upload resource - could not change file permissions of uploaded file"
self.log.debug(msg)
self.log.debug(e)
request.setResponseCode(500, msg.encode('utf8'))
return msg.encode('utf8')
else:
self.log.debug("Changed permissions on {file_name} to {permissions}", file_name=finalFileName, permissions=self._file_permissions)
# publish file upload progress to file_progress_URI
fileupload_publish({
"id": fileId,
"chunk": chunkNumber,
"name": filename,
"total": totalSize,
"remaining": 0,
"status": "finished",
"progress": 1.
})
# remove the file temp folder
self._remove_temp_dir(fileTempDir)
self._uploads.pop(fileId, None)
request.setResponseCode(200)
return b''
def _remove_temp_dir(self, fileTempDir):
fileTempDir.remove()
def _remove_stale_uploads(self):
"""
This only works if there is a temp folder exclusive for crossbar file uploads
if the system temp folder is used then crossbar creates a "crossbar-uploads" there and
uses that as the temp folder for uploads
If you don't clean up regularly an attacker could fill up the OS file system
"""
for _dir in self._tempDir.listdir():
fileTempDir = self._tempDir.child(_dir)
if fileTempDir.isdir() and _dir not in self._uploads:
self._remove_temp_dir(fileTempDir)
def render_GET(self, request):
"""
This method can be used to check whether a chunk has been uploaded already.
It returns with HTTP status code `200` if yes and `404` if not.
The request needs to contain the file identifier and the chunk number to check for.
"""
for param in ['file_name', 'chunk_number']:
if not self._form_fields[param].encode('iso-8859-1') in request.args:
msg = "file upload resource - missing request query parameter '{}', configured from '{}'".format(self._form_fields[param], param)
self.log.debug(msg)
# 400 Bad Request
request.setResponseCode(400, msg.encode('utf8'))
return msg.encode('utf8')
file_name = request.args[self._form_fields['file_name'].encode('iso-8859-1')][0].decode('utf8')
chunk_number = int(request.args[self._form_fields['chunk_number'].encode('iso-8859-1')][0].decode('utf8'))
# a complete upload will be repeated an incomplete upload will be resumed
if file_name in self._uploads and chunk_number in self._uploads[file_name]['chunk_list']:
self.log.debug("Skipping chunk upload {file_name} of chunk {chunk_number}", file_name=file_name, chunk_number=chunk_number)
msg = b"chunk of file already uploaded"
request.setResponseCode(200, msg)
return msg
else:
msg = b"chunk of file not yet uploaded"
request.setResponseCode(404, msg)
return msg
def test_basic(self):
"""
Upload a basic file using the FileUploadResource, in just a single chunk.
"""
upload_dir = FilePath(self.mktemp())
upload_dir.makedirs()
temp_dir = FilePath(self.mktemp())
temp_dir.makedirs()
fields = {
"file_name": "resumableFilename",
"mime_type": "resumableType",
"total_size": "resumableTotalSize",
"chunk_number": "resumableChunkNumber",
"chunk_size": "resumableChunkSize",
"total_chunks": "resumableTotalChunks",
"content": "file",
"on_progress": "on_progress",
"session": "session"
}
mock_session = Mock()
resource = FileUploadResource(upload_dir.path, temp_dir.path, fields, mock_session)
mp = Multipart()
mp.add_part(b"resumableChunkNumber", b"1")
mp.add_part(b"resumableChunkSize", b"1048576")
mp.add_part(b"resumableCurrentChunkSize", b"16")
mp.add_part(b"resumableTotalSize", b"16")
mp.add_part(b"resumableType", b"text/plain")
mp.add_part(b"resumableIdentifier", b"16-examplefiletxt")
mp.add_part(b"resumableFilename", b"examplefile.txt")
mp.add_part(b"resumableRelativePath", b"examplefile.txt")
mp.add_part(b"resumableTotalChunks", b"1")
mp.add_part(b"on_progress", b"com.example.upload.on_progress")
mp.add_part(b"session", b"6891276359801283")
mp.add_part(b"file", b"hello Crossbar!\n",
content_type=b"application/octet-stream",
filename=b"blob")
body, headers = mp.render()
d = renderResource(
resource, b"/", method="POST",
headers=headers,
body=body
)
res = self.successResultOf(d)
self.assertEqual(res.code, 200)
self.assertEqual(len(mock_session.method_calls), 2)
# Starting the upload
self.assertEqual(mock_session.method_calls[0][1][0], u"com.example.upload.on_progress")
self.assertEqual(mock_session.method_calls[0][1][1]["status"], "started")
self.assertEqual(mock_session.method_calls[0][1][1]["id"], "examplefile.txt")
# Upload complete
self.assertEqual(mock_session.method_calls[1][1][0], u"com.example.upload.on_progress")
self.assertEqual(mock_session.method_calls[1][1][1]["status"], "finished")
self.assertEqual(mock_session.method_calls[1][1][1]["id"], "examplefile.txt")
# Nothing in the temp dir, one file in the upload
self.assertEqual(len(temp_dir.listdir()), 0)
self.assertEqual(len(upload_dir.listdir()), 1)
with upload_dir.child("examplefile.txt").open("rb") as f:
self.assertEqual(f.read(), b"hello Crossbar!\n")
def test_resumed_upload(self):
"""
Uploading part of a file, simulating a Crossbar restart, and continuing
to upload works.
"""
upload_dir = FilePath(self.mktemp())
upload_dir.makedirs()
temp_dir = FilePath(self.mktemp())
temp_dir.makedirs()
fields = {
"file_name": "resumableFilename",
"mime_type": "resumableType",
"total_size": "resumableTotalSize",
"chunk_number": "resumableChunkNumber",
"chunk_size": "resumableChunkSize",
"total_chunks": "resumableTotalChunks",
"content": "file",
"on_progress": "on_progress",
"session": "session"
}
mock_session = Mock()
resource = FileUploadResource(upload_dir.path, temp_dir.path, fields, mock_session)
# Make some stuff in the temp dir that wasn't there when it started but
# is put there before a file upload
temp_dir.child("otherjunk").makedirs()
#
# Chunk 1
mp = Multipart()
mp.add_part(b"resumableChunkNumber", b"1")
mp.add_part(b"resumableChunkSize", b"10")
mp.add_part(b"resumableCurrentChunkSize", b"10")
mp.add_part(b"resumableTotalSize", b"16")
mp.add_part(b"resumableType", b"text/plain")
mp.add_part(b"resumableIdentifier", b"16-examplefiletxt")
mp.add_part(b"resumableFilename", b"examplefile.txt")
mp.add_part(b"resumableRelativePath", b"examplefile.txt")
mp.add_part(b"resumableTotalChunks", b"2")
mp.add_part(b"on_progress", b"com.example.upload.on_progress")
mp.add_part(b"session", b"6891276359801283")
mp.add_part(b"file", b"hello Cros",
content_type=b"application/octet-stream",
filename=b"blob")
body, headers = mp.render()
d = renderResource(
resource, b"/", method="POST",
headers=headers,
body=body
)
res = self.successResultOf(d)
self.assertEqual(res.code, 200)
# One directory in the temp dir, nothing in the upload dir, temp dir
# contains one chunk
self.assertEqual(len(temp_dir.listdir()), 1)
self.assertEqual(len(temp_dir.child("examplefile.txt").listdir()), 1)
with temp_dir.child("examplefile.txt").child("chunk_1").open("rb") as f:
self.assertEqual(f.read(), b"hello Cros")
self.assertEqual(len(upload_dir.listdir()), 0)
del resource
# Add some random junk in there that Crossbar isn't expecting
temp_dir.child("junk").setContent(b"just some junk")
temp_dir.child("examplefile.txt").child("hi").setContent(b"what")
# Simulate restarting Crossbar by reinitialising the FileUploadResource
resource = FileUploadResource(upload_dir.path, temp_dir.path, fields, mock_session)
#
# Chunk 2
mp = Multipart()
mp.add_part(b"resumableChunkNumber", b"2")
mp.add_part(b"resumableChunkSize", b"10")
mp.add_part(b"resumableCurrentChunkSize", b"6")
mp.add_part(b"resumableTotalSize", b"16")
mp.add_part(b"resumableType", b"text/plain")
mp.add_part(b"resumableIdentifier", b"16-examplefiletxt")
mp.add_part(b"resumableFilename", b"examplefile.txt")
mp.add_part(b"resumableRelativePath", b"examplefile.txt")
mp.add_part(b"resumableTotalChunks", b"2")
mp.add_part(b"on_progress", b"com.example.upload.on_progress")
mp.add_part(b"session", b"6891276359801283")
mp.add_part(b"file", b"sbar!\n",
content_type=b"application/octet-stream",
filename=b"blob")
body, headers = mp.render()
d = renderResource(
resource, b"/", method="POST",
headers=headers,
body=body
)
res = self.successResultOf(d)
self.assertEqual(res.code, 200)
self.assertEqual(len(mock_session.method_calls), 4)
# Starting the upload
self.assertEqual(mock_session.method_calls[0][1][0], u"com.example.upload.on_progress")
self.assertEqual(mock_session.method_calls[0][1][1]["status"], "started")
self.assertEqual(mock_session.method_calls[0][1][1]["id"], "examplefile.txt")
self.assertEqual(mock_session.method_calls[0][1][1]["chunk"], 1)
# Progress, first chunk done
self.assertEqual(mock_session.method_calls[1][1][0], u"com.example.upload.on_progress")
self.assertEqual(mock_session.method_calls[1][1][1]["status"], "progress")
self.assertEqual(mock_session.method_calls[1][1][1]["id"], "examplefile.txt")
self.assertEqual(mock_session.method_calls[1][1][1]["chunk"], 1)
# Progress, second chunk done
self.assertEqual(mock_session.method_calls[2][1][0], u"com.example.upload.on_progress")
self.assertEqual(mock_session.method_calls[2][1][1]["status"], "progress")
self.assertEqual(mock_session.method_calls[2][1][1]["id"], "examplefile.txt")
self.assertEqual(mock_session.method_calls[2][1][1]["chunk"], 2)
# Upload complete
self.assertEqual(mock_session.method_calls[3][1][0], u"com.example.upload.on_progress")
self.assertEqual(mock_session.method_calls[3][1][1]["status"], "finished")
self.assertEqual(mock_session.method_calls[3][1][1]["id"], "examplefile.txt")
self.assertEqual(mock_session.method_calls[3][1][1]["chunk"], 2)
# No item in the temp dir which we made earlier, one item in the
# upload dir. Otherjunk is removed because it belongs to no upload.
self.assertEqual(len(temp_dir.listdir()), 0)
self.assertEqual(len(upload_dir.listdir()), 1)
with upload_dir.child("examplefile.txt").open("rb") as f:
self.assertEqual(f.read(), b"hello Crossbar!\n")
class DeckStore(object):
def __init__(self,
enabled_directory=config.decks_enabled_directory,
available_directory=config.decks_available_directory):
self.enabled_directory = FilePath(enabled_directory)
self.available_directory = FilePath(available_directory)
self._cache = {}
self._cache_stale = True
def _list(self):
if self._cache_stale:
self._update_cache()
for deck_id, deck in self._cache.iteritems():
yield (deck_id, deck)
def list(self):
decks = []
for deck_id, deck in self._list():
decks.append((deck_id, deck))
return decks
def list_enabled(self):
decks = []
for deck_id, deck in self._list():
if not self.is_enabled(deck_id):
continue
decks.append((deck_id, deck))
return decks
def is_enabled(self, deck_id):
return self.enabled_directory.child(deck_id + '.yaml').exists()
def enable(self, deck_id):
deck_path = self.available_directory.child(deck_id + '.yaml')
if not deck_path.exists():
raise DeckNotFound(deck_id)
deck_enabled_path = self.enabled_directory.child(deck_id + '.yaml')
try:
deck_path.linkTo(deck_enabled_path)
except OSError as ose:
if ose.errno != errno.EEXIST:
raise
def disable(self, deck_id):
deck_enabled_path = self.enabled_directory.child(deck_id + '.yaml')
if not deck_enabled_path.exists():
raise DeckNotFound(deck_id)
deck_enabled_path.remove()
def _update_cache(self):
new_cache = {}
for deck_path in self.available_directory.listdir():
if not deck_path.endswith('.yaml'):
continue
deck = NGDeck(
deck_path=self.available_directory.child(deck_path).path)
new_cache[deck.id] = deck
self._cache = new_cache
self._cache_stale = False
def get(self, deck_id):
if self._cache_stale:
self._update_cache()
try:
return deepcopy(self._cache[deck_id])
except KeyError:
raise DeckNotFound(deck_id)
def test_multichunk(self):
"""
Uploading files that are in multiple chunks works.
"""
upload_dir = FilePath(self.mktemp())
upload_dir.makedirs()
temp_dir = FilePath(self.mktemp())
temp_dir.makedirs()
fields = {
"file_name": "resumableFilename",
"mime_type": "resumableType",
"total_size": "resumableTotalSize",
"chunk_number": "resumableChunkNumber",
"chunk_size": "resumableChunkSize",
"total_chunks": "resumableTotalChunks",
"content": "file",
"on_progress": "on_progress",
"session": "session"
}
mock_session = Mock()
resource = FileUploadResource(upload_dir.path, temp_dir.path, fields, mock_session)
#
# Chunk 1
mp = Multipart()
mp.add_part(b"resumableChunkNumber", b"1")
mp.add_part(b"resumableChunkSize", b"10")
mp.add_part(b"resumableCurrentChunkSize", b"10")
mp.add_part(b"resumableTotalSize", b"16")
mp.add_part(b"resumableType", b"text/plain")
mp.add_part(b"resumableIdentifier", b"16-examplefiletxt")
mp.add_part(b"resumableFilename", b"examplefile.txt")
mp.add_part(b"resumableRelativePath", b"examplefile.txt")
mp.add_part(b"resumableTotalChunks", b"2")
mp.add_part(b"on_progress", b"com.example.upload.on_progress")
mp.add_part(b"session", b"6891276359801283")
mp.add_part(b"file", b"hello Cros",
content_type=b"application/octet-stream",
filename=b"blob")
body, headers = mp.render()
d = renderResource(
resource, b"/", method="POST",
headers=headers,
body=body
)
res = self.successResultOf(d)
res.setResponseCode.assert_called_once_with(200)
# One directory in the temp dir, nothing in the upload dir, temp dir
# contains one chunk
self.assertEqual(len(temp_dir.listdir()), 1)
self.assertEqual(len(temp_dir.child("examplefile.txt").listdir()), 1)
with temp_dir.child("examplefile.txt").child("chunk_1").open("rb") as f:
self.assertEqual(f.read(), b"hello Cros")
self.assertEqual(len(upload_dir.listdir()), 0)
#
# Chunk 2
mp = Multipart()
mp.add_part(b"resumableChunkNumber", b"2")
mp.add_part(b"resumableChunkSize", b"10")
mp.add_part(b"resumableCurrentChunkSize", b"6")
mp.add_part(b"resumableTotalSize", b"16")
mp.add_part(b"resumableType", b"text/plain")
mp.add_part(b"resumableIdentifier", b"16-examplefiletxt")
mp.add_part(b"resumableFilename", b"examplefile.txt")
mp.add_part(b"resumableRelativePath", b"examplefile.txt")
mp.add_part(b"resumableTotalChunks", b"2")
mp.add_part(b"on_progress", b"com.example.upload.on_progress")
mp.add_part(b"session", b"6891276359801283")
mp.add_part(b"file", b"sbar!\n",
content_type=b"application/octet-stream",
filename=b"blob")
body, headers = mp.render()
d = renderResource(
resource, b"/", method="POST",
headers=headers,
body=body
)
res = self.successResultOf(d)
res.setResponseCode.assert_called_once_with(200)
self.assertEqual(len(mock_session.method_calls), 4)
# Starting the upload
self.assertEqual(mock_session.method_calls[0][1][0], u"com.example.upload.on_progress")
self.assertEqual(mock_session.method_calls[0][1][1]["status"], "started")
self.assertEqual(mock_session.method_calls[0][1][1]["id"], "examplefile.txt")
self.assertEqual(mock_session.method_calls[0][1][1]["chunk"], 1)
# Progress, first chunk done
self.assertEqual(mock_session.method_calls[1][1][0], u"com.example.upload.on_progress")
self.assertEqual(mock_session.method_calls[1][1][1]["status"], "progress")
self.assertEqual(mock_session.method_calls[1][1][1]["id"], "examplefile.txt")
self.assertEqual(mock_session.method_calls[1][1][1]["chunk"], 1)
# Progress, second chunk done
self.assertEqual(mock_session.method_calls[2][1][0], u"com.example.upload.on_progress")
self.assertEqual(mock_session.method_calls[2][1][1]["status"], "progress")
self.assertEqual(mock_session.method_calls[2][1][1]["id"], "examplefile.txt")
self.assertEqual(mock_session.method_calls[2][1][1]["chunk"], 2)
# Upload complete
self.assertEqual(mock_session.method_calls[3][1][0], u"com.example.upload.on_progress")
self.assertEqual(mock_session.method_calls[3][1][1]["status"], "finished")
self.assertEqual(mock_session.method_calls[3][1][1]["id"], "examplefile.txt")
self.assertEqual(mock_session.method_calls[3][1][1]["chunk"], 2)
# Nothing in the temp dir, one file in the upload
self.assertEqual(len(temp_dir.listdir()), 0)
self.assertEqual(len(upload_dir.listdir()), 1)
with upload_dir.child("examplefile.txt").open("rb") as f:
self.assertEqual(f.read(), b"hello Crossbar!\n")
class DeckStore(object):
def __init__(self, enabled_directory=config.decks_enabled_directory,
available_directory=config.decks_available_directory):
self.enabled_directory = FilePath(enabled_directory)
self.available_directory = FilePath(available_directory)
self._cache = {}
self._cache_stale = True
def _list(self):
if self._cache_stale:
self._update_cache()
for deck_id, deck in self._cache.iteritems():
yield (deck_id, deck)
def list(self):
decks = []
for deck_id, deck in self._list():
decks.append((deck_id, deck))
return decks
def list_enabled(self):
decks = []
for deck_id, deck in self._list():
if not self.is_enabled(deck_id):
continue
decks.append((deck_id, deck))
return decks
def is_enabled(self, deck_id):
return self.enabled_directory.child(deck_id + '.yaml').exists()
def enable(self, deck_id):
deck_path = self.available_directory.child(deck_id + '.yaml')
if not deck_path.exists():
raise DeckNotFound(deck_id)
deck_enabled_path = self.enabled_directory.child(deck_id + '.yaml')
try:
deck_path.linkTo(deck_enabled_path)
except OSError as ose:
if ose.errno != errno.EEXIST:
raise
def disable(self, deck_id):
deck_enabled_path = self.enabled_directory.child(deck_id + '.yaml')
if not deck_enabled_path.exists():
raise DeckNotFound(deck_id)
deck_enabled_path.remove()
def _update_cache(self):
new_cache = {}
for deck_path in self.available_directory.listdir():
if not deck_path.endswith('.yaml'):
continue
deck = NGDeck(
deck_path=self.available_directory.child(deck_path).path
)
new_cache[deck.id] = deck
self._cache = new_cache
self._cache_stale = False
def get(self, deck_id):
if self._cache_stale:
self._update_cache()
try:
return deepcopy(self._cache[deck_id])
except KeyError:
raise DeckNotFound(deck_id)
class BuildScriptsTests(TestCase):
"""
Tests for L{dist.build_scripts_twisted}.
"""
def setUp(self):
self.source = FilePath(self.mktemp())
self.target = FilePath(self.mktemp())
self.source.makedirs()
self.addCleanup(os.chdir, os.getcwd())
os.chdir(self.source.path)
def buildScripts(self):
"""
Write 3 types of scripts and run the L{build_scripts_twisted}
command.
"""
self.writeScript(self.source, "script1",
("#! /usr/bin/env python2.7\n"
"# bogus script w/ Python sh-bang\n"
"pass\n"))
self.writeScript(self.source, "script2.py",
("#!/usr/bin/python\n"
"# bogus script w/ Python sh-bang\n"
"pass\n"))
self.writeScript(self.source, "shell.sh",
("#!/bin/sh\n"
"# bogus shell script w/ sh-bang\n"
"exit 0\n"))
expected = ['script1', 'script2.py', 'shell.sh']
cmd = self.getBuildScriptsCmd(self.target,
[self.source.child(fn).path
for fn in expected])
cmd.finalize_options()
cmd.run()
return self.target.listdir()
def getBuildScriptsCmd(self, target, scripts):
"""
Create a distutils L{Distribution} with a L{DummyCommand} and wrap it
in L{build_scripts_twisted}.
@type target: L{FilePath}
"""
dist = Distribution()
dist.scripts = scripts
dist.command_obj["build"] = DummyCommand(
build_scripts = target.path,
force = 1,
executable = sys.executable
)
return build_scripts_twisted(dist)
def writeScript(self, dir, name, text):
"""
Write the script to disk.
"""
with open(dir.child(name).path, "w") as f:
f.write(text)
def test_notWindows(self):
"""
L{build_scripts_twisted} does not rename scripts on non-Windows
platforms.
"""
self.patch(os, "name", "twisted")
built = self.buildScripts()
for name in ['script1', 'script2.py', 'shell.sh']:
self.assertTrue(name in built)
def test_windows(self):
"""
L{build_scripts_twisted} renames scripts so they end with '.py' on
the Windows platform.
"""
self.patch(os, "name", "nt")
built = self.buildScripts()
for name in ['script1.py', 'script2.py', 'shell.sh.py']:
self.assertTrue(name in built)
def test_resumed_upload(self):
"""
Uploading part of a file, simulating a Crossbar restart, and continuing
to upload works.
"""
upload_dir = FilePath(self.mktemp())
upload_dir.makedirs()
temp_dir = FilePath(self.mktemp())
temp_dir.makedirs()
fields = {
"file_name": "resumableFilename",
"mime_type": "resumableType",
"total_size": "resumableTotalSize",
"chunk_number": "resumableChunkNumber",
"chunk_size": "resumableChunkSize",
"total_chunks": "resumableTotalChunks",
"content": "file",
"on_progress": "on_progress",
"session": "session",
}
mock_session = Mock()
resource = FileUploadResource(upload_dir.path, temp_dir.path, fields, mock_session)
# Make some stuff in the temp dir that wasn't there when it started but
# is put there before a file upload
temp_dir.child("otherjunk").makedirs()
#
# Chunk 1
mp = Multipart()
mp.add_part(b"resumableChunkNumber", b"1")
mp.add_part(b"resumableChunkSize", b"10")
mp.add_part(b"resumableCurrentChunkSize", b"10")
mp.add_part(b"resumableTotalSize", b"16")
mp.add_part(b"resumableType", b"text/plain")
mp.add_part(b"resumableIdentifier", b"16-examplefiletxt")
mp.add_part(b"resumableFilename", b"examplefile.txt")
mp.add_part(b"resumableRelativePath", b"examplefile.txt")
mp.add_part(b"resumableTotalChunks", b"2")
mp.add_part(b"on_progress", b"com.example.upload.on_progress")
mp.add_part(b"session", b"6891276359801283")
mp.add_part(b"file", b"hello Cros", content_type=b"application/octet-stream", filename=b"blob")
body, headers = mp.render()
d = renderResource(resource, b"/", method="POST", headers=headers, body=body)
res = self.successResultOf(d)
self.assertEqual(res.code, 200)
# One directory in the temp dir, nothing in the upload dir, temp dir
# contains one chunk
self.assertEqual(len(temp_dir.listdir()), 1)
self.assertEqual(len(temp_dir.child("examplefile.txt").listdir()), 1)
with temp_dir.child("examplefile.txt").child("chunk_1").open("rb") as f:
self.assertEqual(f.read(), b"hello Cros")
self.assertEqual(len(upload_dir.listdir()), 0)
del resource
# Add some random junk in there that Crossbar isn't expecting
temp_dir.child("junk").setContent(b"just some junk")
temp_dir.child("examplefile.txt").child("hi").setContent(b"what")
# Simulate restarting Crossbar by reinitialising the FileUploadResource
resource = FileUploadResource(upload_dir.path, temp_dir.path, fields, mock_session)
#
# Chunk 2
mp = Multipart()
mp.add_part(b"resumableChunkNumber", b"2")
mp.add_part(b"resumableChunkSize", b"10")
mp.add_part(b"resumableCurrentChunkSize", b"6")
mp.add_part(b"resumableTotalSize", b"16")
mp.add_part(b"resumableType", b"text/plain")
mp.add_part(b"resumableIdentifier", b"16-examplefiletxt")
mp.add_part(b"resumableFilename", b"examplefile.txt")
mp.add_part(b"resumableRelativePath", b"examplefile.txt")
mp.add_part(b"resumableTotalChunks", b"2")
mp.add_part(b"on_progress", b"com.example.upload.on_progress")
mp.add_part(b"session", b"6891276359801283")
mp.add_part(b"file", b"sbar!\n", content_type=b"application/octet-stream", filename=b"blob")
body, headers = mp.render()
d = renderResource(resource, b"/", method="POST", headers=headers, body=body)
res = self.successResultOf(d)
self.assertEqual(res.code, 200)
self.assertEqual(len(mock_session.method_calls), 4)
# Starting the upload
self.assertEqual(mock_session.method_calls[0][1][0], u"com.example.upload.on_progress")
self.assertEqual(mock_session.method_calls[0][1][1]["status"], "started")
self.assertEqual(mock_session.method_calls[0][1][1]["id"], "examplefile.txt")
self.assertEqual(mock_session.method_calls[0][1][1]["chunk"], 1)
# Progress, first chunk done
self.assertEqual(mock_session.method_calls[1][1][0], u"com.example.upload.on_progress")
self.assertEqual(mock_session.method_calls[1][1][1]["status"], "progress")
self.assertEqual(mock_session.method_calls[1][1][1]["id"], "examplefile.txt")
self.assertEqual(mock_session.method_calls[1][1][1]["chunk"], 1)
# Progress, second chunk done
self.assertEqual(mock_session.method_calls[2][1][0], u"com.example.upload.on_progress")
self.assertEqual(mock_session.method_calls[2][1][1]["status"], "progress")
self.assertEqual(mock_session.method_calls[2][1][1]["id"], "examplefile.txt")
self.assertEqual(mock_session.method_calls[2][1][1]["chunk"], 2)
# Upload complete
self.assertEqual(mock_session.method_calls[3][1][0], u"com.example.upload.on_progress")
self.assertEqual(mock_session.method_calls[3][1][1]["status"], "finished")
self.assertEqual(mock_session.method_calls[3][1][1]["id"], "examplefile.txt")
self.assertEqual(mock_session.method_calls[3][1][1]["chunk"], 2)
# No item in the temp dir which we made earlier, one item in the
# upload dir. Otherjunk is removed because it belongs to no upload.
self.assertEqual(len(temp_dir.listdir()), 0)
self.assertEqual(len(upload_dir.listdir()), 1)
with upload_dir.child("examplefile.txt").open("rb") as f:
self.assertEqual(f.read(), b"hello Crossbar!\n")
def test_multichunk_shuffle(self):
"""
Uploading files that are in multiple chunks and are uploaded in different order works.
"""
upload_dir = FilePath(self.mktemp())
upload_dir.makedirs()
temp_dir = FilePath(self.mktemp())
temp_dir.makedirs()
fields = {
"file_name": "resumableFilename",
"mime_type": "resumableType",
"total_size": "resumableTotalSize",
"chunk_number": "resumableChunkNumber",
"chunk_size": "resumableChunkSize",
"total_chunks": "resumableTotalChunks",
"content": "file",
"on_progress": "on_progress",
"session": "session",
}
mock_session = Mock()
resource = FileUploadResource(upload_dir.path, temp_dir.path, fields, mock_session)
#
# Chunk 2
multipart_body = b"""-----------------------------42560029919436807832069165364\r\nContent-Disposition: form-data; name="resumableChunkNumber"\r\n\r\n2\r\n-----------------------------42560029919436807832069165364\r\nContent-Disposition: form-data; name="resumableChunkSize"\r\n\r\n10\r\n-----------------------------42560029919436807832069165364\r\nContent-Disposition: form-data; name="resumableCurrentChunkSize"\r\n\r\n6\r\n-----------------------------42560029919436807832069165364\r\nContent-Disposition: form-data; name="resumableTotalSize"\r\n\r\n16\r\n-----------------------------42560029919436807832069165364\r\nContent-Disposition: form-data; name="resumableType"\r\n\r\ntext/plain\r\n-----------------------------42560029919436807832069165364\r\nContent-Disposition: form-data; name="resumableIdentifier"\r\n\r\n16-examplefiletxt\r\n-----------------------------42560029919436807832069165364\r\nContent-Disposition: form-data; name="resumableFilename"\r\n\r\nexamplefile.txt\r\n-----------------------------42560029919436807832069165364\r\nContent-Disposition: form-data; name="resumableRelativePath"\r\n\r\nexamplefile.txt\r\n-----------------------------42560029919436807832069165364\r\nContent-Disposition: form-data; name="resumableTotalChunks"\r\n\r\n2\r\n-----------------------------42560029919436807832069165364\r\nContent-Disposition: form-data; name="on_progress"\r\n\r\ncom.example.upload.on_progress\r\n-----------------------------42560029919436807832069165364\r\nContent-Disposition: form-data; name="session"\r\n\r\n8887465641628580\r\n-----------------------------42560029919436807832069165364\r\nContent-Disposition: form-data; name="file"; filename="blob"\r\nContent-Type: application/octet-stream\r\n\r\nsbar!\n\r\n-----------------------------42560029919436807832069165364--\r\n"""
d = renderResource(
resource,
b"/",
method="POST",
headers={
b"content-type": [
b"multipart/form-data; boundary=---------------------------42560029919436807832069165364"
],
b"Content-Length": [b"1688"],
},
body=multipart_body,
)
res = self.successResultOf(d)
self.assertEqual(res.code, 200)
# One directory in the temp dir, nothing in the upload dir, temp dir
# contains one chunk
self.assertEqual(len(temp_dir.listdir()), 1)
self.assertEqual(len(temp_dir.child("examplefile.txt").listdir()), 1)
with temp_dir.child("examplefile.txt").child("chunk_2").open("rb") as f:
self.assertEqual(f.read(), b"sbar!\n")
self.assertEqual(len(upload_dir.listdir()), 0)
#
# Chunk 1
multipart_body = b"""-----------------------------1311987731215707521443909311\r\nContent-Disposition: form-data; name="resumableChunkNumber"\r\n\r\n1\r\n-----------------------------1311987731215707521443909311\r\nContent-Disposition: form-data; name="resumableChunkSize"\r\n\r\n10\r\n-----------------------------1311987731215707521443909311\r\nContent-Disposition: form-data; name="resumableCurrentChunkSize"\r\n\r\n10\r\n-----------------------------1311987731215707521443909311\r\nContent-Disposition: form-data; name="resumableTotalSize"\r\n\r\n16\r\n-----------------------------1311987731215707521443909311\r\nContent-Disposition: form-data; name="resumableType"\r\n\r\ntext/plain\r\n-----------------------------1311987731215707521443909311\r\nContent-Disposition: form-data; name="resumableIdentifier"\r\n\r\n16-examplefiletxt\r\n-----------------------------1311987731215707521443909311\r\nContent-Disposition: form-data; name="resumableFilename"\r\n\r\nexamplefile.txt\r\n-----------------------------1311987731215707521443909311\r\nContent-Disposition: form-data; name="resumableRelativePath"\r\n\r\nexamplefile.txt\r\n-----------------------------1311987731215707521443909311\r\nContent-Disposition: form-data; name="resumableTotalChunks"\r\n\r\n2\r\n-----------------------------1311987731215707521443909311\r\nContent-Disposition: form-data; name="on_progress"\r\n\r\ncom.example.upload.on_progress\r\n-----------------------------1311987731215707521443909311\r\nContent-Disposition: form-data; name="session"\r\n\r\n8887465641628580\r\n-----------------------------1311987731215707521443909311\r\nContent-Disposition: form-data; name="file"; filename="blob"\r\nContent-Type: application/octet-stream\r\n\r\nhello Cros\r\n-----------------------------1311987731215707521443909311--\r\n"""
d = renderResource(
resource,
b"/",
method="POST",
headers={
b"content-type": [
b"multipart/form-data; boundary=---------------------------1311987731215707521443909311"
],
b"Content-Length": [b"1680"],
},
body=multipart_body,
)
res = self.successResultOf(d)
self.assertEqual(res.code, 200)
self.assertEqual(len(mock_session.method_calls), 4)
# Starting the upload
self.assertEqual(mock_session.method_calls[0][1][0], u"com.example.upload.on_progress")
self.assertEqual(mock_session.method_calls[0][1][1]["status"], "started")
self.assertEqual(mock_session.method_calls[0][1][1]["id"], "examplefile.txt")
self.assertEqual(mock_session.method_calls[0][1][1]["chunk"], 2)
# Progress, first chunk done
self.assertEqual(mock_session.method_calls[1][1][0], u"com.example.upload.on_progress")
self.assertEqual(mock_session.method_calls[1][1][1]["status"], "progress")
self.assertEqual(mock_session.method_calls[1][1][1]["id"], "examplefile.txt")
self.assertEqual(mock_session.method_calls[1][1][1]["chunk"], 2)
# Progress, second chunk done
self.assertEqual(mock_session.method_calls[2][1][0], u"com.example.upload.on_progress")
self.assertEqual(mock_session.method_calls[2][1][1]["status"], "progress")
self.assertEqual(mock_session.method_calls[2][1][1]["id"], "examplefile.txt")
self.assertEqual(mock_session.method_calls[2][1][1]["chunk"], 1)
# Upload complete
self.assertEqual(mock_session.method_calls[3][1][0], u"com.example.upload.on_progress")
self.assertEqual(mock_session.method_calls[3][1][1]["status"], "finished")
self.assertEqual(mock_session.method_calls[3][1][1]["id"], "examplefile.txt")
self.assertEqual(mock_session.method_calls[3][1][1]["chunk"], 1)
# Nothing in the temp dir, one file in the upload
self.assertEqual(len(temp_dir.listdir()), 0)
self.assertEqual(len(upload_dir.listdir()), 1)
with upload_dir.child("examplefile.txt").open("rb") as f:
self.assertEqual(f.read(), b"hello Crossbar!\n")
def test_basic(self):
"""
Upload a basic file using the FileUploadResource, in just a single chunk.
"""
upload_dir = FilePath(self.mktemp())
upload_dir.makedirs()
temp_dir = FilePath(self.mktemp())
temp_dir.makedirs()
fields = {
"file_name": "resumableFilename",
"mime_type": "resumableType",
"total_size": "resumableTotalSize",
"chunk_number": "resumableChunkNumber",
"chunk_size": "resumableChunkSize",
"total_chunks": "resumableTotalChunks",
"content": "file",
"on_progress": "on_progress",
"session": "session",
}
mock_session = Mock()
resource = FileUploadResource(upload_dir.path, temp_dir.path, fields, mock_session)
mp = Multipart()
mp.add_part(b"resumableChunkNumber", b"1")
mp.add_part(b"resumableChunkSize", b"1048576")
mp.add_part(b"resumableCurrentChunkSize", b"16")
mp.add_part(b"resumableTotalSize", b"16")
mp.add_part(b"resumableType", b"text/plain")
mp.add_part(b"resumableIdentifier", b"16-examplefiletxt")
mp.add_part(b"resumableFilename", b"examplefile.txt")
mp.add_part(b"resumableRelativePath", b"examplefile.txt")
mp.add_part(b"resumableTotalChunks", b"1")
mp.add_part(b"on_progress", b"com.example.upload.on_progress")
mp.add_part(b"session", b"6891276359801283")
mp.add_part(b"file", b"hello Crossbar!\n", content_type=b"application/octet-stream", filename=b"blob")
body, headers = mp.render()
d = renderResource(resource, b"/", method="POST", headers=headers, body=body)
res = self.successResultOf(d)
self.assertEqual(res.code, 200)
self.assertEqual(len(mock_session.method_calls), 2)
# Starting the upload
self.assertEqual(mock_session.method_calls[0][1][0], u"com.example.upload.on_progress")
self.assertEqual(mock_session.method_calls[0][1][1]["status"], "started")
self.assertEqual(mock_session.method_calls[0][1][1]["id"], "examplefile.txt")
# Upload complete
self.assertEqual(mock_session.method_calls[1][1][0], u"com.example.upload.on_progress")
self.assertEqual(mock_session.method_calls[1][1][1]["status"], "finished")
self.assertEqual(mock_session.method_calls[1][1][1]["id"], "examplefile.txt")
# Nothing in the temp dir, one file in the upload
self.assertEqual(len(temp_dir.listdir()), 0)
self.assertEqual(len(upload_dir.listdir()), 1)
with upload_dir.child("examplefile.txt").open("rb") as f:
self.assertEqual(f.read(), b"hello Crossbar!\n")
def test_multichunk_shuffle(self):
"""
Uploading files that are in multiple chunks and are uploaded in different order works.
"""
upload_dir = FilePath(self.mktemp())
upload_dir.makedirs()
temp_dir = FilePath(self.mktemp())
temp_dir.makedirs()
fields = {
"file_name": "resumableFilename",
"mime_type": "resumableType",
"total_size": "resumableTotalSize",
"chunk_number": "resumableChunkNumber",
"chunk_size": "resumableChunkSize",
"total_chunks": "resumableTotalChunks",
"content": "file",
"on_progress": "on_progress",
"session": "session"
}
mock_session = Mock()
resource = FileUploadResource(upload_dir.path, temp_dir.path, fields, mock_session)
#
# Chunk 2
multipart_body = b"""-----------------------------42560029919436807832069165364\r\nContent-Disposition: form-data; name="resumableChunkNumber"\r\n\r\n2\r\n-----------------------------42560029919436807832069165364\r\nContent-Disposition: form-data; name="resumableChunkSize"\r\n\r\n10\r\n-----------------------------42560029919436807832069165364\r\nContent-Disposition: form-data; name="resumableCurrentChunkSize"\r\n\r\n6\r\n-----------------------------42560029919436807832069165364\r\nContent-Disposition: form-data; name="resumableTotalSize"\r\n\r\n16\r\n-----------------------------42560029919436807832069165364\r\nContent-Disposition: form-data; name="resumableType"\r\n\r\ntext/plain\r\n-----------------------------42560029919436807832069165364\r\nContent-Disposition: form-data; name="resumableIdentifier"\r\n\r\n16-examplefiletxt\r\n-----------------------------42560029919436807832069165364\r\nContent-Disposition: form-data; name="resumableFilename"\r\n\r\nexamplefile.txt\r\n-----------------------------42560029919436807832069165364\r\nContent-Disposition: form-data; name="resumableRelativePath"\r\n\r\nexamplefile.txt\r\n-----------------------------42560029919436807832069165364\r\nContent-Disposition: form-data; name="resumableTotalChunks"\r\n\r\n2\r\n-----------------------------42560029919436807832069165364\r\nContent-Disposition: form-data; name="on_progress"\r\n\r\ncom.example.upload.on_progress\r\n-----------------------------42560029919436807832069165364\r\nContent-Disposition: form-data; name="session"\r\n\r\n8887465641628580\r\n-----------------------------42560029919436807832069165364\r\nContent-Disposition: form-data; name="file"; filename="blob"\r\nContent-Type: application/octet-stream\r\n\r\nsbar!\n\r\n-----------------------------42560029919436807832069165364--\r\n"""
d = renderResource(
resource, b"/", method="POST",
headers={
b"content-type": [b"multipart/form-data; boundary=---------------------------42560029919436807832069165364"],
b"Content-Length": [b"1688"]
},
body=multipart_body
)
res = self.successResultOf(d)
self.assertEqual(res.code, 200)
# One directory in the temp dir, nothing in the upload dir, temp dir
# contains one chunk
self.assertEqual(len(temp_dir.listdir()), 1)
self.assertEqual(len(temp_dir.child("examplefile.txt").listdir()), 1)
with temp_dir.child("examplefile.txt").child("chunk_2").open("rb") as f:
self.assertEqual(f.read(), b"sbar!\n")
self.assertEqual(len(upload_dir.listdir()), 0)
#
# Chunk 1
multipart_body = b"""-----------------------------1311987731215707521443909311\r\nContent-Disposition: form-data; name="resumableChunkNumber"\r\n\r\n1\r\n-----------------------------1311987731215707521443909311\r\nContent-Disposition: form-data; name="resumableChunkSize"\r\n\r\n10\r\n-----------------------------1311987731215707521443909311\r\nContent-Disposition: form-data; name="resumableCurrentChunkSize"\r\n\r\n10\r\n-----------------------------1311987731215707521443909311\r\nContent-Disposition: form-data; name="resumableTotalSize"\r\n\r\n16\r\n-----------------------------1311987731215707521443909311\r\nContent-Disposition: form-data; name="resumableType"\r\n\r\ntext/plain\r\n-----------------------------1311987731215707521443909311\r\nContent-Disposition: form-data; name="resumableIdentifier"\r\n\r\n16-examplefiletxt\r\n-----------------------------1311987731215707521443909311\r\nContent-Disposition: form-data; name="resumableFilename"\r\n\r\nexamplefile.txt\r\n-----------------------------1311987731215707521443909311\r\nContent-Disposition: form-data; name="resumableRelativePath"\r\n\r\nexamplefile.txt\r\n-----------------------------1311987731215707521443909311\r\nContent-Disposition: form-data; name="resumableTotalChunks"\r\n\r\n2\r\n-----------------------------1311987731215707521443909311\r\nContent-Disposition: form-data; name="on_progress"\r\n\r\ncom.example.upload.on_progress\r\n-----------------------------1311987731215707521443909311\r\nContent-Disposition: form-data; name="session"\r\n\r\n8887465641628580\r\n-----------------------------1311987731215707521443909311\r\nContent-Disposition: form-data; name="file"; filename="blob"\r\nContent-Type: application/octet-stream\r\n\r\nhello Cros\r\n-----------------------------1311987731215707521443909311--\r\n"""
d = renderResource(
resource, b"/", method="POST",
headers={
b"content-type": [b"multipart/form-data; boundary=---------------------------1311987731215707521443909311"],
b"Content-Length": [b"1680"]
},
body=multipart_body
)
res = self.successResultOf(d)
self.assertEqual(res.code, 200)
self.assertEqual(len(mock_session.method_calls), 4)
# Starting the upload
self.assertEqual(mock_session.method_calls[0][1][0], u"com.example.upload.on_progress")
self.assertEqual(mock_session.method_calls[0][1][1]["status"], "started")
self.assertEqual(mock_session.method_calls[0][1][1]["id"], "examplefile.txt")
self.assertEqual(mock_session.method_calls[0][1][1]["chunk"], 2)
# Progress, first chunk done
self.assertEqual(mock_session.method_calls[1][1][0], u"com.example.upload.on_progress")
self.assertEqual(mock_session.method_calls[1][1][1]["status"], "progress")
self.assertEqual(mock_session.method_calls[1][1][1]["id"], "examplefile.txt")
self.assertEqual(mock_session.method_calls[1][1][1]["chunk"], 2)
# Progress, second chunk done
self.assertEqual(mock_session.method_calls[2][1][0], u"com.example.upload.on_progress")
self.assertEqual(mock_session.method_calls[2][1][1]["status"], "progress")
self.assertEqual(mock_session.method_calls[2][1][1]["id"], "examplefile.txt")
self.assertEqual(mock_session.method_calls[2][1][1]["chunk"], 1)
# Upload complete
self.assertEqual(mock_session.method_calls[3][1][0], u"com.example.upload.on_progress")
self.assertEqual(mock_session.method_calls[3][1][1]["status"], "finished")
self.assertEqual(mock_session.method_calls[3][1][1]["id"], "examplefile.txt")
self.assertEqual(mock_session.method_calls[3][1][1]["chunk"], 1)
# Nothing in the temp dir, one file in the upload
self.assertEqual(len(temp_dir.listdir()), 0)
self.assertEqual(len(upload_dir.listdir()), 1)
with upload_dir.child("examplefile.txt").open("rb") as f:
self.assertEqual(f.read(), b"hello Crossbar!\n")
def test_multichunk(self):
"""
Uploading files that are in multiple chunks works.
"""
upload_dir = FilePath(self.mktemp())
upload_dir.makedirs()
temp_dir = FilePath(self.mktemp())
temp_dir.makedirs()
fields = {
"file_name": "resumableFilename",
"mime_type": "resumableType",
"total_size": "resumableTotalSize",
"chunk_number": "resumableChunkNumber",
"chunk_size": "resumableChunkSize",
"total_chunks": "resumableTotalChunks",
"content": "file",
"on_progress": "on_progress",
"session": "session"
}
mock_session = Mock()
resource = FileUploadResource(upload_dir.path, temp_dir.path, fields,
mock_session)
#
# Chunk 1
mp = Multipart()
mp.add_part(b"resumableChunkNumber", b"1")
mp.add_part(b"resumableChunkSize", b"10")
mp.add_part(b"resumableCurrentChunkSize", b"10")
mp.add_part(b"resumableTotalSize", b"16")
mp.add_part(b"resumableType", b"text/plain")
mp.add_part(b"resumableIdentifier", b"16-examplefiletxt")
mp.add_part(b"resumableFilename", b"examplefile.txt")
mp.add_part(b"resumableRelativePath", b"examplefile.txt")
mp.add_part(b"resumableTotalChunks", b"2")
mp.add_part(b"on_progress", b"com.example.upload.on_progress")
mp.add_part(b"session", b"6891276359801283")
mp.add_part(b"file",
b"hello Cros",
content_type=b"application/octet-stream",
filename=b"blob")
body, headers = mp.render()
d = renderResource(resource,
b"/",
method="POST",
headers=headers,
body=body)
res = self.successResultOf(d)
res.setResponseCode.assert_called_once_with(200)
# One directory in the temp dir, nothing in the upload dir, temp dir
# contains one chunk
self.assertEqual(len(temp_dir.listdir()), 1)
self.assertEqual(len(temp_dir.child("examplefile.txt").listdir()), 1)
with temp_dir.child("examplefile.txt").child("chunk_1").open(
"rb") as f:
self.assertEqual(f.read(), b"hello Cros")
self.assertEqual(len(upload_dir.listdir()), 0)
#
# Chunk 2
mp = Multipart()
mp.add_part(b"resumableChunkNumber", b"2")
mp.add_part(b"resumableChunkSize", b"10")
mp.add_part(b"resumableCurrentChunkSize", b"6")
mp.add_part(b"resumableTotalSize", b"16")
mp.add_part(b"resumableType", b"text/plain")
mp.add_part(b"resumableIdentifier", b"16-examplefiletxt")
mp.add_part(b"resumableFilename", b"examplefile.txt")
mp.add_part(b"resumableRelativePath", b"examplefile.txt")
mp.add_part(b"resumableTotalChunks", b"2")
mp.add_part(b"on_progress", b"com.example.upload.on_progress")
mp.add_part(b"session", b"6891276359801283")
mp.add_part(b"file",
b"sbar!\n",
content_type=b"application/octet-stream",
filename=b"blob")
body, headers = mp.render()
d = renderResource(resource,
b"/",
method="POST",
headers=headers,
body=body)
res = self.successResultOf(d)
res.setResponseCode.assert_called_once_with(200)
self.assertEqual(len(mock_session.method_calls), 4)
# Starting the upload
self.assertEqual(mock_session.method_calls[0][1][0],
u"com.example.upload.on_progress")
self.assertEqual(mock_session.method_calls[0][1][1]["status"],
"started")
self.assertEqual(mock_session.method_calls[0][1][1]["id"],
"examplefile.txt")
self.assertEqual(mock_session.method_calls[0][1][1]["chunk"], 1)
# Progress, first chunk done
self.assertEqual(mock_session.method_calls[1][1][0],
u"com.example.upload.on_progress")
self.assertEqual(mock_session.method_calls[1][1][1]["status"],
"progress")
self.assertEqual(mock_session.method_calls[1][1][1]["id"],
"examplefile.txt")
self.assertEqual(mock_session.method_calls[1][1][1]["chunk"], 1)
# Progress, second chunk done
self.assertEqual(mock_session.method_calls[2][1][0],
u"com.example.upload.on_progress")
self.assertEqual(mock_session.method_calls[2][1][1]["status"],
"progress")
self.assertEqual(mock_session.method_calls[2][1][1]["id"],
"examplefile.txt")
self.assertEqual(mock_session.method_calls[2][1][1]["chunk"], 2)
# Upload complete
self.assertEqual(mock_session.method_calls[3][1][0],
u"com.example.upload.on_progress")
self.assertEqual(mock_session.method_calls[3][1][1]["status"],
"finished")
self.assertEqual(mock_session.method_calls[3][1][1]["id"],
"examplefile.txt")
self.assertEqual(mock_session.method_calls[3][1][1]["chunk"], 2)
# Nothing in the temp dir, one file in the upload
self.assertEqual(len(temp_dir.listdir()), 0)
self.assertEqual(len(upload_dir.listdir()), 1)
with upload_dir.child("examplefile.txt").open("rb") as f:
self.assertEqual(f.read(), b"hello Crossbar!\n")
class DirDBM:
"""
A directory with a DBM interface.
This class presents a hash-like interface to a directory of small,
flat files. It can only use strings as keys or values.
"""
def __init__(self, name):
"""
@type name: str
@param name: Base path to use for the directory storage.
"""
self.dname = os.path.abspath(name)
self._dnamePath = FilePath(name)
if not self._dnamePath.isdir():
self._dnamePath.createDirectory()
else:
# Run recovery, in case we crashed. we delete all files ending
# with ".new". Then we find all files who end with ".rpl". If a
# corresponding file exists without ".rpl", we assume the write
# failed and delete the ".rpl" file. If only a ".rpl" exist we
# assume the program crashed right after deleting the old entry
# but before renaming the replacement entry.
#
# NOTE: '.' is NOT in the base64 alphabet!
for f in glob.glob(self._dnamePath.child("*.new").path):
os.remove(f)
replacements = glob.glob(self._dnamePath.child("*.rpl").path)
for f in replacements:
old = f[:-4]
if os.path.exists(old):
os.remove(f)
else:
os.rename(f, old)
def _encode(self, k):
"""
Encode a key so it can be used as a filename.
"""
# NOTE: '_' is NOT in the base64 alphabet!
return base64.encodestring(k).replace(b'\n', b'_').replace(b"/", b"-")
def _decode(self, k):
"""
Decode a filename to get the key.
"""
return base64.decodestring(k.replace(b'_', b'\n').replace(b"-", b"/"))
def _readFile(self, path):
"""
Read in the contents of a file.
Override in subclasses to e.g. provide transparently encrypted dirdbm.
"""
with _open(path.path, "rb") as f:
s = f.read()
return s
def _writeFile(self, path, data):
"""
Write data to a file.
Override in subclasses to e.g. provide transparently encrypted dirdbm.
"""
with _open(path.path, "wb") as f:
f.write(data)
f.flush()
def __len__(self):
"""
@return: The number of key/value pairs in this Shelf
"""
return len(self._dnamePath.listdir())
def __setitem__(self, k, v):
"""
C{dirdbm[k] = v}
Create or modify a textfile in this directory
@type k: bytes
@param k: key to set
@type v: bytes
@param v: value to associate with C{k}
"""
if not type(k) == bytes:
raise TypeError("DirDBM key must be bytes")
if not type(v) == bytes:
raise TypeError("DirDBM value must be bytes")
k = self._encode(k)
# We create a new file with extension .new, write the data to it, and
# if the write succeeds delete the old file and rename the new one.
old = self._dnamePath.child(k)
if old.exists():
new = old.siblingExtension(".rpl") # Replacement entry
else:
new = old.siblingExtension(".new") # New entry
try:
self._writeFile(new, v)
except:
new.remove()
raise
else:
if (old.exists()): old.remove()
new.moveTo(old)
def __getitem__(self, k):
"""
C{dirdbm[k]}
Get the contents of a file in this directory as a string.
@type k: bytes
@param k: key to lookup
@return: The value associated with C{k}
@raise KeyError: Raised when there is no such key
"""
if not type(k) == bytes:
raise TypeError("DirDBM key must be bytes")
path = self._dnamePath.child(self._encode(k))
try:
return self._readFile(path)
except (EnvironmentError):
raise KeyError(k)
def __delitem__(self, k):
"""
C{del dirdbm[foo]}
Delete a file in this directory.
@type k: bytes
@param k: key to delete
@raise KeyError: Raised when there is no such key
"""
if not type(k) == bytes:
raise TypeError("DirDBM key must be bytes")
k = self._encode(k)
try:
self._dnamePath.child(k).remove()
except (EnvironmentError):
raise KeyError(self._decode(k))
def keys(self):
"""
@return: a L{list} of filenames (keys).
"""
return list(map(self._decode, self._dnamePath.asBytesMode().listdir()))
def values(self):
"""
@return: a L{list} of file-contents (values).
"""
vals = []
keys = self.keys()
for key in keys:
vals.append(self[key])
return vals
def items(self):
"""
@return: a L{list} of 2-tuples containing key/value pairs.
"""
items = []
keys = self.keys()
for key in keys:
items.append((key, self[key]))
return items
def has_key(self, key):
"""
@type key: bytes
@param key: The key to test
@return: A true value if this dirdbm has the specified key, a false
value otherwise.
"""
if not type(key) == bytes:
raise TypeError("DirDBM key must be bytes")
key = self._encode(key)
return self._dnamePath.child(key).isfile()
def setdefault(self, key, value):
"""
@type key: bytes
@param key: The key to lookup
@param value: The value to associate with key if key is not already
associated with a value.
"""
if key not in self:
self[key] = value
return value
return self[key]
def get(self, key, default=None):
"""
@type key: bytes
@param key: The key to lookup
@param default: The value to return if the given key does not exist
@return: The value associated with C{key} or C{default} if not
L{DirDBM.has_key(key)}
"""
if key in self:
return self[key]
else:
return default
def __contains__(self, key):
"""
@see: L{DirDBM.has_key}
"""
return self.has_key(key)
def update(self, dict):
"""
Add all the key/value pairs in L{dict} to this dirdbm. Any conflicting
keys will be overwritten with the values from L{dict}.
@type dict: mapping
@param dict: A mapping of key/value pairs to add to this dirdbm.
"""
for key, val in dict.items():
self[key] = val
def copyTo(self, path):
"""
Copy the contents of this dirdbm to the dirdbm at C{path}.
@type path: L{str}
@param path: The path of the dirdbm to copy to. If a dirdbm
exists at the destination path, it is cleared first.
@rtype: C{DirDBM}
@return: The dirdbm this dirdbm was copied to.
"""
path = FilePath(path)
assert path != self._dnamePath
d = self.__class__(path.path)
d.clear()
for k in self.keys():
d[k] = self[k]
return d
def clear(self):
"""
Delete all key/value pairs in this dirdbm.
"""
for k in self.keys():
del self[k]
def close(self):
"""
Close this dbm: no-op, for dbm-style interface compliance.
"""
def getModificationTime(self, key):
"""
Returns modification time of an entry.
@return: Last modification date (seconds since epoch) of entry C{key}
@raise KeyError: Raised when there is no such key
"""
if not type(key) == bytes:
raise TypeError("DirDBM key must be bytes")
path = self._dnamePath.child(self._encode(key))
if path.isfile():
return path.getModificationTime()
else:
raise KeyError(key)
class DirDBM:
"""
A directory with a DBM interface.
This class presents a hash-like interface to a directory of small,
flat files. It can only use strings as keys or values.
"""
def __init__(self, name):
"""
@type name: str
@param name: Base path to use for the directory storage.
"""
self.dname = os.path.abspath(name)
self._dnamePath = FilePath(name)
if not self._dnamePath.isdir():
self._dnamePath.createDirectory()
else:
# Run recovery, in case we crashed. we delete all files ending
# with ".new". Then we find all files who end with ".rpl". If a
# corresponding file exists without ".rpl", we assume the write
# failed and delete the ".rpl" file. If only a ".rpl" exist we
# assume the program crashed right after deleting the old entry
# but before renaming the replacement entry.
#
# NOTE: '.' is NOT in the base64 alphabet!
for f in glob.glob(self._dnamePath.child("*.new").path):
os.remove(f)
replacements = glob.glob(self._dnamePath.child("*.rpl").path)
for f in replacements:
old = f[:-4]
if os.path.exists(old):
os.remove(f)
else:
os.rename(f, old)
def _encode(self, k):
"""
Encode a key so it can be used as a filename.
"""
# NOTE: '_' is NOT in the base64 alphabet!
return base64.encodestring(k).replace(b'\n', b'_').replace(b"/", b"-")
def _decode(self, k):
"""
Decode a filename to get the key.
"""
return base64.decodestring(k.replace(b'_', b'\n').replace(b"-", b"/"))
def _readFile(self, path):
"""
Read in the contents of a file.
Override in subclasses to e.g. provide transparently encrypted dirdbm.
"""
with _open(path.path, "rb") as f:
s = f.read()
return s
def _writeFile(self, path, data):
"""
Write data to a file.
Override in subclasses to e.g. provide transparently encrypted dirdbm.
"""
with _open(path.path, "wb") as f:
f.write(data)
f.flush()
def __len__(self):
"""
@return: The number of key/value pairs in this Shelf
"""
return len(self._dnamePath.listdir())
def __setitem__(self, k, v):
"""
C{dirdbm[k] = v}
Create or modify a textfile in this directory
@type k: bytes
@param k: key to set
@type v: bytes
@param v: value to associate with C{k}
"""
if not type(k) == bytes:
raise TypeError("DirDBM key must be bytes")
if not type(v) == bytes:
raise TypeError("DirDBM value must be bytes")
k = self._encode(k)
# We create a new file with extension .new, write the data to it, and
# if the write succeeds delete the old file and rename the new one.
old = self._dnamePath.child(k)
if old.exists():
new = old.siblingExtension(".rpl") # Replacement entry
else:
new = old.siblingExtension(".new") # New entry
try:
self._writeFile(new, v)
except:
new.remove()
raise
else:
if (old.exists()): old.remove()
new.moveTo(old)
def __getitem__(self, k):
"""
C{dirdbm[k]}
Get the contents of a file in this directory as a string.
@type k: bytes
@param k: key to lookup
@return: The value associated with C{k}
@raise KeyError: Raised when there is no such key
"""
if not type(k) == bytes:
raise TypeError("DirDBM key must be bytes")
path = self._dnamePath.child(self._encode(k))
try:
return self._readFile(path)
except (EnvironmentError):
raise KeyError(k)
def __delitem__(self, k):
"""
C{del dirdbm[foo]}
Delete a file in this directory.
@type k: bytes
@param k: key to delete
@raise KeyError: Raised when there is no such key
"""
if not type(k) == bytes:
raise TypeError("DirDBM key must be bytes")
k = self._encode(k)
try:
self._dnamePath.child(k).remove()
except (EnvironmentError):
raise KeyError(self._decode(k))
def keys(self):
"""
@return: a L{list} of filenames (keys).
"""
return list(map(self._decode, self._dnamePath.asBytesMode().listdir()))
def values(self):
"""
@return: a L{list} of file-contents (values).
"""
vals = []
keys = self.keys()
for key in keys:
vals.append(self[key])
return vals
def items(self):
"""
@return: a L{list} of 2-tuples containing key/value pairs.
"""
items = []
keys = self.keys()
for key in keys:
items.append((key, self[key]))
return items
def has_key(self, key):
"""
@type key: bytes
@param key: The key to test
@return: A true value if this dirdbm has the specified key, a false
value otherwise.
"""
if not type(key) == bytes:
raise TypeError("DirDBM key must be bytes")
key = self._encode(key)
return self._dnamePath.child(key).isfile()
def setdefault(self, key, value):
"""
@type key: bytes
@param key: The key to lookup
@param value: The value to associate with key if key is not already
associated with a value.
"""
if key not in self:
self[key] = value
return value
return self[key]
def get(self, key, default = None):
"""
@type key: bytes
@param key: The key to lookup
@param default: The value to return if the given key does not exist
@return: The value associated with C{key} or C{default} if not
L{DirDBM.has_key(key)}
"""
if key in self:
return self[key]
else:
return default
def __contains__(self, key):
"""
@see: L{DirDBM.has_key}
"""
return self.has_key(key)
def update(self, dict):
"""
Add all the key/value pairs in L{dict} to this dirdbm. Any conflicting
keys will be overwritten with the values from L{dict}.
@type dict: mapping
@param dict: A mapping of key/value pairs to add to this dirdbm.
"""
for key, val in dict.items():
self[key]=val
def copyTo(self, path):
"""
Copy the contents of this dirdbm to the dirdbm at C{path}.
@type path: L{str}
@param path: The path of the dirdbm to copy to. If a dirdbm
exists at the destination path, it is cleared first.
@rtype: C{DirDBM}
@return: The dirdbm this dirdbm was copied to.
"""
path = FilePath(path)
assert path != self._dnamePath
d = self.__class__(path.path)
d.clear()
for k in self.keys():
d[k] = self[k]
return d
def clear(self):
"""
Delete all key/value pairs in this dirdbm.
"""
for k in self.keys():
del self[k]
def close(self):
"""
Close this dbm: no-op, for dbm-style interface compliance.
"""
def getModificationTime(self, key):
"""
Returns modification time of an entry.
@return: Last modification date (seconds since epoch) of entry C{key}
@raise KeyError: Raised when there is no such key
"""
if not type(key) == bytes:
raise TypeError("DirDBM key must be bytes")
path = self._dnamePath.child(self._encode(key))
if path.isfile():
return path.getModificationTime()
else:
raise KeyError(key)
