def test_create_upload_id(self):
headers = {'content-range': 'bytes 0-8/18'}
dvvset = DVVSet()
modified_utc = '1515683246'
dot = dvvset.create(dvvset.new(modified_utc), self.user_id)
version = b64encode(json.dumps(dot).encode())
fn = "20180111_165127.jpg"
form_data = {
'prefix': '',
'md5': "437b930db84b8079c2dd804a71936b5f",
'guid': '',
'files[]': (fn, "something"),
'version': version
}
# check_part should work when
# binary data sent, IsCorrectMd5 is False
response = self._upload_request(headers, form_data)
self.assertEqual(response.status_code, 200)
response_json = response.json()
upload_id = response_json['upload_id']
guid = response_json['guid']
meta = self.head(UPLOADS_BUCKET_NAME, upload_id)
self.assertEqual(json.loads(b64decode(meta['version'])), dot)
self.assertEqual(bytes.fromhex(meta['orig-filename']), fn.encode())
self.assertEqual(meta['guid'], guid)
self.assertEqual(meta['bucket_id'], TEST_BUCKET_1)
self.assertEqual(meta['author-id'], self.user_id)
self.assertEqual(meta['is-deleted'], 'false')
# upload id should be specified for the first part only
url = "{}/riak/upload/{}/{}/2/".format(BASE_URL, TEST_BUCKET_1,
upload_id)
headers = {'content-range': 'bytes 9-17/18'}
md5_list = [
"437b930db84b8079c2dd804a71936b5f",
"437b930db84b8079c2dd804a71936b5f"
]
etags = ",".join(
["{},{}".format(i + 1, md5_list[i]) for i in range(len(md5_list))])
form_data.update({
'md5': '437b930db84b8079c2dd804a71936b5f',
'etags[]': etags
})
response = self._upload_request(headers, form_data, url=url)
self.assertEqual(response.status_code, 200)
def test_complete_upload(self):
# first upload a multi-part file
headers = {'content-range': 'bytes 0-18/28'}
dvvset = DVVSet()
modified_utc = '1515683246'
dot = dvvset.create(dvvset.new(modified_utc), self.user_id)
version = b64encode(json.dumps(dot).encode())
form_data = {
'prefix': '',
'md5': "79b547467286b3e20fad13f73fc1bf78",
'guid': '',
'files[]': ("20180111_165127.jpg", "something something"),
'version': version
}
response = self._upload_request(headers, form_data)
self.assertEqual(response.status_code, 200)
response_json = response.json()
guid = response_json['guid']
upload_id = response_json['upload_id']
md5 = response_json['md5']
# now remove the object
real_path = "~object/{}/{}/1_{}".format(guid, upload_id, md5)
self.remove_object(TEST_BUCKET_1, real_path)
url = "{}/riak/upload/{}/{}/2/".format(BASE_URL, TEST_BUCKET_1,
upload_id)
headers = {'content-range': 'bytes 19-27/28'}
md5_list = [
"79b547467286b3e20fad13f73fc1bf78",
"437b930db84b8079c2dd804a71936b5f"
]
etags = ",".join(
["{},{}".format(i + 1, md5_list[i]) for i in range(len(md5_list))])
form_data.update({
'files[]': ("20180111_165127.jpg", "something"),
'md5': '437b930db84b8079c2dd804a71936b5f',
'etags[]': etags
})
response = self._upload_request(headers, form_data, url=url)
self.assertEqual(response.status_code, 400)
self.assertEqual(response.json(), {'error': 51})
def _upload_request(self, headers, form_data, url=None, modified_utc=None):
"""
Returns arguments for upload
"""
if not url:
url = "{}/riak/upload/{}/".format(BASE_URL, TEST_BUCKET_1)
if 'files[]' in form_data:
if not modified_utc:
modified_utc = '1515683246'
else:
fn = "20180111_165127.jpg"
if not modified_utc:
modified_utc = str(int(os.stat(fn).st_mtime))
form_data.update({
'files[]': (fn, "something"),
})
if 'version' not in form_data:
dvvset = DVVSet()
dot = dvvset.create(dvvset.new(modified_utc), self.user_id)
version = b64encode(json.dumps(dot).encode())
form_data.update({
'version': version,
})
if 'md5' in form_data:
md5 = form_data['md5']
else:
md5 = "437b930db84b8079c2dd804a71936b5f"
form_data.update({'md5': md5})
if 'etags[]' not in form_data:
form_data.update({'etags[]': "1,{}".format(md5)})
req_headers = {
'accept': 'application/json',
'authorization': 'Token {}'.format(self.token),
}
req_headers.update(headers)
# send request without the binary data first
return requests.post(url, files=form_data, headers=req_headers)
def _increment_version(self, last_seen_version, modified_utc):
"""
Increments provided version or creates a new one, if not provided.
``last_seen_version`` -- casual version vector value.
It should be encoded as base64(json(value))
``modified_utc`` -- it is used to display modified time in web UI.
"""
dvvset = DVVSet()
if not last_seen_version:
dot = dvvset.create(dvvset.new(modified_utc), self.user_id)
version = b64encode(json.dumps(dot).encode())
else:
# increment version
last_seen_version = json.loads(b64decode(last_seen_version))
context = dvvset.join(last_seen_version)
new_dot = dvvset.update(
dvvset.new_with_history(context, modified_utc),
last_seen_version, self.user_id)
version = dvvset.sync([last_seen_version, new_dot])
version = b64encode(json.dumps(version).encode())
return version
def test_delete_previous_one(self):
"""
Make sure the following objects are removed by the server
- old versions of the same object ( with the same GUID )
- old conflicted copies
"""
# first upload a multi-part file
headers = {'content-range': 'bytes 0-18/28'}
dvvset = DVVSet()
modified_utc = str(int(time.time()))
dot = dvvset.create(dvvset.new(modified_utc), self.user_id)
version = b64encode(json.dumps(dot).encode())
form_data = {
'prefix': '',
'md5': "79b547467286b3e20fad13f73fc1bf78",
'guid': '',
'files[]': ("20180111_165127.jpg", "something something"),
'version': version
}
response = self._upload_request(headers, form_data)
self.assertEqual(response.status_code, 200)
response_json = response.json()
old_guid = response_json['guid']
old_upload_id = response_json['upload_id']
old_md5 = response_json['md5']
url = "{}/riak/upload/{}/{}/2/".format(BASE_URL, TEST_BUCKET_1,
old_upload_id)
headers = {'content-range': 'bytes 19-27/28'}
md5_list = [
"79b547467286b3e20fad13f73fc1bf78",
"437b930db84b8079c2dd804a71936b5f"
]
etags = ",".join(
["{},{}".format(i + 1, md5_list[i]) for i in range(len(md5_list))])
form_data.update({
'files[]': ("20180111_165127.jpg", "something"),
'md5': '437b930db84b8079c2dd804a71936b5f',
'etags[]': etags
})
response = self._upload_request(headers, form_data, url=url)
self.assertEqual(response.status_code, 200)
response_json = response.json()
# make sure object exists, by downloading it and checking its contents
data = self.download_object(
TEST_BUCKET_1, "~object/{}/{}/1_{}".format(old_guid, old_upload_id,
old_md5))
self.assertEqual(data, b'something something')
# upload a new version of file
dot = json.loads(b64decode(response_json['version']))
context = dvvset.join(dot)
new_dot = dvvset.update(
dvvset.new_with_history(context, str(int(time.time()))), dot,
self.user_id)
new_version = dvvset.sync([dot, new_dot])
form_data = {
'prefix': '',
'md5': "79b547467286b3e20fad13f73fc1bf78",
'guid': old_guid,
'files[]': ("20180111_165127.jpg", "something something"),
'version': b64encode(json.dumps(new_version).encode())
}
headers = {'content-range': 'bytes 0-18/28'}
response = self._upload_request(headers, form_data)
self.assertEqual(response.status_code, 200)
response_json = response.json()
upload_id = response_json['upload_id']
url = "{}/riak/upload/{}/{}/2/".format(BASE_URL, TEST_BUCKET_1,
upload_id)
headers = {'content-range': 'bytes 19-27/28'}
md5_list = [
"79b547467286b3e20fad13f73fc1bf78",
"437b930db84b8079c2dd804a71936b5f"
]
etags = ",".join(
["{},{}".format(i + 1, md5_list[i]) for i in range(len(md5_list))])
form_data.update({
'files[]': ("20180111_165127.jpg", "something"),
'md5': '437b930db84b8079c2dd804a71936b5f',
'etags[]': etags
})
response = self._upload_request(headers, form_data, url=url)
self.assertEqual(response.status_code, 200)
# now make sure old object is removed
with self.assertRaises(exceptions.ClientError):
self.download_object(
TEST_BUCKET_1,
"~object/{}/{}/1_{}".format(old_guid, old_upload_id, old_md5))
def test_find_chunk(self):
# first upload multi-part file
headers = {'content-range': 'bytes 0-18/28'}
dvvset = DVVSet()
modified_utc = '1515683246'
dot = dvvset.create(dvvset.new(modified_utc), self.user_id)
version = b64encode(json.dumps(dot).encode())
form_data = {
'prefix': '',
'md5': "79b547467286b3e20fad13f73fc1bf78",
'guid': '',
'files[]': ("20180111_165127.jpg", "something something"),
'version': version
}
response = self._upload_request(headers, form_data)
self.assertEqual(response.status_code, 200)
response_json = response.json()
upload_id = response_json['upload_id']
guid = response_json['guid']
url = "{}/riak/upload/{}/{}/2/".format(BASE_URL, TEST_BUCKET_1,
upload_id)
headers = {'content-range': 'bytes 19-27/28'}
md5_list = [
"79b547467286b3e20fad13f73fc1bf78",
"437b930db84b8079c2dd804a71936b5f"
]
etags = ",".join(
["{},{}".format(i + 1, md5_list[i]) for i in range(len(md5_list))])
form_data.update({
'files[]': ("20180111_165127.jpg", "something"),
'md5': '437b930db84b8079c2dd804a71936b5f',
'etags[]': etags
})
response = self._upload_request(headers, form_data, url=url)
response_json = response.json()
guid = response_json['guid']
self.assertEqual(response.status_code, 200)
self.assertTrue(('orig_name' in response_json))
self.assertEqual(response_json['orig_name'], '20180111_165127.jpg')
# check its contents
contents = self.download_file(TEST_BUCKET_1, '20180111_165127.jpg')
self.assertEquals(contents, b"something somethingsomething")
# now test if two existing chunks are used when parts with the same md5 sums uploaded
dot = json.loads(b64decode(response_json['version']))
context = dvvset.join(dot)
new_dot = dvvset.update(
dvvset.new_with_history(context, str(int(time.time()))), dot,
self.user_id)
new_version = dvvset.sync([dot, new_dot])
headers = {'content-range': 'bytes 0-18/28'}
form_data = {
'prefix': '',
'md5': "79b547467286b3e20fad13f73fc1bf78",
'guid': guid,
'files[]': ("7.jpg", ""),
'version': b64encode(json.dumps(new_version).encode())
}
response = self._upload_request(headers, form_data)
self.assertEqual(response.status_code, 206)
upload_id = response.json()['upload_id']
# make sure object do not exist after empty request
with self.assertRaises(exceptions.ClientError):
self.head(TEST_BUCKET_1, '7.jpg')
url = "{}/riak/upload/{}/{}/2/".format(BASE_URL, TEST_BUCKET_1,
upload_id)
headers = {'content-range': 'bytes 19-27/28'}
md5_list = [
"79b547467286b3e20fad13f73fc1bf78",
"437b930db84b8079c2dd804a71936b5f"
]
etags = ",".join(
["{},{}".format(i + 1, md5_list[i]) for i in range(len(md5_list))])
form_data.update({
'files[]': ("7.jpg", ""),
'md5': '437b930db84b8079c2dd804a71936b5f',
'etags[]': etags
})
response = self._upload_request(headers, form_data, url=url)
self.assertEqual(response.status_code, 206)
# make sure object exists this time
obj_headers = self.head(TEST_BUCKET_1, '7.jpg')
self.assertEqual(obj_headers['upload-id'],
response.json()['upload_id'])
# Another upload
headers = {'content-range': 'bytes 0-18/27'}
form_data = {
'prefix': '',
'md5': "79b547467286b3e20fad13f73fc1bf78",
'guid': guid,
'files[]': ("20180111_165127.jpg", ""),
'version': b64encode(json.dumps(new_version).encode())
}
response = self._upload_request(headers, form_data)
self.assertEqual(response.status_code, 206)
upload_id = response.json()['upload_id']
# the second part is different from what's on server
url = "{}/riak/upload/{}/{}/2/".format(BASE_URL, TEST_BUCKET_1,
upload_id)
headers = {'content-range': 'bytes 19-26/27'}
md5_list = [
"79b547467286b3e20fad13f73fc1bf78",
"25d55ad283aa400af464c76d713c07ad"
]
etags = ",".join(
["{},{}".format(i + 1, md5_list[i]) for i in range(len(md5_list))])
form_data.update({
'files[]': ("20180111_165127.jpg", ""),
'md5': '25d55ad283aa400af464c76d713c07ad',
'etags[]': etags
})
response = self._upload_request(headers, form_data, url=url)
self.assertEqual(response.status_code, 200)
# now finish upload by sending data
form_data.update({
'files[]': ("20180111_165127.jpg", "12345678"),
'md5': '25d55ad283aa400af464c76d713c07ad',
})
response = self._upload_request(headers, form_data, url=url)
self.assertEqual(response.status_code, 200)
response_json = response.json()
self.assertEqual(response_json['orig_name'], '20180111_165127.jpg')
# Another upload. First part is different from what's on server
headers = {'content-range': 'bytes 0-4/27'}
dot = json.loads(b64decode(response_json['version']))
context = dvvset.join(dot)
new_dot = dvvset.update(
dvvset.new_with_history(context, str(int(time.time()))), dot,
self.user_id)
new_version = dvvset.sync([dot, new_dot])
form_data = {
'prefix': '',
'md5': "0ba4439ee9a46d9d9f14c60f88f45f87",
'guid': guid,
'files[]': ("20180111_165127.jpg", ""),
'version': b64encode(json.dumps(new_version).encode())
}
response = self._upload_request(headers, form_data)
self.assertEqual(response.status_code, 200)
form_data.update({'files[]': ("20180111_165127.jpg", "check")})
response = self._upload_request(headers, form_data)
self.assertEqual(response.status_code, 200)
upload_id = response.json()['upload_id']
# the second part is the same as on server
url = "{}/riak/upload/{}/{}/2/".format(BASE_URL, TEST_BUCKET_1,
upload_id)
headers = {'content-range': 'bytes 5-26/27'}
md5_list = [
"0ba4439ee9a46d9d9f14c60f88f45f87",
"79b547467286b3e20fad13f73fc1bf78"
]
etags = ",".join(
["{},{}".format(i + 1, md5_list[i]) for i in range(len(md5_list))])
form_data.update({
'files[]': ("20180111_165127.jpg", ""),
'md5': '79b547467286b3e20fad13f73fc1bf78',
'etags[]': etags
})
response = self._upload_request(headers, form_data, url=url)
self.assertEqual(response.status_code, 206)
self.assertEqual(response.json()['orig_name'], '20180111_165127.jpg')
# check its contents
contents = self.download_file(TEST_BUCKET_1, '20180111_165127.jpg')
self.assertEquals(contents, b"checksomething something")
# check if no existing chunk is used in case when ther'a no matches
form_data = {
'prefix': '',
'md5': "3daae0b62c8032c3c15171e09ef0b8fd",
'guid': '',
'files[]': ("20180111_165127.jpg", ""),
'version': b64encode(json.dumps(new_version).encode())
}
response = self._upload_request(headers, form_data)
self.assertEqual(response.status_code, 200)
response_json = response.json()
self.assertEqual(response_json['guid'], guid)
def test_check_part(self):
headers = {'content-range': 'bytes 0-8/18'}
dvvset = DVVSet()
modified_utc = '1515683246'
dot = dvvset.create(dvvset.new(modified_utc), self.user_id)
version = b64encode(json.dumps(dot).encode())
form_data = {
'prefix': '',
'md5': "437b930db84b8079c2dd804a71936b5f",
'guid': '',
'files[]': ("20180111_165127.jpg", "something"),
'version': version
}
# check_part should work when
# binary data sent, IsCorrectMd5 is False
response = self._upload_request(headers, form_data)
self.assertEqual(response.status_code, 200)
response_json = response.json()
upload_id = response_json['upload_id']
guid = response_json['guid']
# initiate a second upload
headers = {'content-range': 'bytes 0-8/18'}
form_data.update({'files[]': ("SECOND ONE.jpg", "something")})
second_response = self._upload_request(headers, form_data)
second_response_json = second_response.json()
self.assertEqual(second_response.status_code, 200)
second_upload_id = second_response_json['upload_id']
second_guid = second_response_json['guid']
self.assertNotEqual(second_upload_id, upload_id)
self.assertNotEqual(second_guid, guid)
# check_part should fail when IsCorrectMd5 is False
headers = {'content-range': 'bytes 0-8/9'}
form_data.update({'files[]': ("20180111_165127.jpg", "something")})
form_data.update({'md5': '2e059990c316b9e93512937eafa8ef13'})
response = self._upload_request(headers, form_data)
self.assertEqual(response.status_code, 400)
headers = {'content-range': 'bytes 9-17/18'}
md5_list = [
"437b930db84b8079c2dd804a71936b5f",
"437b930db84b8079c2dd804a71936b5f"
]
etags = ",".join(
["{},{}".format(i + 1, md5_list[i]) for i in range(len(md5_list))])
form_data.update({
'md5': '437b930db84b8079c2dd804a71936b5f',
'etags[]': etags
})
# send incorrect upload id and make sure error returned
url = "{}/riak/upload/{}/{}/2/".format(BASE_URL, TEST_BUCKET_1, 'blah')
response = self._upload_request(headers, form_data, url=url)
self.assertEqual(response.status_code, 400)
self.assertEqual(response.json(), {'error': 5})
# send upload id from another upload, make sure server returns error
form_data['guid'] = guid
url = "{}/riak/upload/{}/{}/2/".format(BASE_URL, TEST_BUCKET_1,
second_upload_id)
response = self._upload_request(headers, form_data, url=url)
self.assertEqual(response.status_code, 400)
self.assertEqual(response.json(), {'error': 4})
# create prefix and make sure prefix check works
dir_name = 'test-dir'
dir_response = self.create_pseudo_directory(dir_name)
self.assertEqual(dir_response.status_code, 204)
form_data['prefix'] = dir_name.encode().hex()
form_data['guid'] = second_guid
response = self._upload_request(headers, form_data, url=url)
self.assertEqual(response.status_code, 400)
self.assertEqual(response.json(), {'error': 36})
# test bucket id check
form_data['prefix'] = ''
form_data['guid'] = guid
url = "{}/riak/upload/{}/{}/2/".format(BASE_URL, TEST_BUCKET_2,
second_upload_id)
response = self._upload_request(headers, form_data, url=url)
self.assertEqual(response.status_code, 400)
self.assertEqual(response.json(), {'error': 37})
# empty body, send incorrect upload id and make sure error returned
url = "{}/riak/upload/{}/{}/2/".format(BASE_URL, TEST_BUCKET_1, 'blah')
form_data['files[]'] = ("20180111_165127.jpg", "")
response = self._upload_request(headers, form_data, url=url)
self.assertEqual(response.status_code, 400)
self.assertEqual(response.json(), {'error': 5})
# empty body, send upload id from another upload, make sure server returns error
form_data['guid'] = guid
url = "{}/riak/upload/{}/{}/2/".format(BASE_URL, TEST_BUCKET_1,
second_upload_id)
response = self._upload_request(headers, form_data, url=url)
self.assertEqual(response.status_code, 400)
self.assertEqual(response.json(), {'error': 4})
# empty body, create prefix and make sure prefix check works
form_data['prefix'] = dir_name.encode().hex()
form_data['guid'] = second_guid
response = self._upload_request(headers, form_data, url=url)
self.assertEqual(response.status_code, 400)
self.assertEqual(response.json(), {'error': 36})
# test bucket id check
form_data['files[]'] = ("20180111_165127.jpg", "something")
form_data['prefix'] = ''
form_data['guid'] = guid
url = "{}/riak/upload/{}/{}/2/".format(BASE_URL, TEST_BUCKET_2,
second_upload_id)
response = self._upload_request(headers, form_data, url=url)
self.assertEqual(response.status_code, 400)
self.assertEqual(response.json(), {'error': 37})
# test version check
modified_utc = '1515683247'
dot = dvvset.create(dvvset.new(modified_utc), self.user_id)
form_data['version'] = b64encode(json.dumps(dot).encode())
url = "{}/riak/upload/{}/{}/2/".format(BASE_URL, TEST_BUCKET_1,
second_upload_id)
response = self._upload_request(headers, form_data, url=url)
self.assertEqual(response.status_code, 400)
self.assertEqual(response.json(), {'error': 22})
# finish multipart upload to test if it succeeds
form_data['version'] = version
url = "{}/riak/upload/{}/{}/2/".format(BASE_URL, TEST_BUCKET_1,
upload_id)
response = self._upload_request(headers, form_data, url=url)
self.assertEqual(response.status_code, 200)
def test_get_guid(self):
headers = {'content-range': 'bytes 0-8/9'}
form_data = {'prefix': ''}
dvvset = DVVSet()
# no guid sent, no such object in db -> new GUID created
response = self._upload_request(headers, form_data)
self.assertEqual(response.status_code, 200)
response_json = response.json()
guid = response_json['guid']
upload_id = response_json['upload_id']
md5 = response_json['md5']
self.assertTrue(len(guid) == 36)
data = self.download_object(
TEST_BUCKET_1, "~object/{}/{}/1_{}".format(guid, upload_id, md5))
self.assertEqual(data, b'something')
# no guid sent, object exists -> existing GUID used
# increment version
dot = json.loads(b64decode(response.json()['version']))
context = dvvset.join(dot)
new_dot = dvvset.update(dvvset.new_with_history(context, "1515683247"),
dot, self.user_id)
new_version = dvvset.sync([dot, new_dot])
form_data.update(
{'version': b64encode(json.dumps(new_version).encode())})
response = self._upload_request(headers, form_data)
self.assertEqual(response.status_code, 200)
response_json = response.json()
same_guid = response_json['guid']
self.assertEqual(response_json['orig_name'], '20180111_165127.jpg')
self.assertEqual(guid, same_guid)
# check guid validation function
response = self._upload_request(headers, {'guid': ''})
self.assertEqual(response.status_code, 200)
response_json = response.json()
guid = response_json['guid']
self.assertTrue(len(guid) == 36) # new one should be created
response = self._upload_request(
headers, {'guid': '2418baa6-e39d-5adb-980f-25fc113a1a2d'})
self.assertEqual(response.status_code, 400)
response_json = response.json()
self.assertEqual(response_json, {'error': 42})
response = self._upload_request(
headers, {'guid': '2418baa6-e39d-4adb-c80f-25fc113a1a2d'})
self.assertEqual(response.status_code, 400)
response_json = response.json()
self.assertEqual(response_json, {'error': 42})
response = self._upload_request(
headers, {'guid': '2418baa6-e39d-4adb-880f-25fc113a1a2d'})
self.assertEqual(response.status_code, 200)
response = self._upload_request(
headers, {'guid': '2418baa6-e39d-4adb-980f-25fc113a1a2d'})
self.assertEqual(response.status_code, 200)
response = self._upload_request(
headers, {'guid': '2418baa6-e39d-4adb-a80f-25fc113a1a2d'})
self.assertEqual(response.status_code, 200)
response = self._upload_request(
headers, {'guid': '2418baa6-e39d-4adb-b80f-25fc113a1a2d'})
self.assertEqual(response.status_code, 200)
# guid specified, no object in db -> guid from request should be used
self.purge_test_buckets()
response = self._upload_request(
headers, {'guid': "9f274424-5048-4cb5-9c8c-5b9222e3933e"})
self.assertEqual(response.status_code, 200)
response_json = response.json()
guid = response_json['guid']
self.assertEqual(guid, "9f274424-5048-4cb5-9c8c-5b9222e3933e")
# guid specified, object exists with a different guid -> existing GUID should be used
response = self._upload_request(
headers, {
'guid': "9f274424-5048-4cb5-9c8c-5b9222e39331",
'version': b64encode(json.dumps(new_version).encode())
})
self.assertEqual(response.status_code, 200)
response_json = response.json()
self.assertNotEqual(response_json['guid'],
'9f274424-5048-4cb5-9c8c-5b9222e39331')
# no guid sent, conflict -> existing GUID used
# first increment version
dot = json.loads(b64decode(response.json()['version']))
context = dvvset.join(dot)
new_dot = dvvset.update(dvvset.new_with_history(context, "1515683247"),
dot, self.user_id)
new_version = dvvset.sync([dot, new_dot])
form_data.update(
{'version': b64encode(json.dumps(new_version).encode())})
response = self._upload_request(headers, form_data)
self.assertEqual(response.status_code, 200)
existing_guid = response.json()['guid']
# now create a conflict by uploading previous version
form_data.pop('version')
response = self._upload_request(headers, form_data)
self.assertEqual(response.status_code, 200)
response_json = response.json()
self.assertTrue(response_json['orig_name'].startswith(
'20180111_165127 (Joe Armstrong, conflicted copy '))
self.assertEqual(response_json['guid'], existing_guid)
# existing guid specified, conflict -> existing GUID used
# increment version
dot = json.loads(b64decode(response.json()['version']))
context = dvvset.join(dot)
new_dot = dvvset.update(dvvset.new_with_history(context, "1515683247"),
dot, self.user_id)
new_version = dvvset.sync([dot, new_dot])
response = self._upload_request(
headers, {
'guid': existing_guid,
'version': b64encode(json.dumps(new_version).encode())
})
self.assertEqual(response.status_code, 200)
self.assertEqual(response_json['guid'], existing_guid)
dot = json.loads(b64decode(response.json()['version']))
context = dvvset.join(dot)
new_dot = dvvset.update(dvvset.new_with_history(context, "1515683247"),
dot, self.user_id)
new_version = dvvset.sync([dot, new_dot])
response = self._upload_request(
headers, {
'guid': '34bc4542-af5d-40b0-964c-7b1b25c214c2',
'version': b64encode(json.dumps(new_version).encode())
})
response_json = response.json()
existing_guid = response_json['guid']
conflicted_fn = response_json['orig_name']
self.assertEqual(response.status_code, 200)
self.assertNotEqual('34bc4542-af5d-40b0-964c-7b1b25c214c2',
existing_guid)
# make sure guid do not change in case conflicted copy is edited
dot = json.loads(b64decode(response.json()['version']))
context = dvvset.join(dot)
new_dot = dvvset.update(dvvset.new_with_history(context, "1515683247"),
dot, self.user_id)
new_version = dvvset.sync([dot, new_dot])
response = self._upload_request(
headers, {
'guid': '34bc4542-af5d-40b0-964c-7b1b25c214c2',
'files[]': (conflicted_fn, "something"),
'version': b64encode(json.dumps(new_version).encode())
})
self.assertEqual(response.status_code, 200)
self.assertEqual(response.json()['guid'], existing_guid)
def setUp(self):
self.dvvset = DVVSet()
class TestDVVSet(unittest.TestCase):
def setUp(self):
self.dvvset = DVVSet()
def test_join(self):
A = self.dvvset.new("v1")
A1 = self.dvvset.create(A, "a")
B = self.dvvset.new_with_history(self.dvvset.join(A1), "v2")
B1 = self.dvvset.update(B, A1, "b")
self.assertEqual(self.dvvset.join(A), [])
self.assertEqual(self.dvvset.join(A1), [["a", 1]])
self.assertEqual(self.dvvset.join(B1), [["a", 1], ["b", 1]])
def test_update(self):
A0 = self.dvvset.create(self.dvvset.new("v1"), "a")
A1 = self.dvvset.update(
self.dvvset.new_list_with_history(self.dvvset.join(A0), ["v2"]),
A0, "a")
A2 = self.dvvset.update(
self.dvvset.new_list_with_history(self.dvvset.join(A1), ["v3"]),
A1, "b")
A3 = self.dvvset.update(
self.dvvset.new_list_with_history(self.dvvset.join(A0), ["v4"]),
A1, "b")
A4 = self.dvvset.update(
self.dvvset.new_list_with_history(self.dvvset.join(A0), ["v5"]),
A1, "a")
self.assertEqual(A0, [[["a", 1, ["v1"]]], []])
self.assertEqual(A1, [[["a", 2, ["v2"]]], []])
self.assertEqual(A2, [[["a", 2, []], ["b", 1, ["v3"]]], []])
self.assertEqual(A3, [[["a", 2, ["v2"]], ["b", 1, ["v4"]]], []])
self.assertEqual(A4, [[["a", 3, ["v5", "v2"]]], []])
def test_sync(self):
X = [[["x", 1, []]], []]
A = self.dvvset.create(self.dvvset.new("v1"), "a")
Y = self.dvvset.create(self.dvvset.new_list(["v2"]), "b")
A1 = self.dvvset.create(
self.dvvset.new_list_with_history(self.dvvset.join(A), ["v2"]),
"a")
A3 = self.dvvset.create(
self.dvvset.new_list_with_history(self.dvvset.join(A1), ["v3"]),
"b")
A4 = self.dvvset.create(
self.dvvset.new_list_with_history(self.dvvset.join(A1), ["v3"]),
"c")
W = [[["a", 1, []]], []]
Z = [[["a", 2, ["v2", "v1"]]], []]
self.assertEqual(self.dvvset.sync([W, Z]), [[["a", 2, ["v2"]]], []])
self.assertEqual(self.dvvset.sync([W, Z]), self.dvvset.sync([Z, W]))
self.assertEqual(self.dvvset.sync([A, A1]), self.dvvset.sync([A1, A]))
self.assertEqual(self.dvvset.sync([A4, A3]), self.dvvset.sync([A3,
A4]))
self.assertEqual(self.dvvset.sync(
[A4,
A3]), [[["a", 2, []], ["b", 1, ["v3"]], ["c", 1, ["v3"]]], []])
self.assertEqual(self.dvvset.sync([X, A]),
[[["a", 1, ["v1"]], ["x", 1, []]], []])
self.assertEqual(self.dvvset.sync([X, A]), self.dvvset.sync([A, X]))
self.assertEqual(self.dvvset.sync([X, A]), self.dvvset.sync([A, X]))
self.assertEqual(self.dvvset.sync([A, Y]),
[[["a", 1, ["v1"]], ["b", 1, ["v2"]]], []])
self.assertEqual(self.dvvset.sync([Y, A]), self.dvvset.sync([A, Y]))
self.assertEqual(self.dvvset.sync([Y, A]), self.dvvset.sync([A, Y]))
self.assertEqual(self.dvvset.sync([A, X]), self.dvvset.sync([X, A]))
def test_sync_update(self):
# Mary writes v1 w/o VV
A0 = self.dvvset.create(self.dvvset.new_list(["v1"]), "a")
# Peter reads v1 with version vector (VV)
VV1 = self.dvvset.join(A0)
# Mary writes v2 w/o VV
A1 = self.dvvset.update(self.dvvset.new_list(["v2"]), A0, "a")
# Peter writes v3 with VV from v1
A2 = self.dvvset.update(self.dvvset.new_list_with_history(VV1, ["v3"]),
A1, "a")
self.assertEqual(VV1, [["a", 1]])
self.assertEqual(A0, [[["a", 1, ["v1"]]], []])
self.assertEqual(A1, [[["a", 2, ["v2", "v1"]]], []])
# now A2 should only have v2 and v3, since v3 was causally newer than v1
self.assertEqual(A2, [[["a", 3, ["v3", "v2"]]], []])
def test_event(self):
[A, _] = self.dvvset.create(self.dvvset.new("v1"), "a")
self.assertEqual(self.dvvset.event(A, "a", "v2"),
[["a", 2, ["v2", "v1"]]])
self.assertEqual(self.dvvset.event(A, "b", "v2"),
[["a", 1, ["v1"]], ["b", 1, ["v2"]]])
def test_less(self):
A = self.dvvset.create(self.dvvset.new_list("v1"), ["a"])
B = self.dvvset.create(
self.dvvset.new_list_with_history(self.dvvset.join(A), ["v2"]),
"a")
B2 = self.dvvset.create(
self.dvvset.new_list_with_history(self.dvvset.join(A), ["v2"]),
"b")
B3 = self.dvvset.create(
self.dvvset.new_list_with_history(self.dvvset.join(A), ["v2"]),
"z")
C = self.dvvset.update(
self.dvvset.new_list_with_history(self.dvvset.join(B), ["v3"]), A,
"c")
D = self.dvvset.update(
self.dvvset.new_list_with_history(self.dvvset.join(C), ["v4"]), B2,
"d")
self.assertTrue(self.dvvset.less(A, B))
self.assertTrue(self.dvvset.less(A, C))
self.assertTrue(self.dvvset.less(B, C))
self.assertTrue(self.dvvset.less(B, D))
self.assertTrue(self.dvvset.less(B2, D))
self.assertTrue(self.dvvset.less(A, D))
self.assertFalse(self.dvvset.less(B2, C))
self.assertFalse(self.dvvset.less(B, B2))
self.assertFalse(self.dvvset.less(B2, B))
self.assertFalse(self.dvvset.less(A, A))
self.assertFalse(self.dvvset.less(C, C))
self.assertFalse(self.dvvset.less(D, B2))
self.assertFalse(self.dvvset.less(B3, D))
def test_equal(self):
A = Clock([["a", 4, ["v5", "v0"]], ["b", 0, []], ["c", 1, ["v3"]]],
["v0"])
B = Clock([["a", 4, ["v555", "v0"]], ["b", 0, []], ["c", 1, ["v3"]]],
[])
C = Clock([["a", 4, ["v5", "v0"]], ["b", 0, []]], ["v6", "v1"])
# compare only the causal history
self.assertTrue(self.dvvset.equal(A, B))
self.assertTrue(self.dvvset.equal(B, A))
self.assertFalse(self.dvvset.equal(A, C))
self.assertFalse(self.dvvset.equal(B, C))
def test_size(self):
self.assertEqual(1, self.dvvset.size(self.dvvset.new_list(["v1"]))),
clock = Clock([["a", 4, ["v5", "v0"]], ["b", 0, []], ["c", 1, ["v3"]]],
["v4", "v1"])
self.assertEqual(5, self.dvvset.size(clock))
def test_values(self):
A = [[["a", 4, ["v0", "v5"]], ["b", 0, []], ["c", 1, ["v3"]]], ["v1"]]
B = [[["a", 4, ["v0", "v555"]], ["b", 0, []], ["c", 1, ["v3"]]], []]
C = [[["a", 4, []], ["b", 0, []]], ["v1", "v6"]]
self.assertEqual(self.dvvset.ids(A), ["a", "b", "c"])
self.assertEqual(self.dvvset.ids(B), ["a", "b", "c"])
self.assertEqual(self.dvvset.ids(C), ["a", "b"])
self.assertEqual(sorted(self.dvvset.values(A)),
["v0", "v1", "v3", "v5"])
self.assertEqual(sorted(self.dvvset.values(B)), ["v0", "v3", "v555"])
self.assertEqual(sorted(self.dvvset.values(C)), ["v1", "v6"])
def test_ids_values(self):
A = [[["a", 4, ["v0", "v5"]], ["b", 0, []], ["c", 1, ["v3"]]], ["v1"]]
B = [[["a", 4, ["v0", "v555"]], ["b", 0, []], ["c", 1, ["v3"]]], []]
C = [[["a", 4, []], ["b", 0, []]], ["v1", "v6"]]
self.assertEqual(self.dvvset.ids(A), ["a", "b", "c"])
self.assertEqual(self.dvvset.ids(B), ["a", "b", "c"])
self.assertEqual(self.dvvset.ids(C), ["a", "b"])
self.assertEqual(sorted(self.dvvset.values(A)),
["v0", "v1", "v3", "v5"])
self.assertEqual(sorted(self.dvvset.values(B)), ["v0", "v3", "v555"])
self.assertEqual(sorted(self.dvvset.values(C)), ["v1", "v6"])
def upload_file(self,
url,
fn,
prefix='',
guid='',
last_seen_version=None,
form_data=None,
**kwargs):
"""
Uploads file to server by splitting it to chunks and testing if server
has chunk already, before actual upload.
``url`` -- The base upload API endpoint
``fn`` -- filename
``prefix`` -- an object's prefix on server
``guid`` -- unique identifier ( UUID4 ) for tracking history of changes
``last_seen_version`` -- casual history value, generated by DVVSet()
"""
data = {}
stat = os.stat(fn)
modified_utc = str(int(stat.st_mtime))
size = stat.st_size
if not last_seen_version:
dvvset = DVVSet()
dot = dvvset.create(dvvset.new(modified_utc), self.user_id)
version = b64encode(json.dumps(dot).encode())
else:
# increment version
context = dvvset.join(last_seen_version)
new_dot = dvvset.update(
dvvset.new_with_history(context, modified_utc), dot,
self.user_id)
version = dvvset.sync([last_seen_version, new_dot])
version = b64encode(json.dumps(version)).encode()
result = None
with open(fn, 'rb') as fd:
_read_chunk = lambda: fd.read(FILE_UPLOAD_CHUNK_SIZE)
part_num = 1
md5_list = []
upload_id = None
offset = 0
for chunk in iter(_read_chunk, ''):
md5 = hashlib.md5(chunk)
md5_digest = md5.hexdigest()
md5_list.append(md5_digest)
multipart_form_data = {
'files[]': (fn, ''),
'md5': md5_digest,
'prefix': prefix,
'guid': guid,
'version': version
}
chunk_size = len(chunk)
if form_data:
multipart_form_data.update(form_data)
if size > FILE_UPLOAD_CHUNK_SIZE:
offset = (part_num - 1) * FILE_UPLOAD_CHUNK_SIZE
limit = offset + chunk_size - 1
if limit < 0:
limit = 0
ct_range = "bytes {}-{}/{}".format(offset, limit, size)
else:
ct_range = "bytes 0-{}/{}".format(size - 1, size)
headers = {
'accept': 'application/json',
'authorization': 'Token {}'.format(self.token),
'content-range': ct_range
}
if part_num == 1:
r_url = url
else:
r_url = "{}{}/{}/".format(url, upload_id, part_num)
if offset + chunk_size == size:
# last chunk
etags = ",".join([
"{},{}".format(i + 1, md5_list[i])
for i in range(len(md5_list))
])
multipart_form_data.update({'etags[]': etags})
# send request without binary data first
response = requests.post(r_url,
files=multipart_form_data,
headers=headers)
if response.status_code == 206:
# skip chunk upload, as server has it aleady
response_json = response.json()
upload_id = response_json['upload_id']
guid = response_json['guid']
part_num += 1
if offset + chunk_size == size:
result = response_json
break
else:
continue
self.assertEqual(response.status_code, 200)
response_json = response.json()
upload_id = response_json['upload_id']
guid = response_json['guid'] # server could change GUID
server_md5 = response_json['md5']
self.assertEqual(md5_digest, server_md5)
# upload an actual data now
multipart_form_data.update({
'files[]': (fn, chunk),
'guid': guid # GUID could change
})
response = requests.post(r_url,
files=multipart_form_data,
headers=headers)
self.assertEqual(response.status_code, 200)
response_json = response.json()
if offset + chunk_size == size:
# the last chunk has been processed, expect complete_upload response
expected = set([
'lock_user_tel', 'lock_user_name', 'guid', 'upload_id',
'lock_modified_utc', 'lock_user_id', 'is_locked',
'author_tel', 'is_deleted', 'upload_time', 'md5',
'version', 'height', 'author_id', 'author_name',
'object_key', 'bytes', 'width', 'orig_name', 'end_byte'
])
self.assertEqual(expected, set(response_json.keys()))
result = response_json
break
else:
self.assertEqual(
set([
'end_byte', 'upload_id', 'guid', 'upload_id', 'md5'
]), set(response_json.keys()))
#self.assertEqual(response_json['guid'], guid)
#self.assertEqual(response_json['upload_id'], upload_id)
server_md5 = response_json['md5']
self.assertEqual(md5_digest, server_md5)
upload_id = response_json['upload_id']
part_num += 1
return result