def main():
try:
repo = nog.createRepo('test-resolveimgsrc-optimization')
except RuntimeError:
repo = nog.openRepo('test-resolveimgsrc-optimization')
master = repo.getMaster()
root = nog.Tree()
root.name = 'root'
lines = [
'# {} small squares'.format(N),
'',
]
images = nog.Tree()
images.name = 'images'
for i in range(N):
img = nog.Object()
name = 'img-{}.png'.format(i)
img.name = name
img.blob = '/tmp/test.png'
images.append(img)
lines.append('<img src="./images/{}" width="20">'.format(name))
md = nog.Object()
md.name = 'images.md'
md.text = '\n'.join(lines)
root.append(md)
root.append(images)
repo.commitTree(subject='images', tree=root, parent=master.sha1)
def makeResultTree(treeName):
res = nog.Tree()
res.name = treeName
report = nog.Object()
report.name = 'report.md'
res.append(report)
lines = [
'# {}: {} images'.format(treeName, nImages),
'',
]
images = nog.Tree()
images.name = 'images'
res.append(images)
for i in range(nImages):
img = nog.Object()
name = '{}-img-{}.png'.format(treeName, i)
img.name = name
img.blob = '/tmp/test.png'
images.append(img)
lines.append('<img src="./images/{}" width="20">'.format(name))
report.text = '\n'.join(lines)
return res
def main():
for r in range(nRepos):
repoName = '{0}{1}'.format(repoBaseName, r)
print('Creating repo {0}'.format(repoName))
try:
repo = nog.createRepo(repoName)
except RuntimeError:
repo = nog.openRepo(repoName)
master = repo.getMaster()
root = nog.Tree()
root.name = 'root'
root.meta['files'] = {}
for t in range(nTreesPerRepo):
treeName = 'Tree{0}'.format(t)
tree = nog.Tree()
tree.name = treeName
for i in range(nEntriesPerTree):
entry = nog.Object()
entry.name = 'entry-{}'.format(i)
entry.meta['entryNum'] = i
entry.meta['entryName'] = entry.name
entry.meta['repoName'] = repoName
entry.meta['treeName'] = treeName
tree.append(entry)
root.append(tree)
repo.commitTree(subject='images', tree=root, parent=master.sha1)
def test_empty_blob():
obj = nog.Object()
obj.blob = b''
assert obj.blob == emptyBlobSha1
sha1 = remote.postObject(obj)
obj = remote.getObject(sha1)
assert obj.blob == emptyBlobSha1
def test_blob_from_sha1():
obj = nog.Object()
obj.blob = blobSha1
assert obj.blob == blobSha1
sha1 = remote.postObject(obj)
obj = remote.getObject(sha1)
assert obj.blob == blobSha1
def jobIndexDoc(self):
doc = Document('index.md')
name = os.path.basename(self.params['nog']['program']['name'])
doc.insertLink('Go to result', '../../results/' + name)
obj = nog.Object()
obj.name = doc.name
obj.meta['content'] = doc.content
return obj
def completeJob(job):
job.meta['job']['status'] = 'completed'
log = nog.Object()
log.name = 'log'
log.meta['description'] = 'job execution log'
log.meta['content'] = xout.getvalue()
job.insert(0, log)
return job
def test_reject_invalid_idv1_obj():
obj = nog.Object()
obj.name = 'foo'
obj.meta['content'] = 'bar'
with pytest.raises(RuntimeError) as errinfo:
remote.postObject(obj)
assert ('Invalid Object' in str(errinfo.value))
assert ('foo' in str(errinfo.value))
assert ('idversion 1' in str(errinfo.value))
def test_blob_random_bytes_to_force_upload():
obj = nog.Object()
buf = os.urandom(20)
bufSha1 = sha1(buf)
obj.blob = buf
assert obj.blob == bufSha1
oid = remote.postObject(obj)
obj = remote.getObject(oid)
assert obj.blob == bufSha1
def test_post_buf_size_flush():
saved = nog.POST_BUFFER_SIZE
savedLimit = nog.POST_BUFFER_SIZE_LIMIT
nog.POST_BUFFER_SIZE = 1000
nog.POST_BUFFER_SIZE_LIMIT = 100000
obj = nog.Object()
obj.text = 'a' * 100 * 100
remote.postObject(obj)
nog.POST_BUFFER_SIZE = saved
nog.POST_BUFFER_SIZE_LIMIT = savedLimit
def progIndexDoc(self):
doc = Document('index.md')
doc.insertText(
'**<span style="color:red">Warning: </span>** The current'
' results are created by a non-published version.')
obj = nog.Object()
obj.name = doc.name
obj.meta['content'] = doc.content
return obj
def test_post_buf_size_limit():
saved = nog.POST_BUFFER_SIZE
savedLimit = nog.POST_BUFFER_SIZE_LIMIT
nog.POST_BUFFER_SIZE = 1000
nog.POST_BUFFER_SIZE_LIMIT = 100000
obj = nog.Object()
obj.text = 'a' * 1000 * 1000
with pytest.raises(RuntimeError):
remote.postObject(obj)
nog.POST_BUFFER_SIZE = saved
nog.POST_BUFFER_SIZE_LIMIT = savedLimit
def addLogEntry(log, description, content, auto=None):
auto = auto or True
if auto:
creation = 'automatic'
else:
creation = 'human'
date = datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ')
entry = nog.Object()
entry.name = date + '.md'
entry.meta['logentry'] = {'date': date, 'creation': creation}
entry.meta['description'] = description
entry.meta['content'] = content
log.insert(0, entry)
def addContentsOfDirToTree(dirpath, tree):
for entry in os.listdir(dirpath):
entrypath = os.path.join(dirpath, entry)
if os.path.isfile(entrypath):
obj = nog.Object()
obj.name = entry
obj.blob = entrypath
tree.append(obj)
elif os.path.isdir(entrypath):
subtree = nog.Tree()
subtree.name = entry
tree.append(subtree)
addContentsOfDirToTree(entrypath, subtree)
def test_Tree_collapse():
tree = nog.Tree()
obj = nog.Object()
tree.append(obj)
obj.name = 'foo'
assert next(tree.objects()).name == 'foo'
# Post tree and collapse entries, which detaches children.
remote.postTree(tree)
tree.collapse()
obj.name = 'bar'
assert next(tree.objects()).name == 'foo'
def test_blob_setter():
obj = nog.Object()
assert obj.idversion == 1
obj.blob = None
assert obj.blob is None
obj.blob = NULL_SHA1
assert obj.blob is None
obj.format(0)
assert obj.idversion == 0
obj.blob = None
assert obj.blob == NULL_SHA1
obj.blob = NULL_SHA1
assert obj.blob == NULL_SHA1
def test_fundamental_Object_operations():
obj_v1 = nog.Object()
obj_v1.name = 'foo'
obj_v1.text = 'text'
assert obj_v1.content == objContent_v1
assert obj_v1.sha1 == objId_v1
obj_v0 = deepcopy(obj_v1)
obj_v0.format(0)
assert obj_v0.content == objContent_v0
assert obj_v0.sha1 == objId_v0
assert remote.postObject(obj_v0) == objId_v0
assert remote.postObject(obj_v1) == objId_v1
obj = remote.getObject(objId_v0)
assert obj.idversion == 0
assert obj.sha1 == objId_v0
assert obj.content == objContent_v0
assert obj.text == 'text'
assert obj.meta == {'content': 'text'}
assert obj.blob == NULL_SHA1
obj = remote.getObject(objId_v1)
assert obj.idversion == 1
assert obj.sha1 == objId_v1
assert obj.content == objContent_v1
assert obj.text == 'text'
assert obj.meta == {}
assert obj.blob is None
obj.format(0)
assert obj.sha1 == objId_v0
assert nog.Object(content=objContent_v0).sha1 == objId_v0
assert nog.Object(content=objContent_v1).sha1 == objId_v1
def test_object_with_special_character():
master = remote.getMaster()
root = master.tree
oname = u'BlaBlub-\u00fc\u00e4\u00f6'
tname = u'Wau-Wau-\u72d7'
sname = u'Blafasel-\u0641\u0644\u0627\u0641\u0644'
obj = nog.Object()
obj.name = oname
tr = nog.Tree()
tr.name = tname
tr.append(obj)
root.append(tr)
remote.commitTree(subject=sname, tree=root, parent=master.sha1)
new_master = remote.getMaster()
new_root = new_master.tree
new_tr = next(new_root.trees(tname))
new_obj = next(new_tr.objects(oname))
assert new_obj.name == oname
assert new_tr.name == tname
assert new_master.subject == sname
def test_blob_from_file(tmpdir):
inpath = tmpdir.join('a.dat')
inpath.write(blobBytes)
obj = nog.Object()
obj.blob = str(inpath)
assert obj.blob == blobSha1
sha1 = remote.postObject(obj)
obj = remote.getObject(sha1)
assert obj.blob == blobSha1
with obj.openBlob() as fp:
assert fp.read() == blobBytes
cppath = tmpdir.join('cp.dat')
obj.copyBlob(str(cppath))
with cppath.open('rb') as fp:
assert fp.read() == blobBytes
lnpath = tmpdir.join('ln.dat')
obj.linkBlob(str(lnpath))
with lnpath.open('rb') as fp:
assert fp.read() == blobBytes
def test_default_Object_idversion():
obj = nog.Object()
assert obj.idversion == 1
def main():
try:
repo = nog.createRepo(repoName)
except RuntimeError:
repo = nog.openRepo(repoName)
master = repo.getMaster()
root = nog.Tree()
root.name = 'root'
root.meta['workspace'] = {}
datalist = nog.Tree()
datalist.name = 'datalist'
datalist.meta['datalist'] = {}
for i in range(nImages):
img = nog.Object()
img.name = 'img-{}.png'.format(i)
img.blob = '/tmp/test.png'
datalist.append(img)
root.append(datalist)
programs = nog.Tree()
programs.name = 'programs'
programs.meta['programs'] = {}
root.append(programs)
jobs = nog.Tree()
jobs.name = 'jobs'
jobs.meta['jobs'] = {}
root.append(jobs)
# The result tree can be organized as:
#
# ```
# /results/result-0/report.md
# /results/result-1/report.md
# ...
# ```
#
# or as:
#
# ```
# /results/subresults-0/subresult-0-0/report.md
# /results/subresults-0/subresult-0-1/report.md
# /results/subresults-1/subresult-1-0/report.md
# ...
# ```
#
# Simulate both structures in a single tree.
results = nog.Tree()
results.name = 'results'
results.meta['results'] = {}
root.append(results)
for i in range(nResults):
results.append(makeResultTree('result-{}'.format(i)))
subresults = nog.Tree()
subresults.name = 'subresults-{}'.format(i)
results.append(subresults)
for j in range(nResults):
r = makeResultTree('subresult-{}-{}'.format(i, j))
for k in range(nRepeatResults):
subresults.append(r)
repo.commitTree(subject='images', tree=root, parent=master.sha1)
def test_fundamental_Tree_operations():
tree = nog.Tree()
tree.name = 'tree'
tree.meta['foo'] = 'bar'
tree.append(nog.Object(content=objContent_v0))
tree.append(nog.Object(content=objContent_v1))
assert tree.content == treeContent
assert tree.sha1 == treeId
remote.postTree(tree)
tree = remote.getTree(treeId)
assert tree.content == treeContent
assert tree.sha1 == treeId
tree.pop()
tree.pop()
tree.insert(0, nog.Object(content=objContent_v1))
tree.insert(0, nog.Object(content=objContent_v0))
assert tree.content == treeContent
assert tree.sha1 == treeId
tree.pop(0)
tree.insert(0, nog.Object(content=objContent_v0))
assert tree.content == treeContent
assert tree.sha1 == treeId
childTree = deepcopy(tree)
tree.append(childTree)
it = tree.entries()
assert next(it).sha1 == objId_v0
assert next(it).sha1 == objId_v1
assert next(it).sha1 == treeId
with pytest.raises(StopIteration):
next(it)
it = tree.enumerateEntries()
(i, e) = next(it)
assert i == 0
assert e.sha1 == objId_v0
(i, e) = next(it)
assert i == 1
assert e.sha1 == objId_v1
(i, e) = next(it)
assert i == 2
assert e.sha1 == treeId
with pytest.raises(StopIteration):
next(it)
it = tree.objects()
assert next(it).sha1 == objId_v0
assert next(it).sha1 == objId_v1
with pytest.raises(StopIteration):
next(it)
it = tree.enumerateObjects()
(i, e) = next(it)
assert i == 0
assert e.sha1 == objId_v0
(i, e) = next(it)
assert i == 1
assert e.sha1 == objId_v1
with pytest.raises(StopIteration):
next(it)
it = tree.trees()
assert next(it).sha1 == treeId
with pytest.raises(StopIteration):
next(it)
it = tree.enumerateTrees()
(i, e) = next(it)
assert i == 2
assert e.sha1 == treeId
with pytest.raises(StopIteration):
next(it)