def test_GitRepo_files_decorator(): class testclass(object): def __init__(self): self.path = os.path.join('some', 'where') @normalize_paths def decorated(self, files): return files test_instance = testclass() files_to_test = os.path.join(test_instance.path, 'deep', get_most_obscure_supported_name()) assert_equal(test_instance.decorated(files_to_test), [_normalize_path(test_instance.path, files_to_test)]) files_to_test = get_most_obscure_supported_name() assert_equal(test_instance.decorated(files_to_test), [_normalize_path(test_instance.path, files_to_test)]) files_to_test = os.path.join(get_most_obscure_supported_name(), 'beyond', 'obscure') assert_equal(test_instance.decorated(files_to_test), [_normalize_path(test_instance.path, files_to_test)]) files_to_test = os.path.join(os.getcwd(), 'somewhere', 'else', get_most_obscure_supported_name()) assert_raises(FileNotInRepositoryError, test_instance.decorated, files_to_test) files_to_test = ['now', os.path.join('a list', 'of'), 'paths'] expect = [] for item in files_to_test: expect.append(_normalize_path(test_instance.path, item)) assert_equal(test_instance.decorated(files_to_test), expect) assert_raises(ValueError, test_instance.decorated, 1)
def test_gitattributes(path): gr = GitRepo(path, create=True) # starts without any attributes file ok_(not op.exists(op.join(gr.path, '.gitattributes'))) eq_(gr.get_gitattributes('.')['.'], {}) # bool is a tag or unsets, anything else is key/value gr.set_gitattributes([('*', {'tag': True}), ('*', {'sec.key': 'val'})]) ok_(op.exists(op.join(gr.path, '.gitattributes'))) eq_(gr.get_gitattributes('.')['.'], {'tag': True, 'sec.key': 'val'}) # unset by amending the record, but does not remove notion of the # tag entirely gr.set_gitattributes([('*', {'tag': False})]) eq_(gr.get_gitattributes('.')['.'], {'tag': False, 'sec.key': 'val'}) # attributes file is not added or commited, we can ignore such # attributes eq_(gr.get_gitattributes('.', index_only=True)['.'], {}) # we can send absolute path patterns and write to any file, and # the patterns will be translated relative to the target file gr.set_gitattributes([(op.join(gr.path, 'relative', 'ikethemike/**'), { 'bang': True })], attrfile=op.join('relative', '.gitattributes')) # directory and file get created ok_(op.exists(op.join(gr.path, 'relative', '.gitattributes'))) eq_( gr.get_gitattributes( op.join(gr.path, 'relative', 'ikethemike', 'probe')), # always comes out relative to the repo root, even if abs goes in { op.join('relative', 'ikethemike', 'probe'): { 'tag': False, 'sec.key': 'val', 'bang': True } }) if get_encoding_info()['default'] != 'ascii': # do not perform this on obscure systems without anything like UTF # it is not relevant whether a path actually exists, and paths # with spaces and other funky stuff are just fine funky = u'{} {}'.format(get_most_obscure_supported_name(), get_most_obscure_supported_name()) gr.set_gitattributes([(funky, {'this': 'that'})]) eq_( gr.get_gitattributes(funky)[funky], { 'this': 'that', 'tag': False, 'sec.key': 'val', }) # mode='w' should replace the entire file: gr.set_gitattributes([('**', {'some': 'nonsense'})], mode='w') eq_(gr.get_gitattributes('.')['.'], {'some': 'nonsense'})
def test_ssh_copy(sourcedir, sourcefile1, sourcefile2): port = get_ssh_port('datalad-test') remote_url = 'ssh://datalad-test:{}'.format(port) manager = SSHManager() ssh = manager.get_connection(remote_url) # write to obscurely named file in sourcedir obscure_file = opj(sourcedir, get_most_obscure_supported_name()) with open(obscure_file, 'w') as f: f.write("three") # copy tempfile list to remote_url:sourcedir sourcefiles = [sourcefile1, sourcefile2, obscure_file] ssh.put(sourcefiles, opj(remote_url, sourcedir)) # docs promise that connection is auto-opened in case of multiplex if _ssh_manager_is_multiplex: ok_(ssh.is_open()) # recursive copy tempdir to remote_url:targetdir targetdir = sourcedir + '.c opy' ssh.put(sourcedir, opj(remote_url, targetdir), recursive=True, preserve_attrs=True) # check if sourcedir copied to remote_url:targetdir ok_(isdir(targetdir)) # check if scp preserved source directory attributes # if source_mtime=1.12s, scp -p sets target_mtime = 1.0s, test that eq_(getmtime(targetdir), int(getmtime(sourcedir)) + 0.0) # check if targetfiles(and its content) exist in remote_url:targetdir, # this implies file(s) and recursive directory copying pass for targetfile, content in zip(sourcefiles, ["one", "two", "three"]): targetpath = opj(targetdir, targetfile) ok_(exists(targetpath)) with open(targetpath, 'r') as fp: eq_(content, fp.read()) # and now a quick smoke test for get # but simplify the most obscure filename slightly to not trip `scp` itself togetfile = Path(targetdir) / ( get_most_obscure_supported_name().replace('`', '') + '2') togetfile.write_text(str('something')) ssh.get(opj(remote_url, str(togetfile)), sourcedir) ok_((Path(sourcedir) / togetfile.name).exists()) ssh.close()
def test_GitRepo_add(src, path): gr = GitRepo.clone(src, path) filename = get_most_obscure_supported_name() with open(op.join(path, filename), 'w') as f: f.write("File to add to git") added = gr.add(filename) eq_(added, {'success': True, 'file': filename}) assert_in(filename, gr.get_indexed_files(), "%s not successfully added to %s" % (filename, path)) # uncommitted: ok_(gr.dirty) filename = "another.txt" with open(op.join(path, filename), 'w') as f: f.write("Another file to add to git") # include committing: added2 = gr.add(filename) gr.commit(msg="Add two files.") eq_(added2, {'success': True, 'file': filename}) assert_in(filename, gr.get_indexed_files(), "%s not successfully added to %s" % (filename, path)) ok_clean_git(path)
def test_get_most_obscure_supported_name(): n = get_most_obscure_supported_name() if platform.system() in ('Linux', 'Darwin'): eq_(n, OBSCURE_PREFIX + OBSCURE_FILENAMES[1]) else: # ATM no one else is as good ok_(n in OBSCURE_PREFIX + OBSCURE_FILENAMES[2:])
def test_GitRepo_fetch(test_path, orig_path, clone_path): origin = GitRepo.clone(test_path, orig_path) clone = GitRepo.clone(orig_path, clone_path) filename = get_most_obscure_supported_name() origin.checkout("new_branch", ['-b']) with open(op.join(orig_path, filename), 'w') as f: f.write("New file.") origin.add(filename) origin.commit("new file added.") fetched = clone.fetch(remote='origin') # test FetchInfo list returned by fetch eq_([u'origin/' + clone.get_active_branch(), u'origin/new_branch'], [commit.name for commit in fetched]) ok_clean_git(clone.path, annex=False) assert_in("origin/new_branch", clone.get_remote_branches()) assert_in(filename, clone.get_files("origin/new_branch")) assert_false(op.exists(op.join(clone_path, filename))) # not checked out # create a remote without an URL: origin.add_remote('not-available', 'git://example.com/not/existing') origin.config.unset('remote.not-available.url', where='local') # fetch without provided URL fetched = origin.fetch('not-available') # nothing was done, nothing returned: eq_([], fetched)
def test_GitRepo_add(src, path): gr = GitRepo.clone(src, path) filename = get_most_obscure_supported_name() with open(op.join(path, filename), 'w') as f: f.write("File to add to git") added = gr.add(filename) eq_(added, {'success': True, 'file': filename}) assert_in(filename, gr.get_indexed_files(), "%s not successfully added to %s" % (filename, path)) # uncommitted: ok_(gr.dirty) filename = "another.txt" with open(op.join(path, filename), 'w') as f: f.write("Another file to add to git") assert_raises(AssertionError, gr.add, filename, git=False) assert_raises(AssertionError, gr.add, filename, git=None) # include committing: added2 = gr.add(filename) gr.commit(msg="Add two files.") eq_(added2, {'success': True, 'file': filename}) assert_in(filename, gr.get_indexed_files(), "%s not successfully added to %s" % (filename, path)) ok_clean_git(path)
def test_get_most_obscure_supported_name(): n = get_most_obscure_supported_name() ok_startswith(n, OBSCURE_PREFIX) ok_(len(OBSCURE_FILENAMES) > 1) # from more complex to simpler ones ok_(len(OBSCURE_FILENAMES[0]) > len(OBSCURE_FILENAMES[-1])) print(repr(n))
def test_GitRepo_pull(test_path, orig_path, clone_path): origin = GitRepo.clone(test_path, orig_path) clone = GitRepo.clone(orig_path, clone_path) filename = get_most_obscure_supported_name() with open(op.join(orig_path, filename), 'w') as f: f.write("New file.") origin.add(filename) origin.commit("new file added.") clone.pull() ok_(op.exists(op.join(clone_path, filename))) # While at it, let's test _get_remotes_having_commit a bit clone.add_remote("very_origin", test_path) clone.fetch("very_origin") eq_( clone._get_remotes_having_commit(clone.get_hexsha()), ['origin'] ) prev_commit = clone.get_hexsha('HEAD^') eq_( set(clone._get_remotes_having_commit(prev_commit)), {'origin', 'very_origin'} )
def test_gitattributes(path): gr = GitRepo(path, create=True) # starts without any attributes file ok_(not op.exists(op.join(gr.path, '.gitattributes'))) eq_(gr.get_gitattributes('.')['.'], {}) # bool is a tag or unsets, anything else is key/value gr.set_gitattributes([('*', {'tag': True}), ('*', {'sec.key': 'val'})]) ok_(op.exists(op.join(gr.path, '.gitattributes'))) eq_(gr.get_gitattributes('.')['.'], {'tag': True, 'sec.key': 'val'}) # unset by amending the record, but does not remove notion of the # tag entirely gr.set_gitattributes([('*', {'tag': False})]) eq_(gr.get_gitattributes('.')['.'], {'tag': False, 'sec.key': 'val'}) # attributes file is not added or commited, we can ignore such # attributes eq_(gr.get_gitattributes('.', index_only=True)['.'], {}) # we can send absolute path patterns and write to any file, and # the patterns will be translated relative to the target file gr.set_gitattributes([ (op.join(gr.path, 'relative', 'ikethemike/**'), {'bang': True})], attrfile=op.join('relative', '.gitattributes')) # directory and file get created ok_(op.exists(op.join(gr.path, 'relative', '.gitattributes'))) eq_(gr.get_gitattributes( op.join(gr.path, 'relative', 'ikethemike', 'probe')), # always comes out relative to the repo root, even if abs goes in {op.join('relative', 'ikethemike', 'probe'): {'tag': False, 'sec.key': 'val', 'bang': True}}) if get_encoding_info()['default'] != 'ascii': # do not perform this on obscure systems without anything like UTF # it is not relevant whether a path actually exists, and paths # with spaces and other funky stuff are just fine funky = u'{} {}'.format( get_most_obscure_supported_name(), get_most_obscure_supported_name()) gr.set_gitattributes([(funky, {'this': 'that'})]) eq_(gr.get_gitattributes(funky)[funky], { 'this': 'that', 'tag': False, 'sec.key': 'val', }) # mode='w' should replace the entire file: gr.set_gitattributes([('**', {'some': 'nonsense'})], mode='w') eq_(gr.get_gitattributes('.')['.'], {'some': 'nonsense'})
def test_GitRepo_files_decorator(): class testclass(object): def __init__(self): self.path = op.join('some', 'where') # TODO # yoh: logic is alien to me below why to have two since both look identical! @normalize_paths def decorated_many(self, files): return files @normalize_paths def decorated_one(self, file_): return file_ test_instance = testclass() # When a single file passed -- single path returned obscure_filename = get_most_obscure_supported_name() file_to_test = op.join(test_instance.path, 'deep', obscure_filename) # file doesn't exist eq_(test_instance.decorated_one(file_to_test), _normalize_path(test_instance.path, file_to_test)) eq_(test_instance.decorated_one(file_to_test), _normalize_path(test_instance.path, file_to_test)) file_to_test = obscure_filename eq_(test_instance.decorated_many(file_to_test), _normalize_path(test_instance.path, file_to_test)) eq_(test_instance.decorated_one(file_to_test), _normalize_path(test_instance.path, file_to_test)) file_to_test = op.join(obscure_filename, 'beyond', 'obscure') eq_(test_instance.decorated_many(file_to_test), _normalize_path(test_instance.path, file_to_test)) file_to_test = op.join(getpwd(), 'somewhere', 'else', obscure_filename) assert_raises(FileNotInRepositoryError, test_instance.decorated_many, file_to_test) # If a list passed -- list returned files_to_test = ['now', op.join('a list', 'of'), 'paths'] expect = [] for item in files_to_test: expect.append(_normalize_path(test_instance.path, item)) eq_(test_instance.decorated_many(files_to_test), expect) eq_(test_instance.decorated_many(''), []) assert_raises(ValueError, test_instance.decorated_many, 1) assert_raises(ValueError, test_instance.decorated_one, 1)
def test_serve_path_via_http(): for test_fpath in ['test1.txt', 'test_dir/test2.txt', 'test_dir/d2/d3/test3.txt', 'file with space test4', u'Джэйсон', get_most_obscure_supported_name(), ]: yield _test_serve_path_via_http, test_fpath # just with the last one check that we did remove proxy setting with patch.dict('os.environ', {'http_proxy': 'http://127.0.0.1:9/'}): yield _test_serve_path_via_http, test_fpath
def test_url_fragments_and_query(): url = URL(hostname="host", query=OrderedDict((('a', 'x/b'), ('b', 'y')))) eq_(str(url), '//host?a=x%2Fb&b=y') eq_(url.query, 'a=x%2Fb&b=y') eq_(url.query_dict, {'a': 'x/b', 'b': 'y'}) url = URL(hostname="host", fragment=OrderedDict((('b', 'x/b'), ('a', 'y')))) eq_(str(url), '//host#b=x/b&a=y') eq_(url.fragment, 'b=x/b&a=y') eq_(url.fragment_dict, {'a': 'y', 'b': 'x/b'}) fname = get_most_obscure_supported_name() url = URL(hostname="host", fragment={'a': fname}) eq_(url.fragment_dict, {'a': fname})
def test_GitRepo_push_n_checkout(orig_path, clone_path): origin = GitRepo(orig_path) clone = GitRepo.clone(orig_path, clone_path) filename = get_most_obscure_supported_name() with open(op.join(clone_path, filename), 'w') as f: f.write("New file.") clone.add(filename) clone.commit("new file added.") # TODO: need checkout first: clone.push('origin', '+master:new-branch') origin.checkout('new-branch') ok_(op.exists(op.join(orig_path, filename)))
def test_serve_path_via_http(): for test_fpath in [ 'test1.txt', Path('test_dir', 'test2.txt'), Path('test_dir', 'd2', 'd3', 'test3.txt'), 'file with space test4', u'Джэйсон', get_most_obscure_supported_name(), ]: yield _test_serve_path_via_http, test_fpath, False, None yield _test_serve_path_via_http, test_fpath, True, None yield _test_serve_path_via_http, test_fpath, False, ('ernie', 'bert') # just with the last one check that we did remove proxy setting with patch.dict('os.environ', {'http_proxy': 'http://127.0.0.1:9/'}): yield _test_serve_path_via_http, test_fpath, False, None
def test_ssh_copy(sourcedir, sourcefile1, sourcefile2): remote_url = 'ssh://localhost:22' manager = SSHManager() ssh = manager.get_connection(remote_url) # write to obscurely named file in sourcedir obscure_file = opj(sourcedir, get_most_obscure_supported_name()) with open(obscure_file, 'w') as f: f.write("three") # copy tempfile list to remote_url:sourcedir sourcefiles = [sourcefile1, sourcefile2, obscure_file] ssh.put(sourcefiles, opj(remote_url, sourcedir)) # docs promise that connection is auto-opened ok_(ssh.is_open()) # recursive copy tempdir to remote_url:targetdir targetdir = sourcedir + '.c opy' ssh.put(sourcedir, opj(remote_url, targetdir), recursive=True, preserve_attrs=True) # check if sourcedir copied to remote_url:targetdir ok_(isdir(targetdir)) # check if scp preserved source directory attributes # if source_mtime=1.12s, scp -p sets target_mtime = 1.0s, test that eq_(getmtime(targetdir), int(getmtime(sourcedir)) + 0.0) # check if targetfiles(and its content) exist in remote_url:targetdir, # this implies file(s) and recursive directory copying pass for targetfile, content in zip(sourcefiles, ["one", "two", "three"]): targetpath = opj(targetdir, targetfile) ok_(exists(targetpath)) with open(targetpath, 'r') as fp: eq_(content, fp.read()) # and now a quick smoke test for get togetfile = Path(targetdir) / '2|g>e"t.t&x;t' togetfile.write_text(str('something')) ssh.get(opj(remote_url, str(togetfile)), sourcedir) ok_((Path(sourcedir) / '2|g>e"t.t&x;t').exists()) ssh.close()
def test_ssh_copy(sourcedir, sourcefile1, sourcefile2): remote_url = 'ssh://localhost:22' manager = SSHManager() ssh = manager.get_connection(remote_url) # write to obscurely named file in sourcedir obscure_file = opj(sourcedir, get_most_obscure_supported_name()) with open(obscure_file, 'w') as f: f.write("three") # copy tempfile list to remote_url:sourcedir sourcefiles = [sourcefile1, sourcefile2, obscure_file] ssh.put(sourcefiles, opj(remote_url, sourcedir)) # docs promise that connection is auto-opened ok_(ssh.is_open()) # recursive copy tempdir to remote_url:targetdir targetdir = sourcedir + '.c opy' ssh.put(sourcedir, opj(remote_url, targetdir), recursive=True, preserve_attrs=True) # check if sourcedir copied to remote_url:targetdir ok_(isdir(targetdir)) # check if scp preserved source directory attributes # if source_mtime=1.12s, scp -p sets target_mtime = 1.0s, test that eq_(getmtime(targetdir), int(getmtime(sourcedir)) + 0.0) # check if targetfiles(and its content) exist in remote_url:targetdir, # this implies file(s) and recursive directory copying pass for targetfile, content in zip(sourcefiles, ["one", "two", "three"]): targetpath = opj(targetdir, targetfile) ok_(exists(targetpath)) with open(targetpath, 'r') as fp: eq_(content, fp.read()) # and now a quick smoke test for get togetfile = Path(targetdir) / '2|g>e"t.t&x;t' togetfile.write_text(text_type('something')) ssh.get(opj(remote_url, text_type(togetfile)), sourcedir) ok_((Path(sourcedir) / '2|g>e"t.t&x;t').exists()) ssh.close()
def test_GitRepo_pull(test_path, orig_path, clone_path): origin = GitRepo.clone(test_path, orig_path) clone = GitRepo.clone(orig_path, clone_path) filename = get_most_obscure_supported_name() with open(op.join(orig_path, filename), 'w') as f: f.write("New file.") origin.add(filename) origin.commit("new file added.") clone.pull() ok_(op.exists(op.join(clone_path, filename))) # While at it, let's test _get_remotes_having_commit a bit clone.add_remote("very_origin", test_path) clone.fetch("very_origin") eq_(clone._get_remotes_having_commit(clone.get_hexsha()), ['origin']) prev_commit = clone.get_hexsha('HEAD^') eq_(set(clone._get_remotes_having_commit(prev_commit)), {'origin', 'very_origin'})
def test_ssh_copy(sourcedir, sourcefile1, sourcefile2): remote_url = 'ssh://localhost' manager = SSHManager() ssh = manager.get_connection(remote_url) ssh.open() # write to obscurely named file in sourcedir obscure_file = opj(sourcedir, get_most_obscure_supported_name()) with open(obscure_file, 'w') as f: f.write("three") # copy tempfile list to remote_url:sourcedir sourcefiles = [sourcefile1, sourcefile2, obscure_file] ssh.copy(sourcefiles, opj(remote_url, sourcedir)) # recursive copy tempdir to remote_url:targetdir targetdir = sourcedir + '.c opy' ssh.copy(sourcedir, opj(remote_url, targetdir), recursive=True, preserve_attrs=True) # check if sourcedir copied to remote_url:targetdir ok_(isdir(targetdir)) # check if scp preserved source directory attributes # if source_mtime=1.12s, scp -p sets target_mtime = 1.0s, test that eq_(getmtime(targetdir), int(getmtime(sourcedir)) + 0.0) # check if targetfiles(and its content) exist in remote_url:targetdir, # this implies file(s) and recursive directory copying pass for targetfile, content in zip(sourcefiles, ["one", "two", "three"]): targetpath = opj(targetdir, targetfile) ok_(exists(targetpath)) with open(targetpath, 'r') as fp: eq_(content, fp.read()) ssh.close()
def test_GitRepo_commit(path): gr = GitRepo(path) filename = get_most_obscure_supported_name() with open(op.join(path, filename), 'w') as f: f.write("File to add to git") gr.add(filename) gr.commit("Testing GitRepo.commit().") ok_clean_git(gr) eq_("Testing GitRepo.commit().{}".format(linesep), gr.repo.head.commit.message) with open(op.join(path, filename), 'w') as f: f.write("changed content") gr.add(filename) gr.commit("commit with options", options=to_options(dry_run=True)) # wasn't actually committed: ok_(gr.dirty) # commit with empty message: gr.commit() ok_clean_git(gr) # nothing to commit doesn't raise by default: gr.commit() # but does with careless=False: assert_raises(CommandError, gr.commit, careless=False) # committing untracked file raises: with open(op.join(path, "untracked"), "w") as f: f.write("some") assert_raises(FileNotInRepositoryError, gr.commit, files="untracked") # not existing file as well: assert_raises(FileNotInRepositoryError, gr.commit, files="not-existing")
def test_get_most_obscure_supported_name(): n = get_most_obscure_supported_name() assert_in(n, [OBSCURE_PREFIX + OF for OF in OBSCURE_FILENAMES[1:]])
assert_raises(ValueError, manager.get_connection, ':datalad-test') # we can do what urlparse cannot # assert_raises(ValueError, manager.get_connection, 'someone@localhost') # next one is considered a proper url by urlparse (netloc:'', # path='/localhost), but eventually gets turned into SSHRI(hostname='ssh', # path='/localhost') -- which is fair IMHO -> invalid test # assert_raises(ValueError, manager.get_connection, 'ssh:/localhost') manager.close() @skip_if_on_windows @skip_ssh @with_tree(tree={'f0': 'f0', 'f1': 'f1'}) @with_tempfile(suffix=get_most_obscure_supported_name(), content="1") def test_ssh_open_close(tmp_path, tfile1): manager = SSHManager() socket_path = None if _ssh_manager_is_multiplex: socket_path = opj(str(manager.socket_dir), get_connection_hash('datalad-test', bundled=True)) # TODO: facilitate the test when it didn't exist existed_before = exists(socket_path) c1 = manager.get_connection('ssh://datalad-test') c1.open() if socket_path: # control master exists for sure now