def generate_yara_update_file(path=self._yara_repo_rules_dir):
import vcs
# don't move these imports! they're not needed on the worker
from constance import config
rule_repo = None
try:
rule_repo = vcs.get_backend(config.YARA_REPO_TYPE)(path, create=True, src_url=config.YARA_REPO_URL)
except vcs.RepositoryError:
# this means its already there ....
rule_repo = vcs.get_repo(path=path, create=False)
# ensure that we have the latest copy
# todo detect when the repo url is changed and blow it away
rule_repo.run_git_command("pull")
tmp_path = tempfile.mktemp(suffix='.tar')
rule_repo.run_git_command("checkout")
rule_repo.run_git_command("archive master -o {0}".format(tmp_path))
temp_ver_path = tempfile.mktemp()
with open(temp_ver_path, 'w') as version_file_obj:
version_file_obj.write(str(rule_repo.get_changeset().revision))
version_file_obj.flush()
with open(temp_ver_path, 'r') as version_file_obj:
with tarfile.open(tmp_path, 'a') as tf:
version_info = tf.gettarinfo(name=self._version_file, fileobj=version_file_obj, arcname=self._version_file)
tf.addfile(version_info, fileobj=version_file_obj)
tf.close()
pfs_update = PickleableFileSample.path_factory(tmp_path)
unlink(tmp_path)
unlink(temp_ver_path)
return pfs_update
def generate_yara_update_file(path=self._yara_repo_rules_dir):
import vcs
# don't move these imports! they're not needed on the worker
from constance import config
rule_repo = None
try:
rule_repo = vcs.get_backend(config.YARA_REPO_TYPE)(path, create=True, src_url=config.YARA_REPO_URL)
except vcs.RepositoryError:
# this means its already there ....
rule_repo = vcs.get_repo(path=path, create=False)
# ensure that we have the latest copy
# todo detect when the repo url is changed and blow it away
rule_repo.run_git_command("pull")
tmp_path = tempfile.mktemp(suffix='.tar')
rule_repo.run_git_command("checkout")
rule_repo.run_git_command("archive master -o {0}".format(tmp_path))
temp_ver_path = tempfile.mktemp()
with open(temp_ver_path, 'w') as version_file_obj:
version_file_obj.write(str(rule_repo.get_changeset().revision))
version_file_obj.flush()
with open(temp_ver_path, 'r') as version_file_obj:
with tarfile.open(tmp_path, 'a') as tf:
version_info = tf.gettarinfo(name=self._version_file, fileobj=version_file_obj, arcname=self._version_file)
tf.addfile(version_info, fileobj=version_file_obj)
tf.close()
pfs_update = PickleableFileSample.path_factory(tmp_path)
unlink(tmp_path)
unlink(temp_ver_path)
return pfs_update
def __get_instance(self):
repo_full_path = self.repo_full_path
try:
alias = get_scm(repo_full_path)[0]
log.debug('Creating instance of %s repository', alias)
backend = get_backend(alias)
except VCSError:
log.error(traceback.format_exc())
log.error('Perhaps this repository is in db and not in '
'filesystem run rescan repositories with '
'"destroy old data " option from admin panel')
return
if alias == 'hg':
repo = backend(safe_str(repo_full_path), create=False,
baseui=self._ui)
#skip hidden web repository
if repo._get_hidden():
return
else:
repo = backend(repo_full_path, create=False)
return repo
def test_get_repo(self):
alias = 'hg'
path = TEST_HG_REPO
backend = get_backend(alias)
repo = backend(path)
self.assertEqual(repo.__class__, get_repo(path, alias).__class__)
self.assertEqual(repo.path, get_repo(path, alias).path)
def test_get_repo_autoalias_git(self):
alias = 'git'
path = TEST_GIT_REPO
backend = get_backend(alias)
repo = backend(path)
self.assertEqual(repo.__class__, get_repo(path).__class__)
self.assertEqual(repo.path, get_repo(path).path)
def test_get_repo_autoalias_hg(self):
alias = "hg"
path = TEST_HG_REPO
backend = get_backend(alias)
repo = backend(path)
self.assertEqual(repo.__class__, get_repo(path).__class__)
self.assertEqual(repo.path, get_repo(path).path)
def test_get_repo_autoalias_git(self):
alias = 'git'
path = TEST_GIT_REPO
backend = get_backend(alias)
repo = backend(path)
self.assertEqual(repo.__class__, get_repo(path).__class__)
self.assertEqual(repo.path, get_repo(path).path)
def test_get_repo(self):
alias = 'hg'
path = TEST_HG_REPO
backend = get_backend(alias)
repo = backend(path)
self.assertEqual(repo.__class__, get_repo(path, alias).__class__)
self.assertEqual(repo.path, get_repo(path, alias).path)
def _clone_and_create_update_tar(self):
import vcs
g = vcs.get_backend('git')
self.gr = g(self.temp_repo_dir,
src_url=self.test_repo_update_url,
create=True,
update_after_clone=True)
self.update_file_factory_func = self._scanner.get_update_file_factory()
def create_repo_fork(form_data, cur_user):
from rhodecode.model.repo import RepoModel
try:
log = create_repo_fork.get_logger()
except:
log = logging.getLogger(__name__)
repo_model = RepoModel(get_session())
repo_model.create(form_data, cur_user, just_db=True, fork=True)
repo_name = form_data['repo_name']
repos_path = get_repos_path()
repo_path = os.path.join(repos_path, repo_name)
repo_fork_path = os.path.join(repos_path, form_data['fork_name'])
alias = form_data['repo_type']
log.info('creating repo fork %s as %s', repo_name, repo_path)
backend = get_backend(alias)
backend(str(repo_fork_path), create=True, src_url=str(repo_path))
def repo_scan(self, repos_path=None):
"""Listing of repositories in given path. This path should not be a
repository itself. Return a dictionary of repository objects
:param repos_path: path to directory containing repositories
"""
if repos_path is None:
repos_path = self.repos_path
log.info('scanning for repositories in %s', repos_path)
baseui = make_ui('db')
repos_list = {}
for name, path in get_filesystem_repos(repos_path, recursive=True):
# name need to be decomposed and put back together using the /
# since this is internal storage separator for rhodecode
name = Repository.url_sep().join(name.split(os.sep))
try:
if name in repos_list:
raise RepositoryError('Duplicate repository name %s '
'found in %s' % (name, path))
else:
klass = get_backend(path[0])
if path[0] == 'hg' and path[0] in BACKENDS.keys():
# for mercurial we need to have an str path
repos_list[name] = klass(safe_str(path[1]),
baseui=baseui)
if path[0] == 'git' and path[0] in BACKENDS.keys():
repos_list[name] = klass(path[1])
except OSError:
continue
return repos_list
def repo_scan(self, repos_path=None):
"""Listing of repositories in given path. This path should not be a
repository itself. Return a dictionary of repository objects
:param repos_path: path to directory containing repositories
"""
log.info('scanning for repositories in %s', repos_path)
if repos_path is None:
repos_path = self.repos_path
baseui = make_ui('db')
repos_list = {}
for name, path in get_filesystem_repos(repos_path, recursive=True):
try:
if name in repos_list:
raise RepositoryError('Duplicate repository name %s '
'found in %s' % (name, path))
else:
klass = get_backend(path[0])
if path[0] == 'hg' and path[0] in BACKENDS.keys():
# for mercurial we need to have an str path
repos_list[name] = klass(safe_str(path[1]),
baseui=baseui)
if path[0] == 'git' and path[0] in BACKENDS.keys():
repos_list[name] = klass(path[1])
except OSError:
continue
return repos_list
def get_backend(cls):
return vcs.get_backend(cls.backend_alias)
def get_backend(cls):
return vcs.get_backend(cls.backend_alias)
def test_alias_detect_git(self):
alias = 'git'
path = TEST_GIT_REPO
backend = get_backend(alias)
repo = backend(path)
self.assertEqual('git', repo.alias)
def get_vcs_backend(self):
return vcs.get_backend(self.vcs_type)
def test_alias_detect_git(self):
alias = 'git'
path = TEST_GIT_REPO
backend = get_backend(alias)
repo = backend(path)
self.assertEqual('git',repo.alias)
def test_alias_detect_hg(self):
alias = 'hg'
path = TEST_HG_REPO
backend = get_backend(alias)
repo = backend(path)
self.assertEqual('hg',repo.alias)
def test_get_backend(self):
hg = get_backend('hg')
self.assertEqual(hg, MercurialRepository)
def get_backend(self):
return vcs.get_backend(self.backend_alias)
def get_vcs_backend(self):
return vcs.get_backend(self.vcs_type)
def _clone_and_create_update_tar(self):
import vcs
g = vcs.get_backend('git')
self.gr = g(self.temp_repo_dir, src_url=self.test_repo_update_url, create=True, update_after_clone=True)
self.update_file_factory_func = self._scanner.get_update_file_factory()
def branch(self):
return get_backend(self.alias).DEFAULT_BRANCH_NAME
def test_get_backend(self):
hg = get_backend('hg')
self.assertEqual(hg, MercurialRepository)
def get_backend(self):
return vcs.get_backend(self.backend_alias)
def test_alias_detect_hg(self):
alias = 'hg'
path = TEST_HG_REPO
backend = get_backend(alias)
repo = backend(path)
self.assertEqual('hg', repo.alias)
