def _prepare_local_repository(self, opts):
url = opts['url']
branch = opts['branch']
if url is None:
raise Exception('Remote git URL not set.')
# set local repository path to be cloned
repo_name = url[url.rindex('/') + 1:]
default_clone_dir = os.path.join(os.path.dirname(__file__), 'clones', repo_name)
local_path = opts.get('local_clone_path', default_clone_dir)
# create a directory to store cloned repositories
if not os.path.exists(os.path.dirname(local_path)):
os.mkdir(os.path.dirname(local_path), 0o755)
if os.path.exists(local_path):
repo_local = Repo.init(local_path)
else:
try:
repo_local = Repo.clone_from(url, local_path, branch=branch)
except Exception as e:
raise Exception('Unable to clone remote repo from %s [%s]: %s' %
(url, branch, str(e)))
return repo_local, repo_local.remote('origin')
def _clone_repo(repo_url, branch='master'):
user_home = os.path.expanduser('~')
# Assuming git url is of form [email protected]:user/git-repo.git
repo_name = repo_url[repo_url.rfind('/') + 1: repo_url.rfind('.')]
abs_local_path = os.path.join(user_home, repo_name)
Repo.clone_from(repo_url, abs_local_path, branch=branch))
return abs_local_path
def get_text(self, sha=None):
if self.chapter.tutorial:
tutorial = self.chapter.tutorial
else:
tutorial = self.chapter.part.tutorial
repo = Repo(tutorial.get_path())
# find hash code
if sha is None:
sha = tutorial.sha_draft
manifest = get_blob(repo.commit(sha).tree, "manifest.json")
tutorial_version = json_reader.loads(manifest)
if "parts" in tutorial_version:
for part in tutorial_version["parts"]:
if "chapters" in part:
for chapter in part["chapters"]:
if "extracts" in chapter:
for extract in chapter["extracts"]:
if extract["pk"] == self.pk:
path_ext = extract["text"]
break
if "chapter" in tutorial_version:
chapter = tutorial_version["chapter"]
if "extracts" in chapter:
for extract in chapter["extracts"]:
if extract["pk"] == self.pk:
path_ext = extract["text"]
break
if path_ext:
return get_blob(repo.commit(sha).tree, path_ext)
else:
return None
def git_pull(remote_url, local, idc_dir):
"""
@remote: git remote address
@local: local git repo dir
"""
commit_msg = None
if not os.path.exists(local):
os.mkdir(local)
logging.info('git start pull from remote repo.')
if not os.path.exists(
"{0}/.git".format(local)) or not git.repo.fun.is_git_dir(
"{0}/.git".format(local)):
repo_clone = Repo.clone_from(remote_url, local)
if not repo_clone:
logging.debug("clone repo from {0} to {1}failed.\n".format(
remote_url, local))
return
repo = Repo(local)
remote = None
git_commit = git.Git(idc_dir)
last_commit_log = "/var/log/last_commit.log"
last_commit = None
if os.path.exists(last_commit_log):
with file(last_commit_log) as fhand:
last_log_commit = fhand.read().strip()
# Warning: for performance only check 500 commit, use this write mysql
for commit in repo.iter_commits('master', max_count=500):
if last_commit == commit.hexsha:
last_commit = last_log_commit
break
if not last_commit:
last_commit = repo.head.commit.hexsha
with FileLock(os.path.split(local)[1]) as _flock:
for remote_name in repo.remotes:
if remote_name.url == remote_url:
remote = remote_name
break
if remote is None:
remote_name = "%x" % (random.randint(0, 1000000)
^ int(time.time()))
remote = repo.create_remote(remote_name, remote_url)
ret = 0
try:
info_list = remote.pull()
for info in info_list:
ret = info.flags & (git.FetchInfo.REJECTED
| git.FetchInfo.ERROR)
if ret > 0:
logging.warning("[conf-pull] pull from remote error!\n")
commit_msg = git_commit.log("{0}..HEAD".format(last_commit),
"--pretty=%H,%ai,%s", idc_dir)
except (git.GitCommandError, git.RepositoryDirtyError), e:
logging.warning("git pull error, error={0}\n".format(str(e)))
except Exception:
logging.warning("git pull error, other exception.\n")
def __init__(self, source_repo_path, source_repo_branch, target_repo_path, target_repo_relative_directory, target_repo_commit_branch):
self.source_repo_path = source_repo_path
self.source_repo_branch = source_repo_branch
self.target_repo_path = target_repo_path
self.target_repo_commit_branch = target_repo_commit_branch
# Instantiate repo objects
self.source_repo = Repo(self.source_repo_path)
self.target_repo = Repo(self.target_repo_path)
# saving the head of the source repo
self.source_head = self.source_repo.head.name
# extract repo names
self.source_repo_name = get_repo_name(self.source_repo)
self.target_repo_name = get_repo_name(self.target_repo)
# export paths
self.target_repo_archive_path_root = "{}/{}".format(COMMIT_EXPORT_PATH, self.source_repo_name)
self.target_repo_archive_path_tar = "{}/{}".format(self.target_repo_archive_path_root, "commit_archive.tar")
self.target_repo_directory = "{}/{}".format(self.target_repo_path, target_repo_relative_directory)
# ingores to add at each commit
self.ignores_to_add = extract_gitinore(self.source_repo_path)
# initialize commit infos
self._exact_commits()
def pull_request(pathList):
addList, deleteList = [], []
for path in pathList:
repo = Repo(path)
# 检查版本库是否为空
if repo.bare:
return None
# 获取远程默认版本库 origin
remote = repo.remote()
try:
# 从远程版本库拉取分支
remote.pull()
except Exception as e:
print(path)
print(str(e))
git = repo.git
# 读取本地版本库的信息
strList = git.log('--numstat').split()
# 需要计算的文件
file = 'Units.cpp'
# 运行前需要设置时间
add, delete = count_add_delete(strList, file, 'Jun',10, 'Jun', 16)
addList.append(add)
deleteList.append(delete)
print('add:', addList)
print('delete:', deleteList)
write_excel(addList, deleteList)
def get_text(self, sha=None):
if self.chapter.tutorial:
tutorial = self.chapter.tutorial
else:
tutorial = self.chapter.part.tutorial
repo = Repo(tutorial.get_path())
# find hash code
if sha is None:
sha = tutorial.sha_draft
manifest = get_blob(repo.commit(sha).tree, "manifest.json")
tutorial_version = json_reader.loads(manifest)
if "parts" in tutorial_version:
for part in tutorial_version["parts"]:
if "chapters" in part:
for chapter in part["chapters"]:
if "extracts" in chapter:
for extract in chapter["extracts"]:
if extract["pk"] == self.pk:
path_ext = extract["text"]
break
if "chapter" in tutorial_version:
chapter = tutorial_version["chapter"]
if "extracts" in chapter:
for extract in chapter["extracts"]:
if extract["pk"] == self.pk:
path_ext = extract["text"]
break
if path_ext:
return get_blob(repo.commit(sha).tree, path_ext)
else:
return None
def setup(self):
git_opts = self._config
if git_opts['url'] is None:
raise Exception('Remote git URL not set.')
self._url = git_opts['url']
self._branch = git_opts['branch']
default_clone_dir = os.path.join(os.path.dirname(__file__), 'clones')
self._local_path = git_opts.get('local_clone_path', default_clone_dir)
self._poll_interval = git_opts.get('poll_interval',
self._poll_interval)
if os.path.exists(self._local_path):
self._repo = Repo.init(self._local_path)
else:
try:
self._repo = Repo.clone_from(self._url,
self._local_path,
branch=self._branch)
except Exception:
self._logger.exception(
'Unable to clone remote repo from %s branch %s', self._url,
self._branch)
raise
self._remote = self._repo.remote('origin')
def __init__(self):
try:
self.repo = Repo('.')
except InvalidGitRepositoryError:
msg = 'Not in a git repository'
raise ConfigurationError(msg, 1)
self._load_config()
self._missing_config = []
config = {
'deploy.repo-name': {
'required': True,
},
'deploy.required-umask': {
'required': False,
'default': None,
},
'user.name': {
'required': True,
},
'user.email': {
'required': True,
},
}
self._register_config(config)
self.register_drivers()
def _prepare(cls, create, **kwargs):
extract = super(ExtractFactory, cls)._prepare(create, **kwargs)
chapter = kwargs.pop('chapter', None)
with open(extract.get_path(relative=False), "w") as f:
f.write("This dumb content is just here to prove you zep12 is far better than old module")
if chapter:
if chapter.tutorial:
repo = Repo(os.path.join(settings.ZDS_APP['tutorial']['repo_path'],
chapter.tutorial.get_phy_slug()))
index = repo.index
index.add([extract.get_path(relative=True)])
man_path = os.path.join(chapter.tutorial.get_path(), "manifest.json")
chapter.tutorial.dump_json(path=man_path)
index.add(["manifest.json"])
chapter.tutorial.sha_draft = index.commit(
"bla",
author=Actor("bla", "*****@*****.**"),
committer=Actor("bla", "*****@*****.**"))
chapter.tutorial.save()
elif chapter.part:
repo = Repo(os.path.join(settings.ZDS_APP['tutorial']['repo_path'],
chapter.part.tutorial.get_phy_slug()))
index = repo.index
index.add([extract.get_path(relative=True)])
man_path = os.path.join(chapter.part.tutorial.get_path(), "manifest.json")
chapter.part.tutorial.dump_json(path=man_path)
index.add(["manifest.json"])
chapter.part.tutorial.sha_draft = index.commit(
"bla",
author=Actor("bla", "*****@*****.**"),
committer=Actor("bla", "*****@*****.**"))
chapter.part.tutorial.save()
return extract
def _prepare(cls, create, **kwargs):
light = kwargs.pop('light', False)
tuto = super(BigTutorialFactory, cls)._prepare(create, **kwargs)
path = tuto.get_path()
real_content = content
if light:
real_content = content_light
if not os.path.isdir(path):
os.makedirs(path, mode=0o777)
man = export_tutorial(tuto)
repo = Repo.init(path, bare=False)
repo = Repo(path)
f = open(os.path.join(path, 'manifest.json'), "w")
f.write(json_writer.dumps(man, indent=4, ensure_ascii=False).encode('utf-8'))
f.close()
f = open(os.path.join(path, tuto.introduction), "w")
f.write(real_content.encode('utf-8'))
f.close()
f = open(os.path.join(path, tuto.conclusion), "w")
f.write(real_content.encode('utf-8'))
f.close()
repo.index.add(['manifest.json', tuto.introduction, tuto.conclusion])
cm = repo.index.commit("Init Tuto")
tuto.sha_draft = cm.hexsha
tuto.sha_beta = None
tuto.gallery = GalleryFactory()
for author in tuto.authors.all():
UserGalleryFactory(user=author, gallery=tuto.gallery)
return tuto
def _clone_repo(repo_url):
user_home = os.path.expanduser('~')
# Assuming git url is of form [email protected]:user/git-repo.git
repo_name = repo_url[repo_url.rfind('/') + 1: repo_url.rfind('.')]
abs_local_path = os.path.join(user_home, repo_name)
Repo.clone_from(repo_url, abs_local_path)
return abs_local_path
def get_latest_status(self):
"""
Property for viewing repository latest commit status message.
"""
repo = Repo(Repo.get_repository_location(self.owner.username, self.name))
return repo.get_latest_status()
def _prepare(cls, create, **kwargs):
tuto = super(MiniTutorialFactory, cls)._prepare(create, **kwargs)
path = tuto.get_path()
if not os.path.isdir(path):
os.makedirs(path, mode=0o777)
man = export_tutorial(tuto)
repo = Repo.init(path, bare=False)
repo = Repo(path)
file = open(os.path.join(path, 'manifest.json'), "w")
file.write(
json_writer.dumps(
man,
indent=4,
ensure_ascii=False).encode('utf-8'))
file.close()
file = open(os.path.join(path, tuto.introduction), "w")
file.write(u'Test')
file.close()
file = open(os.path.join(path, tuto.conclusion), "w")
file.write(u'Test')
file.close()
repo.index.add(['manifest.json', tuto.introduction, tuto.conclusion])
cm = repo.index.commit("Init Tuto")
tuto.sha_draft = cm.hexsha
return tuto
def load_json_for_public(self, sha=None):
if sha is None:
sha = self.sha_public
repo = Repo(self.get_path())
mantuto = get_blob(repo.commit(sha).tree, 'manifest.json')
data = json_reader.loads(mantuto)
return data
def load_json_for_public(self, sha=None):
if sha is None:
sha = self.sha_public
repo = Repo(self.get_path())
mantuto = get_blob(repo.commit(sha).tree, 'manifest.json')
data = json_reader.loads(mantuto)
return data
def get_latest_status(self):
"""
Property for viewing repository latest commit status message.
"""
repo = Repo(
Repo.get_repository_location(self.owner.username, self.name))
return repo.get_latest_status()
def _check_repo_and_getcommit(self):
#TODO improve this logic. I would like to retrain not only for new commits but also fo new data.
# Also add tests
repo = Repo(search_parent_directories=True)
if not repo.is_dirty() or True:
return repo.head.object.hexsha
else:
raise ValueError('working directory is not clean! please commit before running the server')
def commits_stats(request, username, repository):
"""
Returns number of commits for the given repository.
"""
repo = Repo(Repo.get_repository_location(username, repository))
stats = GitStatistics(repo, repo.get_head())
return HttpResponse(json.dumps({'weekly': stats.for_commits(weekly, js), 'monthly': stats.for_commits(monthly, js)}))
def git_pull(remote_url, git_local):
"""
@remote_url: git remote address
@git_local: local git repo dir
"""
logging.info('git start pull from remote repo.')
# check if we need to clean the local git
clean_local_git(False, git_local, remote_url)
# git clone if not exist
if not os.path.exists("{0}/.git".format(git_local)) or \
not git.repo.fun.is_git_dir("{0}/.git".format(git_local)):
logging.info('git clone from [%s] to [%s], and depth=1.', remote_url, git_local)
repo_clone = Repo.clone_from(remote_url, git_local,depth=1)
if not repo_clone:
logging.error(
"clone repo from {0} to {1}failed.".format(remote_url, git_local))
logging.info('========run stop========\n\n')
sys.exit(1)
# get repo
repo = Repo(git_local)
type(repo.git).GIT_PYTHON_TRACE = 'full'
remote = None
# get remote url
for item in repo.remotes:
if item.url == remote_url:
remote = item
break
if remote is None:
clean_local_git(True, git_local, remote_url)
# start pull
ssh_executable = os.path.join(g_script_path, 'debug_ssh.sh')
with repo.git.custom_environment(GIT_SSH_COMMAND=ssh_executable):
try:
logging.info('git pull from remote=%s, url=%s',
remote.name, remote.url)
info_list = remote.pull()
for info in info_list:
ret = info.flags & (git.FetchInfo.REJECTED |
git.FetchInfo.ERROR)
if ret > 0:
logging.warning("git pull from remote ERROR: %s", info_list)
logging.info('========run stop========\n\n')
sys.exit(1)
else:
logging.info("git pull return mesg: %s", info_list)
except BadName:
logging.warn("git pull warn: ", exc_info=True)
except Exception:
clean_local_git(True, git_local, remote_url)
# get last commit msg
last_commit_id = str(repo.head.commit)
return last_commit_id
def update(options, error, debug, info):
path = getcwd()
repo = Repo(path)
if not repo.is_dirty():
update_from_remote(debug, info, repo, 'origin')
update_from_remote(debug, info, repo, 'upstream')
else:
error('Please commit or stash your changes before continuing.')
def on_get(self, req, res):
repoPath = './IntallPlist' #本地仓库路径
result = 'success' #默认返回success
msg = []
repo = None
# 判断本地是否有仓库
if os.path.exists(repoPath):
repo = Repo(repoPath)
msg.append('已有仓库')
else:
repo = Repo.clone_from(
url='[email protected]:hillyoung/TestPlist.git', to_path=repoPath)
msg.append('不存在仓库')
git = repo.git
git.checkout('*')
remote = repo.remote()
remote.pull()
msg.append('完成拉取远程仓库代码')
# 判断是否传入了文件名
name = req.params.get('name', None)
if name is None:
msg.append('传入无效参数')
result = 'fail'
else:
msg.append('传入应用名：' + name)
# 构造plist文件内容及相关控制逻辑
directoryPath = repoPath + '/' + name
if os.path.exists(directoryPath) != True:
os.mkdir(directoryPath)
filePath = directoryPath + '/install.plist' #plist文件相对路径
rContent = None #用于缓存仓库中plist文件的内容
flag = 'w' #文件的打开访问，默认以写入方式打开
if os.path.exists(filePath): #判断仓库中是否存在plist文件
flag = 'r+' #如果plist文件存在，则以读写的方式打开
msg.append('存在plist文件')
content = '<?xml version="1.0" encoding="UTF-8"?><!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd"><plist version="1.0"><dict><key>items</key><array><dict><key>assets</key><array><dict><key>kind</key><string>software-package</string><key>url</key><string>https://raw.githubusercontent.com/hillyoung/AppInstaller/master/app/' + name + '/LiemsMobileEnterprise.ipa</string></dict></array><key>metadata</key><dict><key>bundle-identifier</key> <string>net.luculent.liems.hhgs</string><key>bundle-version</key><string>1.0.1</string><key>kind</key><string>software</string><key>title</key><string>测试安装</string></dict></dict></array></dict></plist>'
with open(filePath, flag) as flie_object: #写入plist
if flag == 'r+':
rContent = flie_object.read()
flie_object.seek(0)
msg.append(rContent)
if rContent != content: #写入plist内容
flie_object.write(content)
flie_object.close()
#提交plist文件
git.add('*')
git.commit('-m', '添加' + name)
remote.push()
res.body = json.dumps({'result': result, 'msg': msg})
res.status = falcon.HTTP_200 # This is the default status
res.append_header('Access-Control-Allow-Origin', '*')
def initial(self, branch):
if not os.path.exists(self.local_path):
os.makedirs(self.local_path)
git_local_path = os.path.join(self.local_path, '.git')
if not is_git_dir(git_local_path):
self.repo = Repo.clone_from(self.repo_url,
to_path=self.local_path,
branch=branch)
else:
self.repo = Repo(self.local_path)
def _decorator(request, *args, **kwargs):
try:
repo = Repo(Repo.get_repository_location(kwargs['username'], kwargs['repository']))
branches = repo.get_branches()
if 'rev' not in kwargs or kwargs['rev'] in branches + ['HEAD']:
return func(request, *args, **kwargs)
else:
raise Http404()
except:
raise Http404()
def load_json_for_public(self, sha=None):
if sha is None:
sha = self.sha_public
repo = Repo(self.get_path())
mantuto = get_blob(repo.commit(sha).tree, 'manifest.json')
data = json_reader.loads(mantuto)
if 'licence' in data:
data['licence'] = Licence.objects.filter(
code=data['licence']).first()
return data
def init_bare_repo_signal(sender, instance, created, **kwargs):
"""
Initialize a bare git repository on server's deposit after it's creation.
"""
if created:
# Create repository folder under GIT_DEPOSIT_ROOT/username/repository.git and initialize it as bare.
repository_location = Repo.get_repository_location(instance.owner.username, instance.name)
repo = Repo(repository_location)
repo.init_bare_repo()
def getCommits(repo_name):
shas = set()
commits_list = []
repo = Repo(os.path.join(work_dir, repo_name))
for b in repo.remote().fetch():
if '/' not in b.name:
continue
print("start to check branch {} of {}".format(b.name, repo_name))
branch_name = b.name.split('/')[1]
repo.git.checkout('-B', branch_name, b.name)
commits = list(repo.iter_commits())
for idx, commit in enumerate(commits):
if str(commit) in shas:
continue
else:
shas.add(str(commit))
commit_collection = {}
xml_cnt = 0
kot_jav_cnt = 0
file_list = list(commit.stats.files)
for file in file_list:
if len(file) > 4 and file[-4:] == '.xml':
xml_cnt += 1
elif len(file) > 3 and file[-3:] == '.kt' or len(
file) > 5 and file[-5:] == '.java':
kot_jav_cnt += 1
if xml_cnt >= 1 and kot_jav_cnt >= 1:
commit_collection["commit_id"] = str(commit)
commit_collection["commit_msg"] = str.strip(commit.message)
commit_collection["commit_time"] = str(
commit.committed_datetime)
commit_collection["committer_name"] = commit.author.name
commit_collection["committer_email"] = commit.author.email
diff_files = []
if not commit.parents:
continue
else:
for diff in commit.parents[0].diff(commit):
diff_file = {}
diff_file["file_path"] = diff.a_path
diff_file["change_type"] = diff.change_type
diff_file["lang"] = os.path.splitext(
diff.a_path)[1][1:]
diff_files.append(diff_file)
commit_collection["diff_files"] = diff_files
commit_collection["parent_commit_num"] = len(commit.parents)
commits_list.append(commit_collection)
repo_dump = json.dumps(commits_list, indent=2, ensure_ascii=False)
with open(os.path.join(stats_dir, repo_name + ".json"), "w") as commit_fd:
commit_fd.write(repo_dump)
print(repo_name + " done")
def readme(request, username, repository, rev):
"""
Returns README content of a folder inside the given repository.
"""
repo = Repo(Repo.get_repository_location(username, repository))
request_sections = partition_url(request.path_info)
rdm = GitBlob(repo=repo, path='/'.join(request_sections[5:]),
rev=rev).show()
return HttpResponse(rdm)
def init_bare_repo_signal(sender, instance, created, **kwargs):
"""
Initialize a bare git repository on server's deposit after it's creation.
"""
if created:
# Create repository folder under GIT_DEPOSIT_ROOT/username/repository.git and initialize it as bare.
repository_location = Repo.get_repository_location(
instance.owner.username, instance.name)
repo = Repo(repository_location)
repo.init_bare_repo()
def get_info_refs(request, username, repository):
"""
Responds to '/info/refs' requests for the given service, username and repository.
"""
requested_repo = Repo(Repo.get_repository_location(username, repository))
response = GitResponse(service=request.GET['service'],
action=GIT_ACTION_ADVERTISEMENT,
repository=requested_repo,
data=None)
return response.get_http_info_refs()
def _clone_repo(repo_url, verifyssl=True, branch='master'):
user_home = os.path.expanduser('~')
# Assuming git url is of form [email protected]:user/git-repo.git
repo_name = repo_url[repo_url.rfind('/') + 1: repo_url.rfind('.')]
abs_local_path = os.path.join(user_home, repo_name)
# Disable SSL cert checking if explictly asked
if not verifyssl:
os.environ['GIT_SSL_NO_VERIFY'] = 'true'
Repo.clone_from(repo_url, abs_local_path, branch=branch)
return abs_local_path
def __init__(self, database):
logging.info("Initializing GitSync.")
self.database = database
self.charts = dict()
self.url = DEFAULT_GITREPO
try:
self.repo = Repo(REPO_DIRECTORY)
except InvalidGitRepositoryError:
logging.info("Cloning repository in %s", REPO_DIRECTORY)
self.repo = Repo.clone_from(self.url, REPO_DIRECTORY)
def _setup_repo(self, already_cloned):
if already_cloned:
self._git_repo = Repo(self.repo_dir_path)
else:
_log.info('Cloning repo from %s into %s' % (
self._github_repo_url, self.repo_dir_path))
self._git_repo = Repo.clone_from(
self._github_repo_url, self.repo_dir_path)
_log.info('-Done-')
if self.branch:
self._git_repo.git.checkout(self.branch)
def last_update(self):
"""
Property for viewing latest activty date on repository in "ISO 8601-like" format and 'UTC' timezone.
"""
last_update = Repo(
Repo.get_repository_location(self.owner.username,
self.name)).get_last_update()
if last_update is None:
return time_to_utc(str(self.creation_date))
else:
return last_update
def __init__(self, database):
logging.info("Initializing GitSync.")
self.database = database
self.charts = dict()
self.url = DEFAULT_GITREPO
try:
self.repo = Repo(REPO_DIRECTORY)
except InvalidGitRepositoryError:
logging.info("Cloning repository in %s", REPO_DIRECTORY)
self.repo = Repo.clone_from(self.url, REPO_DIRECTORY)
def _clone_repo(repo_url, verifyssl=True, branch="master"):
user_home = os.path.expanduser("~")
# Assuming git url is of form [email protected]:user/git-repo.git
repo_name = DownloadGitRepoAction._eval_repo_name(repo_url)
abs_local_path = os.path.join(user_home, repo_name)
# Disable SSL cert checking if explictly asked
if not verifyssl:
os.environ["GIT_SSL_NO_VERIFY"] = "true"
# Shallow clone the repo to avoid getting all the metadata. We only need HEAD of a
# specific branch so save some download time.
Repo.clone_from(repo_url, abs_local_path, branch=branch, depth=1)
return abs_local_path
def code_clone(code_link):
download_link = 'https://github.com/' + code_link.split(
'/')[3] + '/' + code_link.split('/')[4] + '.git'
to_dir = './git_repository/' + code_link.split('/')[4]
try:
Repo.clone_from(download_link, to_dir)
except:
if os.path.exists('./git_repository/' + code_link.split('/')[4]):
print('-----duplicated project-----')
else:
print('something wrong happen in git clone')
else:
print('-----successfully download: ' + download_link + '-----')
def detect_from_commits_list(self, args):
historage = Repo(args.historage_dir)
results = []
try:
for a_commit_hash, b_commit_hash in csv.reader(open(args.commits_list)):
a_commit = historage.commit(a_commit_hash)
b_commit = historage.commit(b_commit_hash)
results.extend(detect_shingle_pullup_method(a_commit, b_commit))
except ValueError:
print("Invalid input.")
return
except BadObject, name:
print("Invalid hash of the commit:", name.message)
def get_conclusion(self, sha=None):
# find hash code
if sha is None:
sha = self.sha_draft
repo = Repo(self.get_path())
manifest = get_blob(repo.commit(sha).tree, "manifest.json")
tutorial_version = json_reader.loads(manifest)
if "introduction" in tutorial_version:
path_tuto = tutorial_version["conclusion"]
if path_tuto:
return get_blob(repo.commit(sha).tree, path_tuto)
def _clone_repo(repo_url, verifyssl=True, branch='master'):
user_home = os.path.expanduser('~')
# Assuming git url is of form [email protected]:user/git-repo.git
repo_name = repo_url[repo_url.rfind('/') + 1: repo_url.rfind('.')]
abs_local_path = os.path.join(user_home, repo_name)
# Disable SSL cert checking if explictly asked
if not verifyssl:
os.environ['GIT_SSL_NO_VERIFY'] = 'true'
# Shallow clone the repo to avoid getting all the metadata. We only need HEAD of a
# specific branch so save some download time.
Repo.clone_from(repo_url, abs_local_path, branch=branch, depth=1)
return abs_local_path
def detect_from_commits_list(self, args):
historage = Repo(args.historage_dir)
extract_method_information = []
try:
for a_commit_hash, b_commit_hash in csv.reader(open(args.commits_list)):
a_commit = historage.commit(a_commit_hash)
b_commit = historage.commit(b_commit_hash)
extract_method_information.extend(detect_extract_method_from_commit(a_commit, b_commit))
except ValueError:
print "Invalid input."
return
except BadObject, name:
print "Invalid hash of the commit:", name.message
def _clone_repo(repo_url, verifyssl=True, branch='master'):
user_home = os.path.expanduser('~')
# Assuming git url is of form [email protected]:user/git-repo.git
repo_name = DownloadGitRepoAction._eval_repo_name(repo_url)
abs_local_path = os.path.join(user_home, repo_name)
# Disable SSL cert checking if explictly asked
if not verifyssl:
os.environ['GIT_SSL_NO_VERIFY'] = 'true'
# Shallow clone the repo to avoid getting all the metadata. We only need HEAD of a
# specific branch so save some download time.
Repo.clone_from(repo_url, abs_local_path, branch=branch, depth=1)
return abs_local_path
def get_conclusion(self, sha=None):
# find hash code
if sha is None:
sha = self.sha_draft
repo = Repo(self.get_path())
manifest = get_blob(repo.commit(sha).tree, "manifest.json")
tutorial_version = json_reader.loads(manifest)
if "introduction" in tutorial_version:
path_tuto = tutorial_version["conclusion"]
if path_tuto:
return get_blob(repo.commit(sha).tree, path_tuto)
def _prepare(cls, create, **kwargs):
tuto = super(MiniTutorialFactory, cls)._prepare(create, **kwargs)
path = tuto.get_path()
if not os.path.isdir(path):
os.makedirs(path, mode=0o777)
man = export_tutorial(tuto)
repo = Repo.init(path, bare=False)
repo = Repo(path)
file = open(os.path.join(path, 'manifest.json'), "w")
file.write(
json_writer.dumps(
man,
indent=4,
ensure_ascii=False).encode('utf-8'))
file.close()
file = open(os.path.join(path, tuto.introduction), "w")
file.write(contenu.encode('utf-8'))
file.close()
file = open(os.path.join(path, tuto.conclusion), "w")
file.write(contenu.encode('utf-8'))
file.close()
repo.index.add(['manifest.json', tuto.introduction, tuto.conclusion])
cm = repo.index.commit("Init Tuto")
tuto.sha_draft = cm.hexsha
return tuto
def remove_repo_signal(sender, instance, using, **kwargs):
"""
Remove repository folder from server deposit after it's deletion.
"""
# Remove repository folder under GIT_DEPOSIT_ROOT/username/repository.git
remove_tree(Repo.get_repository_location(instance.owner.username, instance.name))
def __init__(self):
try:
self.repo = Repo('.')
except InvalidGitRepositoryError:
msg = 'Not in a git repository'
raise ConfigurationError(msg, 1)
self._load_config()
self._missing_config = []
config = {
'deploy.repo-name': {
'required': True,
},
'deploy.required-umask': {
'required': False,
'default': None,
},
'user.name': {
'required': True,
},
'user.email': {
'required': True,
},
}
self._register_config(config)
self.register_drivers()
def _clone_repo(temp_dir, repo_url, verifyssl=True, ref='master'):
# Switch to non-interactive mode
os.environ['GIT_TERMINAL_PROMPT'] = '0'
os.environ['GIT_ASKPASS'] = '******'
# Disable SSL cert checking if explictly asked
if not verifyssl:
os.environ['GIT_SSL_NO_VERIFY'] = 'true'
# Clone the repo from git; we don't use shallow copying
# because we want the user to work with the repo in the
# future.
repo = Repo.clone_from(repo_url, temp_dir)
use_branch = False
# Try to match the reference to a branch name (i.e. "master")
gitref = DownloadGitRepoAction._get_gitref(repo, "origin/%s" % ref)
if gitref:
use_branch = True
# Try to match the reference to a commit hash, a tag, or "master"
if not gitref:
gitref = DownloadGitRepoAction._get_gitref(repo, ref)
# Try to match the reference to a "vX.Y.Z" tag
if not gitref and re.match(PACK_VERSION_REGEX, ref):
gitref = DownloadGitRepoAction._get_gitref(repo, "v%s" % ref)
# Giving up ¯\_(ツ)_/¯
if not gitref:
format_values = [ref, repo_url]
msg = '"%s" is not a valid version, hash, tag or branch in %s.'
valid_versions = DownloadGitRepoAction._get_valid_versions_for_repo(repo=repo)
if len(valid_versions) >= 1:
valid_versions_string = ', '.join(valid_versions)
msg += ' Available versions are: %s.'
format_values.append(valid_versions_string)
raise ValueError(msg % tuple(format_values))
# We're trying to figure out which branch the ref is actually on,
# since there's no direct way to check for this in git-python.
branches = repo.git.branch('-a', '--contains', gitref.hexsha)
branches = branches.replace('*', '').split()
if 'master' not in branches or use_branch:
branch = "origin/%s" % ref if use_branch else branches[0]
short_branch = ref if use_branch else branches[0].split('/')[-1]
repo.git.checkout('-b', short_branch, branch)
branch = repo.head.reference
else:
branch = 'master'
repo.git.checkout(gitref.hexsha)
repo.git.branch('-f', branch, gitref.hexsha)
repo.git.checkout(branch)
return temp_dir
def last_update(self):
"""
Property for viewing latest activty date on repository in "ISO 8601-like" format and 'UTC' timezone.
"""
last_update = Repo(Repo.get_repository_location(self.owner.username, self.name)).get_last_update()
if last_update is None:
return time_to_utc(str(self.creation_date))
else:
return last_update
def readme(request, username, repository, rev):
"""
Returns README content of a folder inside the given repository.
"""
repo = Repo(Repo.get_repository_location(username, repository))
request_sections = partition_url(request.path_info)
rdm = GitBlob(repo=repo, path='/'.join(request_sections[5:]), rev=rev).show()
return HttpResponse(rdm)
def main():
parser=OptionParser(usage=__doc__)
parser.add_option("-l","--iscc-binary", dest="iscc_binary", default=os.path.join(os.path.dirname(__file__),'innosetup','ISCC.exe'), help="Path to ISCC compiler (default: %default)")
parser.add_option("-v","--wapt-version", dest="waptversion", default=waptutils.__version__, help="Wapt edition to build (community, enterprise...). (default: %default)")
parser.add_option("-e","--wapt-edition", dest="waptedition", default='community', help="Wapt edition to build (community, enterprise...). (default: %default)")
parser.add_option("-k","--sign-key", dest="sign_key_path", help="Sign with this key. (default: %default)")
parser.add_option("-w","--sign-key-pwd-path", dest="sign_key_pwd_path", help="Path to password file. (default: %default)")
parser.add_option("-x","--sign-exe-filenames", dest="exe_filenames", help="Additional executables to sign. (default: %default)")
(options,args) = parser.parse_args()
if len(args) != 1:
parser.usage
sys.exit(1)
for iss_path in args:
iss_path = os.path.abspath(iss_path)
(iss_rootname,issext) = os.path.splitext(iss_path)
# add a revision.txt file with git short
r = Repo(search_parent_directories=True)
rev_file = open(os.path.join(os.path.dirname(iss_path), '..', 'revision.txt'), 'w')
rev_file.write(r.head.object.hexsha[:8])
rev_file.close()
r.close()
iss_file = iss_rootname + ".iss"
if options.sign_key_path and options.exe_filenames:
exes = options.exe_filenames.split(',')
for exe_fn in exes:
sign_exe(exe_fn,options.sign_key_path,open(options.sign_key_pwd_path,'rb').read())
cmd = '"%(issc_binary)s" /Dwapt%(waptedition)s %(issfile)s' % {
'issc_binary':options.iscc_binary,
'issfile':iss_file,
'waptedition':options.waptedition.lower()
}
res = setuphelpers.run(cmd)
exe_fn = res.splitlines()[-1]
if options.sign_key_path:
sign_exe(exe_fn,options.sign_key_path,open(options.sign_key_pwd_path,'rb').read())
def get_conclusion(self, sha=None):
tutorial = self.tutorial
# find hash code
if sha is None:
sha = tutorial.sha_draft
repo = Repo(tutorial.get_path())
manifest = get_blob(repo.commit(sha).tree, "manifest.json")
tutorial_version = json_reader.loads(manifest)
if "parts" in tutorial_version:
for part in tutorial_version["parts"]:
if part["pk"] == self.pk:
path_part = part["conclusion"]
break
if path_part:
return get_blob(repo.commit(sha).tree, path_part)
else:
return None
def setup(self):
git_opts = self._config
if git_opts['url'] is None:
raise Exception('Remote git URL not set.')
self._url = git_opts['url']
default_clone_dir = os.path.join(os.path.dirname(__file__), 'clones')
self._local_path = git_opts.get('local_clone_path', default_clone_dir)
self._poll_interval = git_opts.get('poll_interval', self._poll_interval)
if os.path.exists(self._local_path):
self._repo = Repo.init(self._local_path)
else:
try:
self._repo = Repo.clone_from(self._url, self._local_path)
except Exception:
self._logger.exception('Unable to clone remote repo from %s',
self._url)
raise
self._remote = self._repo.remote('origin')
def save(self):
"""
Save method to create a new repository with provided information from registration form.
"""
data = self.cleaned_data
if self.edit:
self._update_instance(data)
if self.instance.name != self.repo_entering_name: # rename repository folder if it's name has changed.
rename_tree(Repo.get_repository_location(
self.instance.owner.username, self.repo_entering_name), '{0}.git'.format(self.instance.name))
elif not self.edit:
repository = Repository.objects.create(name=data['name'],
description=data['description'], owner=self.user, private=data['private'])
repository.save()
def __init__(self):
try:
self.repo = Repo('.')
except InvalidGitRepositoryError:
msg = 'Not in a git repository'
raise ConfigurationError(msg, 1)
self._load_config()
self._missing_config = {}
config = {
'repo-name': ('deploy', 'repo-name', None),
'user.name': ('user', 'name', None),
'user.email': ('user', 'email', None),
}
self._register_config(config)
self.register_drivers()
self._check_config()
def __init__(self, org_git_repo_dir, historage_dir, syntax_trees_dir=None):
if org_git_repo_dir:
self.org_repo = Repo(org_git_repo_dir)
self.check_and_make_working_dir(historage_dir)
self.historage_dir = historage_dir
self.use_tempdir = syntax_trees_dir is None
if self.use_tempdir:
self.syntax_trees_dir = mkdtemp()
print(self.syntax_trees_dir)
else:
self.check_and_make_working_dir(syntax_trees_dir)
self.syntax_trees_dir = syntax_trees_dir
self.num_commits = 0
self.is_bare_repo = False
def _clone_repo(temp_dir, repo_url, verifyssl=True, ref='master'):
# Switch to non-interactive mode
os.environ['GIT_TERMINAL_PROMPT'] = '0'
# Disable SSL cert checking if explictly asked
if not verifyssl:
os.environ['GIT_SSL_NO_VERIFY'] = 'true'
# Clone the repo from git; we don't use shallow copying
# because we want the user to work with the repo in the
# future.
repo = Repo.clone_from(repo_url, temp_dir)
# Try to match the reference to a commit hash, a tag, or "master"
gitref = DownloadGitRepoAction._get_gitref(repo, ref)
# Try to match the reference to a "vX.Y.Z" tag
if not gitref and re.match(SEMVER_REGEX, ref):
gitref = DownloadGitRepoAction._get_gitref(repo, "v%s" % ref)
# Try to match the reference to a branch name
if not gitref:
gitref = DownloadGitRepoAction._get_gitref(repo, "origin/%s" % ref)
# Giving up ¯\_(ツ)_/¯
if not gitref:
raise ValueError(
"\"%s\" is not a valid version, hash, tag, or branch in %s." % (ref, repo_url)
)
# We're trying to figure out which branch the ref is actually on,
# since there's no direct way to check for this in git-python.
branches = repo.git.branch('--color=never', '--all', '--contains', gitref.hexsha)
branches = branches.replace('*', '').split()
if 'master' not in branches:
branch = branches[0]
repo.git.checkout('--track', branches[0])
branch = repo.head.reference
else:
branch = 'master'
repo.git.checkout('-B', branch, gitref.hexsha)
return temp_dir
def _prepare(cls, create, **kwargs):
article = super(ArticleFactory, cls)._prepare(create, **kwargs)
path = article.get_path()
if not os.path.isdir(path):
os.makedirs(path, mode=0o777)
man = export_article(article)
repo = Repo.init(path, bare=False)
repo = Repo(path)
f = open(os.path.join(path, 'manifest.json'), "w")
f.write(json_writer.dumps(man, indent=4, ensure_ascii=False).encode('utf-8'))
f.close()
f = open(os.path.join(path, article.text), "w")
f.write(u'Test')
f.close()
repo.index.add(['manifest.json', article.text])
cm = repo.index.commit("Init Article")
article.sha_draft = cm.hexsha
return article
def main():
from argparse import ArgumentParser
parser = ArgumentParser(description=__doc__)
parser.add_argument('anr_repo', help='Path to the OCTN ANR plugin repository.')
parser.add_argument('cards_path', help="Path to store cards metadata in.")
args = parser.parse_args()
try:
mkdir(args.cards_path)
except OSError as e:
if e.errno != 17: #17 is file exists
raise
now = time()
r = Repo(args.anr_repo)
r.remotes.origin.pull()
octgn_sha = r.head.commit.hexsha
r = Request(NETRUNNERCARDS_URL)
try:
with open(join(args.cards_path, 'cards.json'), 'r') as f:
data = json_load(f)
if 'octgn_sha' in data and data['octgn_sha'] == octgn_sha:
r.add_header("If-Modified-Since", format_date_time(json_load(f)['modified_since']))
except IOError as e:
if e.errno != 2:
raise
try:
cards = json_loads(urlopen(r).read())
except HTTPError as e:
if e.code != 304:
raise
card_sets_path = join(args.anr_repo, 'o8g', 'Sets')
card_octgn_data = {}
for path in walk(card_sets_path):
if len(path[-1]) > 0 and path[-1][0] == 'set.xml' and not path[0].endswith('Markers'):
set_data = octgnSetToDict(join(card_sets_path, path[0], path[-1][0]))
for card in set_data['cards'].values():
card['set_id'] = set_data['id']
card['game_id'] = set_data['gameid']
card_octgn_data.update(set_data['cards'])
for card in cards:
renameCardKeys(card)
print card['name'].encode('utf8')
octgn_card = None
try:
octgn_card = card_octgn_data[card['card_id']]
except KeyError as e:
print 'EE', card['card_id'], card['set_name']
continue
card['set_id'] = octgn_card['set_id']
card['game_id'] = octgn_card['game_id']
card['id'] = octgn_card['id']
card_image_path = join(args.cards_path, split(card['imagesrc'])[-1])
if not exists(card_image_path):
try:
image_src = urlopen(NETRUNNERCARDS_BASE + card['imagesrc'])
with open(card_image_path, 'w') as f:
f.write(image_src.read())
except HTTPError as e:
print 'ERROR: image missing for card #', card['id']
cards = {'modified_since': now, 'octgn_sha': octgn_sha, 'cards': cards}
with open(join(args.cards_path, 'cards.json'), 'w') as f:
json_dump(cards, f)
