def is_repository(directory):
"""Checks if the repository is a VCS repository"""
try:
pygit2.discover_repository(path_join(directory))
return True
except KeyError:
return False
def init_repository(url=None):
"""Creates a new Gitless's repository in the cwd.
Args:
url: if given the local repository will be a clone of the remote repository
given by this url.
"""
cwd = os.getcwd()
try:
pygit2.discover_repository(cwd)
raise GlError('You are already in a Gitless repository')
except KeyError: # Expected
if not url:
repo = pygit2.init_repository(cwd)
# We also create an initial root commit
git.commit(allow_empty=True, m='Initialize repository')
return repo
try:
git.clone(url, cwd)
except ErrorReturnCode as e:
raise GlError(stderr(e))
# We get all remote branches as well and create local equivalents
repo = Repository()
remote = repo.remotes['origin']
for rb in (remote.lookup_branch(bn) for bn in remote.listall_branches()):
if rb.branch_name == 'master':
continue
new_b = repo.create_branch(rb.branch_name, rb.head)
new_b.upstream = rb
return repo
def analyze(url, api_token):
github_url = url.strip().replace("http://www.github.com/", "")
github_url = github_url.strip().replace("https://www.github.com/", "")
github_url = github_url.strip().replace("http://github.com/", "")
github_url = github_url.strip().replace("https://github.com/", "")
print("Start analyzing: %s" % github_url)
start_time = time.perf_counter()
g = Github(api_token)
repo = g.get_repo(github_url)
stats = open(os.path.join("github", repo.owner.login + "-" + repo.name), "w")
stats.write("forks;branches;watchers;stars\n")
stats.write("%d;%d;%d;%d\n"
% (repo.forks_count, len(list(repo.get_branches())),
repo.watchers_count, repo.stargazers_count))
stats.flush()
stats.close()
repo_path = os.path.join(os.getcwd(), "git", repo.owner.login, repo.name)
# initialize repo
if os.path.exists(repo_path):
cloned_repo = Repository(discover_repository(repo_path))
else:
cloned_repo = clone_repository(repo.clone_url, repo_path)
# run churny
run("churny " + repo_path, os.path.join("overall", repo.owner.login + "-" + repo.name))
run("churny -m " + repo_path, os.path.join("monthly", repo.owner.login + "-" + repo.name))
elapsed_time = time.perf_counter() - start_time
print("Stop analyzing: %s (%d s)" % (github_url, elapsed_time))
def clone(location):
"""
Clone a Git repository if it doesn't exist into a temporary directory.
:param str location: the Git URL to clone
:rtype: pygit2.Repository
"""
path = join(config.CLONE_ROOT, _simple_location_name(location))
log.info('Cloning to path {}'.format(path))
try:
log.info('Looking for existing repository')
repository_path = discover_repository(path)
except KeyError:
log.info('No existing repository found, cloning fresh')
return clone_repository(location, path)
else:
log.info('Repository already exists at {}'.format(repository_path))
repository = Repository(repository_path)
remote = repository.remotes[0]
remote.sideband_progress = log.info
log.info('Fetching to get the latest objects from remote')
remote.fetch()
return repository
def GitableDir(path):
"""
A directory in which you can do Git things -- i.e. a git
repository or a subdirectory of a checked_out repository.
Parameters:
path (string): A path name. \"~\"-expansion will be performed.
Returns:
Tuple (supplied path name, absolute path name) upon success
Raises:
argparse.ArgumentTypeError if no Git repository is found
"""
realpath=os.path.abspath(os.path.expanduser(path))
if not os.path.isdir(realpath):
err_str = "'{}' does not exist or is not a directory".format(path)
raise argparse.ArgumentTypeError(err_str)
try:
repo_path=pygit2.discover_repository(realpath)
return(path,repo_path)
except KeyError, e:
err_str = "No Git repo found in {} or its parent dirs".format(e.args[0])
raise argparse.ArgumentTypeError(err_str)
def __init__(self, repo_url, dir_manager, credentials_manager):
"""
:repo_url: URL of the repository
:repo_dir: directory where the repository is expected to reside
"""
self.up2date = False
self.repo_url = repo_url
dir_path = dir_manager.get_repo_dir(repo_url)
my_credentials = credentials_manager.get_credentials(repo_url)
if not os.path.isfile(dir_path + '/HEAD'):
try:
self.repo = pygit2.clone_repository(repo_url, dir_path, bare=True, credentials=my_credentials)
except pygit2.GitError as e:
raise GitException("Cloning failed. {}".format(e.message))
else:
self.repo = pygit2.Repository(pygit2.discover_repository(dir_path))
self.up2date = True
def _remote_credentials(url, username_from_url, allowed_types):
return credentials_manager.get_credentials(url)
for remote in self.repo.remotes:
remote.credentials = _remote_credentials
transfer_progress = remote.fetch()
if transfer_progress.received_objects:
self.up2date = False
def from_cwd(cls):
"""Get the Repository object implied by the current directory."""
try:
repo_path = pygit2.discover_repository(os.getcwd())
except KeyError:
raise NotARepository("current directory not part of a repository")
return cls(repo_path)
def dendrifier_for_path(dirname, _ceiling_dir_for_testing='', report_to_stdout=False):
try:
repo = git.discover_repository(dirname, False, _ceiling_dir_for_testing)
except KeyError:
raise ValueError('could not find git repo starting from {}'.format(dirname))
kwargs = {'report': dendrify.ReportToStdout()} if report_to_stdout else {}
return dendrify.Dendrifier(repo, **kwargs)
def init(self):
if pygit2 is None:
self.text = glyphs.BRANCH + ' ' + glyphs.WRITE_ONLY
self.bg = colors.background(colors.RED)
self.fg = colors.foreground(colors.WHITE)
return
try:
repo_path = pygit2.discover_repository(os.getcwd())
self.repo = pygit2.Repository(repo_path)
except Exception:
self.active = False
return
branch_name = self.get_branch_name()
if not branch_name:
self.active = False
return
git_colors = self.get_working_dir_status_decorations()
self.bg = colors.background(git_colors[0])
self.fg = colors.foreground(git_colors[1])
current_commit_text = self.get_current_commit_decoration_text()
self.text = (glyphs.BRANCH + ' ' + branch_name + ' ' + current_commit_text).strip()
def get_repo(path):
path = os.path.abspath(path)
os.chdir(path)
try:
repo_path = pygit2.discover_repository(path)
return pygit2.Repository(repo_path)
except KeyError:
print("Cannot find a repository in '{}'.".format(path))
exit(1)
def _discover_repository(where, *, follow_links=False):
try:
repo_path = pygit2.discover_repository(str(where), follow_links)
except KeyError:
# KeyError is GIT_ENOTFOUND, which in this case means this doesn't
# appear to be a repository
raise NoSuchRepository
else:
return Path(repo_path)
def __init__(self):
try:
self.repo_dir = git.discover_repository('.')
except KeyError:
raise NotInRepoError()
self.repo = git.Repository(self.repo_dir)
self.commit = self.repo.head.target.hex
self.basedir = self.repo.workdir
def __init__(self, context, repo_path=None):
self.context = context
rp = IStorageInfo(context).path
try:
self.repo = Repository(discover_repository(rp))
except KeyError:
# discover_repository may have failed.
raise PathNotFoundError('repository does not exist at path')
self.checkout() # defaults to HEAD.
def _vcs_root(self, path):
"""Find the root of the deepest repository containing path."""
# Assume that nothing funny is going on; in particular, that we aren't
# dealing with a bare repo.
gitdir = _pygit2.discover_repository(path)
self._pygit_repository = _pygit2.Repository(gitdir)
dirname,tip = os.path.split(gitdir)
if tip == '': # split('x/y/z/.git/') == ('x/y/z/.git', '')
dirname,tip = os.path.split(dirname)
assert tip == '.git', tip
return dirname
def set_repository(value):
from pygit2 import discover_repository, Repository
global _repository
if value is None:
_repository = None
return
try:
path = discover_repository(value)
except KeyError:
raise GitError('no repository found in "{}"'.format(value))
_repository = Repository(path)
def __init__(self):
"""Create a Repository out of the current working repository."""
try:
path = pygit2.discover_repository(os.getcwd())
except KeyError:
raise NotInRepoError('You are not in a Gitless\'s repository')
self.git_repo = pygit2.Repository(path)
self.remotes = RemoteCollection(self.git_repo.remotes, self)
self.path = self.git_repo.path
self.root = self.path[:-6] # strip trailing /.git/
self.config = self.git_repo.config
def add_snapshot(self, path):
try:
repo = git.discover_repository(path)
except KeyError:
pass
else:
raise KeyError("path `{0}' already belongs to a git repository: {1}".format(path, repo))
# TODO warning message: shared repo
stat = subprocess.call(["git", "clone", "--shared", self.repo.workdir, path])
assert stat == 0
self.snapshots.append(os.path.realpath(path))
def do_git(self, update = False):
# TODO: Write support for url@branch or url@sha
if (not (os.path.exists(self.path))):
repo = pygit2.clone_repository(self.url[4:], self.path)
else:
repo = pygit2.Repository(pygit2.discover_repository(self.path))
if (update):
repo.remotes[0].fetch()
repo.create_reference('refs/remotes/origin/master', 'refs/heads/master', True)
repo.checkout_head()
return repo
def __init__(self, path):
"""
Initialize the Repository instance in the given path. The directory
under path has to contain .git.
@type path: str
@raise NotAGitRepoError: The given path is not a git directory.
"""
self.path = path
try:
gitdir = pygit2.discover_repository(path)
self.repo = pygit2.Repository(gitdir)
except KeyError:
raise NotAGitRepoError("Not a git repository: %s" % path)
def clone_or_update_repo(github_repo, account, target_path):
if not os.path.exists(target_path + '/.git/'):
try:
return pygit2.clone_repository(github_repo.clone_url, target_path,
credentials=account.credentials)
except pygit2.GitError as e:
raise GitException("Cloning failed. {}".format(e.message))
else:
repo = pygit2.Repository(pygit2.discover_repository(target_path))
for remote in repo.remotes:
remote.credentials = _credential_lambda(account)
transfer_progress = remote.fetch()
return repo
def repo_path():
"""
Get the path to the git_dir of the mainline linux git repository to use.
Typically obtained from the LINUX_GIT environment variable.
"""
try:
search_path = subprocess.check_output(
os.path.join(libdir(), "..",
"linux_git.sh")).decode().strip()
except subprocess.CalledProcessError:
print("Error: Could not determine mainline linux git repository path.",
file=sys.stderr)
sys.exit(1)
return pygit2.discover_repository(search_path)
def stream(repo_uri, stream_uri, verbose, assume, sort, before=None, after=None):
"""Stream git history policy changes to destination.
Default stream destination is a summary of the policy changes to stdout, one
per line. Also supported for stdout streaming is `jsonline`.
AWS Kinesis and SQS destinations are specified by providing the ARN.
Database destinations are supported by providing a sqlalchemy DSN. Note
SQLAlchemy and db drivers must be installed separately as they an optional
dependency.
When using database destinations, streaming defaults to incremental.
"""
logging.basicConfig(
format="%(asctime)s: %(name)s:%(levelname)s %(message)s",
level=(verbose and logging.DEBUG or logging.INFO))
logging.getLogger('botocore').setLevel(logging.WARNING)
if before:
before = parse(before)
if after:
after = parse(after)
if sort:
sort = six.moves.reduce(operator.or_, [SORT_TYPE[s] for s in sort])
with contextlib.closing(TempDir().open()) as temp_dir:
if repo_uri is None:
repo_uri = pygit2.discover_repository(os.getcwd())
log.debug("Using repository %s", repo_uri)
if repo_uri.startswith('http') or repo_uri.startswith('git@'):
log.info("Cloning repository: %s", repo_uri)
repo = pygit2.clone_repository(repo_uri, temp_dir.path)
else:
repo = pygit2.Repository(repo_uri)
load_resources()
policy_repo = PolicyRepo(repo_uri, repo)
change_count = 0
with contextlib.closing(transport(stream_uri, assume)) as t:
if after is None and isinstance(t, IndexedTransport):
after = t.last()
for change in policy_repo.delta_stream(after=after, before=before):
change_count += 1
t.send(change)
log.info("Streamed %d policy repo changes", change_count)
return change_count
def diff(repo_uri, source, target, output, verbose):
"""Policy diff between two arbitrary revisions.
Revision specifiers for source and target can use fancy git refspec syntax
for symbolics, dates, etc.
See: https://git-scm.com/book/en/v2/Git-Tools-Revision-Selection
Default revision selection is dependent on current working tree
branch. The intent is for two use cases, if on a non-master branch
then show the diff to master. If on master show the diff to
previous commit on master. For repositories not using the
`master` convention, please specify explicit source and target.
"""
logging.basicConfig(
format="%(asctime)s: %(name)s:%(levelname)s %(message)s",
level=(verbose and logging.DEBUG or logging.INFO))
logging.getLogger('botocore').setLevel(logging.WARNING)
if repo_uri is None:
repo_uri = pygit2.discover_repository(os.getcwd())
repo = pygit2.Repository(repo_uri)
load_resources()
# If on master show diff between last commit to current head
if repo.head.shorthand == 'master':
if source is None:
source = 'master@{1}'
if target is None:
target = 'master'
# Else show difference between master and current head
elif target is None:
target = repo.head.shorthand
if source is None:
source = 'master'
policy_repo = PolicyRepo(repo_uri, repo)
changes = list(policy_repo.delta_commits(
repo.revparse_single(source), repo.revparse_single(target)))
output.write(
yaml.safe_dump({
'policies': [c.policy.data for c in changes
if c.kind != ChangeType.REMOVE]}).encode('utf8'))
def run_job(job):
logging.debug(job.body)
cmdline = job.body.split()
parse_cmd = ArgumentParser(add_help=False)
parse_cmd.add_argument('-c', '--cmd')
cmd_args, unknown_args = parse_cmd.parse_known_args(cmdline)
if cmd_args.cmd == 'gmpush':
import pygit2
import pygit2.remote
parse_mpush = ArgumentParser(add_help=False, parents=[parse_cmd])
parse_mpush.add_argument('-p', '--pwd')
parse_mpush.add_argument('--rawcmdline') # base64，否则冲突
mpush_args, unknown_args = parse_mpush.parse_known_args(cmdline)
logging.info('mpush parse args done:' + str(mpush_args) + str(unknown_args))
cwd = os.getcwd()
os.chdir(mpush_args.pwd)
rpath = pygit2.discover_repository('.')
repo = pygit2.Repository(rpath)
print(rpath, repo)
def dump_repo(repo):
print('path:' + repo.path)
print('workdir:' + repo.workdir)
print('is_bare:' + str(repo.is_bare))
print('is_empty:' + str(repo.is_empty))
print('remotes: ' + str(repo.remotes))
for remote in repo.remotes:
print(' %s=%s' % (remote.name, remote.url))
print('')
return
dump_repo(repo)
os.chdir('/tmp')
else:
logging.debug('unknown task.')
return
def run():
"""Run the gitgraph main app."""
if len(sys.argv) != 2:
usage()
sys.exit(1)
path = sys.argv[1]
try:
gitpath = pygit2.discover_repository(path)
except KeyError:
msg = 'git repository not found in {}, aborting.'.format(path)
print(msg, file=sys.stderr)
sys.exit(1)
repo = pygit2.Repository(gitpath)
data = retrieve_repo_activity(repo)['continuous_days_commits']
draw_activity([day for day in data])
def repo_clone(repo):
"""Clone repo or checkout last changes on branch
:param repo: (Repository) object with
"url", "path" and "branch" attributes
"""
# TODO: if exists
if os.path.exists(repo.path):
repo_path = pygit2.discover_repository(repo.path)
repo_obj = pygit2.Repository(repo_path)
repo_obj.remotes[0].fetch()
repo_obj.checkout_head()
return
return pygit2.clone_repository(
repo.url,
repo.path,
checkout_branch=repo.branch
)
def main(init=True):
# public, pseudo_private = my_global_id()
# username = pseudo_private or public
# password = Keyring().get_password(r'github.com', username)
# # github_create(username, password)
# repo = Path().absolute().name
# url = r'https://github.com/{}/{}'.format(username, repo)
victim = Path(__file__).with_name(r'victim')
str_victim = str(victim)
if init:
_init(victim, str_victim)
repoobj = Repository(discover_repository(str_victim))
# try:
# repoobj.create_remote(r'origin', url)
# except GitError:
# pass
# repoobj.create_commit(r'refs/heads/master')
tree = _tree(repoobj, victim)
_commit(repoobj, tree)
def __init__(self, path=None):
"""
:param string path: Optional. Defines a the path of the git
repository. If not specified, defaults to the current working
directory.
"""
if pygit2 is None:
logger.warning("without pylibgit2 installed, falling back "
"to wrapped git repo implementation.")
return libgiza.git.GitRepo(path)
if path is None:
self.path = os.getcwd()
else:
self.path = path
self.path = pygit2.discover_repository(self.path)
self.repo = pygit2.Repository(self.path)
logger.debug("created git repository management object for {0}".format(self.path))
def commit_changes(package, message):
repo = pygit2.Repository(pygit2.discover_repository(getcwd()))
repo.index.add(path.join('versions', package + '.json'))
repo.index.write()
tree = repo.index.write_tree()
config = pygit2.Config.get_global_config()
name = list(config.get_multivar('user.name'))[0].encode('utf-8')
email = list(config.get_multivar('user.email'))[0].encode('utf-8')
author = pygit2.Signature(name, email)
repo.create_commit(
repo.head.name,
author,
author,
'[Reaver Project CI Scripts/version] ' + message,
tree,
[ repo.head.get_object().oid ]
)
def __init__(self, directory, url=None, default_branch='master'):
try:
self.directory = pygit2.discover_repository(path_join(directory))
self._repo = pygit2.Repository(directory)
except KeyError as e:
if url != None:
self._repo = None
self.url = url
else:
raise ValueError("Repository not found at {}".format(directory))
if self._repo != None and not hasattr(self, 'url'):
remote = [r for r in self._repo.remotes if r.name == 'origin']
if len(remote) < 1:
remote = self._repo.remotes[0]
else:
remote = remote[0]
self.url = remote.url
self.directory = self._repo.workdir
self.default_branch = default_branch
super(GitRepository, self).__init__(directory)
def lambda_handler(event, context):
print json.dumps(event)
keybucket = event['context']['key-bucket']
outputbucket = event['context']['output-bucket']
pubkey = event['context']['public-key']
### Source IP ranges to allow requests from, if the IP is in one of these the request will not be chacked for an api key
ipranges = []
for i in event['context']['allowed-ips'].split(','):
ipranges.append(ip_network(u'%s' % i))
### APIKeys, it is recommended to use a different API key for each repo that uses this function
apikeys = event['context']['api-secrets'].split(',')
ip = ip_address(event['context']['source-ip'])
secure = False
for net in ipranges:
if ip in net:
secure = True
if 'X-Gitlab-Token' in event['params']['header'].keys():
if event['params']['header']['X-Gitlab-Token'] in apikeys:
secure = True
if 'X-Git-Token' in event['params']['header'].keys():
if event['params']['header']['X-Git-Token'] in apikeys:
secure = True
if 'X-Gitlab-Token' in event['params']['header'].keys():
if event['params']['header']['X-Gitlab-Token'] in apikeys:
secure = True
if 'X-Hub-Signature' in event['params']['header'].keys():
for k in apikeys:
if hmac.new(
str(k), str(event['context']['raw-body']),
hashlib.sha1).hexdigest() == str(
event['params']['header']['X-Hub-Signature'].replace(
'sha1=', '')):
secure = True
if not secure:
logger.error('Source IP %s is not allowed' %
event['context']['source-ip'])
raise Exception('Source IP %s is not allowed' %
event['context']['source-ip'])
try:
repo_name = event['body-json']['project']['path_with_namespace']
except:
repo_name = event['body-json']['repository']['full_name']
try:
remote_url = event['body-json']['project']['git_ssh_url']
except:
try:
remote_url = 'git@' + event['body-json']['repository'][
'links']['html']['href'].replace('https://', '').replace(
'/', ':', 1) + '.git'
except:
remote_url = event['body-json']['repository']['ssh_url']
repo_path = '/tmp/%s' % repo_name
creds = RemoteCallbacks(credentials=get_keys(keybucket, pubkey), )
try:
repository_path = discover_repository(repo_path)
repo = Repository(repository_path)
logger.info('found existing repo, using that...')
except:
logger.info('creating new repo for %s in %s' % (remote_url, repo_path))
repo = create_repo(repo_path, remote_url, creds)
pull_repo(repo, remote_url, creds)
zipfile = zip_repo(repo_path, repo_name)
s3_tags = dict(
source_revision=event['body-json']['head_commit']['id'],
source_html_url=event['body-json']['repository']['html_url'],
source_ref=event['body-json']['ref'])
push_s3(zipfile, repo_name, outputbucket, s3_tags)
if cleanup:
logger.info('Cleanup Lambda container...')
shutil.rmtree(repo_path)
shutil.rm(zipfile)
shutil.rm('/tmp/id_rsa')
shutil.rm('/tmp/id_rsa.pub')
return 'Successfully updated %s' % repo_name
parser = argparse.ArgumentParser(
description="Sort input lines according to the upstream order of "
"commits that each line represents, with the first word on the line "
"taken to be a commit id.")
parser.add_argument("-d",
"--dump-heads",
action="store_true",
help="Print the branch heads used for sorting "
"(debugging).")
args = parser.parse_args()
try:
path = os.environ["GIT_DIR"]
except KeyError:
try:
path = pygit2.discover_repository(os.getcwd())
except KeyError:
print("Error: Not a git repository", file=sys.stderr)
sys.exit(1)
repo = pygit2.Repository(path)
if args.dump_heads:
needs_rebuild = False
try:
with Cache() as cache:
try:
print("Cached heads (version %d):" % cache["version"])
except CKeyError:
print("No usable cache")
needs_rebuild = True
else:
merged_map = index_map
else:
# Entry in HEAD, merge the index over it
print(f"Merging map: {path}")
assert not (status & pygit2.GIT_STATUS_INDEX_NEW)
head_map = dmm.DMM.from_bytes(head_blob.read_raw())
merged_map = merge_map(index_map, head_map)
# write to the index
blob_id = repo.create_blob(merged_map.to_bytes())
repo.index.add(pygit2.IndexEntry(path, blob_id, index_entry.mode))
changed += 1
# write to the working directory if that's clean
if status & (pygit2.GIT_STATUS_WT_DELETED
| pygit2.GIT_STATUS_WT_MODIFIED):
print(
f"Warning: {path} has unindexed changes, not overwriting them"
)
else:
merged_map.to_file(os.path.join(repo.workdir, path))
if changed:
repo.index.write()
print(f"Merged {changed} maps.")
return 0
if __name__ == '__main__':
exit(main(pygit2.Repository(pygit2.discover_repository(os.getcwd()))))
def local_repo_from_repo_path(repo_path):
"""return pygit2 repo object from a repo_path"""
return pygit2.Repository(pygit2.discover_repository(repo_path))
def test_discover_repo(self):
repo = init_repository(self._temp_dir, False)
subdir = os.path.join(self._temp_dir, "test1", "test2")
os.makedirs(subdir)
assert repo.path == discover_repository(subdir)
def test_discover_repo_aspath(self):
repo = init_repository(Path(self._temp_dir), False)
subdir = Path(self._temp_dir) / "test1" / "test2"
os.makedirs(subdir)
assert repo.path == discover_repository(subdir)
def test_discover_repo_not_found(self):
assert discover_repository(tempfile.tempdir) is None
def export(self, extra_attr=[], controls={}):
'''
export the tree, sequences, frequencies to json files for visualization
in the browser
'''
prefix = self.build_data_path
# export json file that contains alignment diversity column by column
self.seqs.export_diversity(prefix + 'entropy.json')
# exports the tree and the sequences inferred for all clades in the tree
if hasattr(self, 'tree') and self.tree is not None:
self.tree.export(path=prefix,
extra_attr=extra_attr +
["muts", "aa_muts", "attr", "clade"])
# local function or round frequency estimates to useful precision (reduces file size)
def process_freqs(freq):
return [round(x, 4) for x in freq]
# construct a json file containing all frequency estimate
# the format is region_protein:159F for mutations and region_clade:123 for clades
if hasattr(self, 'pivots'):
freq_json = {'pivots': process_freqs(self.pivots)}
if hasattr(self, 'mutation_frequencies'):
freq_json['counts'] = {
x: list(counts)
for x, counts in self.mutation_frequency_counts.iteritems()
}
for (region,
gene), tmp_freqs in self.mutation_frequencies.iteritems():
for mut, freq in tmp_freqs.iteritems():
label_str = region + "_" + gene + ':' + str(mut[0] +
1) + mut[1]
freq_json[label_str] = process_freqs(freq)
# repeat for clade frequencies in trees
if hasattr(self, 'tree_frequencies'):
for region in self.tree_frequencies:
for clade, freq in self.tree_frequencies[region].iteritems():
label_str = region + '_clade:' + str(clade)
freq_json[label_str] = process_freqs(freq)
# repeat for named clades
if hasattr(self, 'clades_to_nodes') and hasattr(
self, 'tree_frequencies'):
for region in self.tree_frequencies:
for clade, node in self.clades_to_nodes.iteritems():
label_str = region + '_' + str(clade)
freq_json[label_str] = process_freqs(
self.tree_frequencies[region][node.clade])
# write to one frequency json
if hasattr(self, 'tree_frequencies') or hasattr(
self, 'mutation_frequencies'):
write_json(freq_json, prefix + 'frequencies.json', indent=None)
if len(controls):
controls_json = self.make_control_json(controls)
write_json(controls_json, prefix + 'controls.json')
# write out metadata json# Write out metadata
print("Writing out metadata")
meta_json = {}
meta_json["updated"] = time.strftime("X%d %b %Y").replace('X0',
'X').replace(
'X', '')
try:
from pygit2 import Repository, discover_repository
current_working_directory = os.getcwd()
repository_path = discover_repository(current_working_directory)
repo = Repository(repository_path)
commit_id = repo[repo.head.target].id
meta_json["commit"] = str(commit_id)
except ImportError:
meta_json["commit"] = "unknown"
write_json(meta_json, prefix + 'meta.json')
def __init__(self, location):
#
try:
self._repo = Repository(discover_repository())
except KeyError:
self._repo = init_repository()
def discover_repository() -> bool:
return bool(discover_repository(LOCAL_REPO_PATH))
def get_repo_obj(git_workspace_path=None):
if git_workspace_path is None:
git_workspace_path = os.getcwd()
repository_path = pygit2.discover_repository(git_workspace_path)
repo = pygit2.Repository(repository_path)
return repo
parser.add_argument("--blame",
dest='blame',
action='store_true',
default=False,
help='Run git blame on the BLACKLISTs')
parser.add_argument("--runtests",
dest='testrun',
action='store_true',
default=False,
help='Run the tests')
args = parser.parse_args()
testsPath = os.path.abspath(args.path)
print("Looking at tests in {0}".format(testsPath))
repositoryPath = pygit2.discover_repository(testsPath)
repo = None
try:
repo = pygit2.Repository(repositoryPath)
print("Git repo path: {0}".format(repositoryPath))
except:
print("Git repo not found: blame disabled")
print("Platform: {0}".format(args.platform))
def blacklistDirs(path):
for root, _, files in os.walk(path):
for file in files:
if file == "BLACKLIST" and not "selftests/blacklisted" in root:
yield root
import subprocess
import urllib
import re
import cPickle
import cProfile
old_path = sys.argv[1]
new_path = sys.argv[2]
if len(sys.argv) == 4:
hg_map_file = sys.argv[3]
else:
hg_map_file = None
old_repo = pygit2.Repository(pygit2.discover_repository(old_path))
new_repo = pygit2.Repository(pygit2.discover_repository(new_path))
timers = {}
class Timer:
def __init__(self, name):
self.name = name
def __enter__(self):
self.t = time.time()
return self
def __exit__(self, type, value, traceback):
t = time.time() - self.t
timers[self.name] = timers.get(self.name, 0) + t
def lambda_handler(event, context):
keybucket = event['context']['key-bucket']
outputbucket = event['context']['output-bucket']
pubkey = event['context']['public-key']
# Source IP ranges to allow requests from, if the IP is in one of these the request will not be chacked for an api key
ipranges = []
for i in event['context']['allowed-ips'].split(','):
ipranges.append(ip_network(u'%s' % i))
# APIKeys, it is recommended to use a different API key for each repo that uses this function
apikeys = event['context']['api-secrets'].split(',')
ip = ip_address(event['context']['source-ip'])
secure = False
for net in ipranges:
if ip in net:
secure = True
if 'X-Gitlab-Token' in event['params']['header'].keys():
if event['params']['header']['X-Gitlab-Token'] in apikeys:
secure = True
if 'X-Git-Token' in event['params']['header'].keys():
if event['params']['header']['X-Git-Token'] in apikeys:
secure = True
if 'X-Gitlab-Token' in event['params']['header'].keys():
if event['params']['header']['X-Gitlab-Token'] in apikeys:
secure = True
if 'X-Hub-Signature' in event['params']['header'].keys():
for k in apikeys:
k1 = hmac.new(str(k), str(event['context']['raw-body']), hashlib.sha1).hexdigest()
k2 = str(event['params']['header']['X-Hub-Signature'].replace('sha1=', ''))
if k1 == k2:
secure = True
# TODO: Add the ability to clone TFS repo using SSH keys
try:
full_name = event['body-json']['repository']['full_name']
except KeyError:
try:
full_name = event['body-json']['repository']['fullName']
except KeyError:
full_name = event['body-json']['project']['path_with_namespace']
if not secure:
logger.error('Source IP %s is not allowed' % event['context']['source-ip'])
raise Exception('Source IP %s is not allowed' % event['context']['source-ip'])
if('action' in event['body-json'] and event['body-json']['action'] == 'published'):
branch_name = 'tags/%s' % event['body-json']['release']['tag_name']
repo_name = full_name + '/release'
else:
try:
branch_name = 'master'
repo_name = event['body-json']['project']['path_with_namespace']
except:
if 'ref' in event['body-json']:
branch_name = event['body-json']['ref'].replace('refs/heads/', '')
else:
branch_name = 'master'
repo_name = full_name + '/branch/' + branch_name
try:
remote_url = event['body-json']['project']['git_ssh_url']
except Exception:
try:
remote_url = 'git@'+event['body-json']['repository']['links']['html']['href'].replace('https://', '').replace('/', ':', 1)+'.git'
except:
remote_url = event['body-json']['repository']['ssh_url']
repo_path = '/tmp/%s' % repo_name
# 14.09.2018, [email protected]
# evaluate branch
branch_name = event['body-json']['ref'].replace('refs/heads/', '')
logger.info('DevOpsGitlab repo_name: %s' % repo_name)
logger.info('DevOpsGitlab branch_name: %s' % branch_name)
if branch_name == 'develop':
# 04.09.2018, [email protected]
# in case of DevOpsGitlab we use 'Deployment'-Keys
# creds = RemoteCallbacks(credentials=get_keys(keybucket, pubkey), )
creds = None
remote_url = event['body-json']['repository']['git_http_url']
logger.info('DevOpsGitlab base remote_url: %s' % remote_url)
remote_url = 'https://' + get_userpass() + '@' + remote_url.replace('https://', '')
logger.info('DevOpsGitlab clone url: %s' % remote_url)
try:
repository_path = discover_repository(repo_path)
repo = Repository(repository_path)
logger.info('found existing repo, using that...')
except Exception:
logger.info('creating new repo for %s in %s' % (remote_url, repo_path))
repo = create_repo(repo_path, remote_url, creds)
pull_repo(repo, branch_name, remote_url, creds)
zipfile = zip_repo(repo_path, repo_name)
push_s3(zipfile, repo_name, outputbucket)
if cleanup:
logger.info('Cleanup Lambda container...')
shutil.rmtree(repo_path)
os.remove(zipfile)
os.remove('/tmp/id_rsa')
os.remove('/tmp/id_rsa.pub')
returntxt = 'Successfully updated %s' % repo_name
else:
returntxt = 'No action on branch %s' % branch_name
return returntxt
def __init__(self):
self.gitdir = pygit2.discover_repository('.')
self.objmap = os.path.join(self.gitdir, 'objmap')
self.repo = Repository(self.gitdir)
def get_repo_root_from_file_in_repo(path_to_file_in_repo):
git_folder = discover_repository(path_to_file_in_repo)
if not git_folder:
return None
return os.path.dirname(os.path.dirname(git_folder))
def dir_is_repo(gitdir):
"""Check if dir is a git repository."""
return pygit2.discover_repository(gitdir)