def get_bitbucket_users(self):
uname = self.conf['username']
appwd = self.conf['password']
email = self.conf['email']
auth = BasicAuthenticator(uname, appwd, email)
bitbucket = Client(auth)
teams = Team.find_teams_for_role(role="admin", client=bitbucket)
for t in teams:
for m in t.members():
pprint(m)
login = m['nickname']
name = m['display_name']
email = ''
self.init_user(login, name, email)
r = requests.get("{}/users/{}/ssh-keys".format(
BitbucketConn.v1url, login),
auth=(uname, appwd))
if r.status_code == 200:
key_response = r.json()
for keyvalue in key_response['values']:
key = sshpubkeys.SSHKey(keyvalue['key'])
fingerprint = key.hash_md5()
self.user_list[login][PUBKEYS][fingerprint] = keyvalue[
'key']
else:
print(r.status_code)
def find_bitbucket_repository(user, password, email):
bitbucket = Client(
BasicAuthenticator(user, password, email)
)
try:
print("Finding run project in bitbucket...")
repository = Repository.find_repository_by_name_and_owner(
repository_name="run",
client=bitbucket
)
except HTTPError:
print("Project not found")
print("Creating project run...")
repository = Repository.create(
payload=RepositoryPayload({
"name": "run",
"is_private": True,
"fork_policy": RepositoryForkPolicy.NO_FORKS,
}),
client=bitbucket
)
for link in repository.links["clone"]:
if link["name"] == "https":
return link["href"]
def connect(self):
if self._privatekey and ':' in self._privatekey:
login, password = self._privatekey.split(':')
else:
login = self._username
password = self._privatekey
if not login or not password:
raise ConnectionError('Could not connect to BitBucket. Please configure .gitconfig with your bitbucket credentials.')
auth = BasicAuthenticator(login, password, '*****@*****.**')
self.bb.client.config = auth
self.bb.client.session = self.bb.client.config.session = auth.start_http_session(self.bb.client.session)
try:
_ = self.bb.client.config.who_am_i()
except ResourceError as err:
raise ConnectionError('Could not connect to BitBucket. Not authorized, wrong credentials.') from err
def connect(self):
if self._privatekey and ':' in self._privatekey:
login, password = self._privatekey.split(':')
else:
login = self._username
password = self._privatekey
if not login or not password:
raise ConnectionError('Could not connect to BitBucket. Please configure .gitconfig with your bitbucket credentials.')
auth = BasicAuthenticator(login, password, '*****@*****.**')
self.bb.client.config = auth
self.bb.client.session = self.bb.client.config.session = auth.start_http_session(self.bb.client.session)
try:
_ = self.bb.client.config.who_am_i()
except ResourceError as err:
raise ConnectionError('Could not connect to BitBucket. Not authorized, wrong credentials.') from err
def bitbucket():
from pybitbucket.auth import BasicAuthenticator
return BbClient(
BasicAuthenticator(
BB_USER,
APP_KEYS['bitbucket'],
BB_EMAIL
)
)
def __init__(self, username, userpass, useremail, logger):
"""Get the api routes currently supported by pybitbucket, extend with
with additional routes we need, then create methods for them,
which can be retrieved by calling `relationships()`
"""
self.logger = logger
data = loads(entrypoints_json)
extra_data = loads(extra_entrypoints_json)
data['_links'].update(extra_data['_links'])
BitbucketBase.__init__(self, data=data)
self.client = Client(BasicAuthenticator(username, userpass, useremail))
def __init__(self, username, password, email, owner_username,
repository_name, branch_name):
self.username = username
self.password = password
self.email = email
self.owner_username = owner_username
self.repository_name = repository_name
self.branch_name = branch_name
self.client = Client(BasicAuthenticator(username, password, email))
self._fetch_pull_request()
def init(self, user, password, repo_root, dry_run):
self.dry_run = dry_run
if user and password:
self.client = Client(
BasicAuthenticator(user, password,
'*****@*****.**'))
else:
logging.warn(
"VCS username and password not configured - assuming git executable has appropriate "
"authorization for repo checks")
self.client = None
self.team_root_user = repo_root
self.bitbucket_repo = repository.Repository
def setup(self):
sensor_config = self._config.get('sensor', None)
if not sensor_config:
raise ValueError('"sensor" config value is required')
# validate the format of all repository names
self.targets = sensor_config.get('targets')
if not self.targets:
raise ValueError('"targets" parameter in the "sensor" is required')
if not all(
['repository' in x and 'branches' in x for x in self.targets]):
raise ValueError(
'"repository" and "branches" are mandatory in "sensor.target"')
if not all(
[len(x['repository'].split('/')) == 2 for x in self.targets]):
raise ValueError(
'Invalid repository name is specified in the "targets" parameter'
)
self.TIMEOUT_SECONDS = sensor_config.get('timeout',
self.TIMEOUT_SECONDS)
# initialize global parameter
self.commits = {}
self.service_type = sensor_config.get('bitbucket_type')
if self.service_type == 'server':
# initialization for BitBucket Server
self.client = stashy.connect(
sensor_config.get('bitbucket_server_url'),
self._config.get('username'), self._config.get('password'))
self._init_server_last_commit()
elif self.service_type == 'cloud':
# initialization for BitBucket Cloud
self.client = Client(
BasicAuthenticator(
self._config.get('username'),
self._config.get('password'),
self._config.get('email'),
))
self._init_cloud_last_commit()
else:
raise ValueError('specified bitbucket type (%s) is not supported' %
self.service_type)
self._increment_event_id()
self._logger.info("It's ready to monitor events.")
def _get_repository_data(self):
print("Getting repository info from " + self.host + " ...")
data = None
try:
bitbucket = Client(
BasicAuthenticator('your_username_here',
'your_secret_password_here',
'*****@*****.**'))
for snip in Snippet.find_snippets_for_role(client=bitbucket):
print(snip)
except Exception as e:
print("\nGet repository data error: " + str(e))
raise e
return data
def bitbucket_login():
bitbucket_config = RawConfigParser()
try:
bitbucket_config.read("bitbucket.cfg")
except FileNotFoundError:
print("missing API config file")
exit(1)
try:
user = bitbucket_config.get("bitbucket", "user")
app_password = bitbucket_config.get("bitbucket", "app_password")
email = bitbucket_config.get("bitbucket", "email")
except:
print("error reading Bitbucket API config")
exit(1)
bitbucket = Client(BasicAuthenticator(user, app_password, email))
return bitbucket
def load_packages(packages, username, email, password=None):
"""
Creates BitBucketPackage instances for the given list of packages,
using the given credentials to authenticate requests against the API.
Arguments
---------
packages: list of tuple
tuples are of the format (package_name, owner_name, repository_name)
"""
if password is None:
password = getpass('password for {}@bitbucket: '.format(username))
bitbucket = Client(BasicAuthenticator(username, password, email))
del password
bb_packages = []
for (pkg, owner, repo) in packages:
bb_packages.append(BitBucketPackage(pkg, owner, repo, bitbucket))
return bb_packages
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--clone_dir', default='/tmp')
parser.add_argument('--authfile', default='credentials.json')
args = parser.parse_args()
with open(args.authfile, 'r') as f:
creds = json.load(f)
# log into bitbucket and github
github = Github(creds['github']['auth_token'])
github_user = github.get_user()
# verify github credentials by running a dummy command
try:
data = [(s.name, s.name) for s in github.get_user().get_repos()]
except:
print('Invalid GitHub token!')
exit(1)
bitbucket = Client(
BasicAuthenticator(creds['bitbucket']['user'],
creds['bitbucket']['auth_token'],
creds['bitbucket']['mail']))
# update the git command shell environment with SSH agent stuff
global_git = git.Git()
global_git.update_environment(
**{k: os.environ[k]
for k in os.environ if k.startswith('SSH')})
# list bitbucket repos
repos = list()
failed = list()
for repo in Repository.find_repositories_by_owner_and_role(
role='owner', client=bitbucket):
item = {
'name': repo.name,
'description': repo.description,
'private': repo.is_private,
'link': repo.links['clone'][1]['href'],
'copy': False
}
if repo.scm != 'git':
print(
'Warning: repository {} will be ignored as it does not use git as SCM'
.format(repo.name))
item['reason'] = 'Non-git SCM: {}'.format(repo.scm)
failed.append(item)
continue
repos.append(item)
# ask the user which repos to copy
proceed = False
while not proceed:
print('BitBucket Repositories to copy:')
print(tabulate(repos, headers="keys"))
print(
'Type a name of a repo to toggle it, "all" to toggle all or "go" to proceed with the current selection'
)
while True:
choice = raw_input('Choice [go|all]: ')
if choice == 'go':
proceed = True
break
elif choice == 'all':
for r in repos:
r['copy'] = not r['copy']
break
item = next((item for item in repos if choice == item["name"]),
None)
if item is not None:
item['copy'] = not item['copy']
break
print('{} not found!'.format(choice))
# fliter repos with copy=False
copy_repos = [it for it in repos if it['copy']]
print('Final list to copy:')
print(tabulate(copy_repos, headers="keys"))
# do the copying
for repo in copy_repos:
print('[{}]'.format(repo['name']))
# fetch or clone the bitbucket repository
destdir = os.path.join(args.clone_dir, repo['name'])
if os.path.exists(destdir):
local_repo = git.Repo(destdir)
print('{} exists and is a valid repo, fetching updates'.format(
destdir))
local_repo.remotes.origin.fetch()
else:
print('-- Cloning {}'.format(repo['link']))
try:
local_repo = git.Repo.clone_from(repo['link'], destdir)
except:
print('Clone failed')
repo['reason'] = 'Clone failed'
failed.append(repo)
continue
# try to create the repo on github
try:
print('-- Creating a GitHub repo')
github_user.create_repo(repo['name'],
description=repo['description'],
private=repo['private'])
except GithubException as e:
print(e.data['message'])
repo['reason'] = e.data['message']
failed.append(repo)
continue
github_repo = github_user.get_repo(repo['name'])
# push to github repo
print('-- Pushing')
try:
if not 'github' in local_repo.remotes:
remote = local_repo.create_remote('github',
github_repo.ssh_url)
else:
remote = local_repo.remotes.github
remote.push()
except git.exc.GitCommandError as e:
print('Failed to push')
print(e)
repo['reason'] = 'Push failed'
failed.append(repo)
continue
if len(failed) > 0:
print('Migration failed for: ')
print(tabulate(failed, headers="keys"))
import arrow
from pybitbucket.bitbucket import Client
from pybitbucket.pullrequest import PullRequest
from pybitbucket.repository import Repository
from pybitbucket.auth import BasicAuthenticator
bitbucket = Client(
BasicAuthenticator(
'', # Username
'', # Password/API Key
'', # E-mail
))
def get_user_for_activity(activity):
for value in activity.values():
if 'user' in value:
return value['user']
elif 'author' in value:
return value['author']
repositories = [
repo.slug for repo in Repository.find_repositories_by_owner_and_role(
role='owner', client=bitbucket)
]
for repo in repositories:
print("------- {repo} -------".format(repo=repo))
for pr in PullRequest.find_pullrequests_for_repository_by_state(
repo, client=bitbucket):
def get_bitbucket_pullrequests(client):
bitbucket = Client(
BasicAuthenticator(
client.username, # Username
client.password, # Password/API Key
client.email, # E-mail
))
repositories = bitbucket.remote_relationship(
BITBUCKET_ENDPOINT,
owner=client.username,
q='project.key="ACTIVE"',
)
existing_keys = []
for repo in repositories:
repo = repo.slug
log.error("Repository: %s" % repo)
for pr in PullRequest.find_pullrequests_for_repository_by_state(
repo, client=bitbucket):
if type(pr) == dict:
continue
pull_request_key = "{repo}-{pr_id}".format(repo=repo.upper(),
pr_id=pr.id)
existing_keys.append(pull_request_key)
pull_request = PullRequestModel.objects.filter(
client=client).filter(key=pull_request_key).first()
if not pull_request:
pull_request = PullRequestModel()
pull_request.key = pull_request_key
pull_request.client = client
activity = list(pr.activity())
# Get last update
pull_request.updated_on = arrow.get(pr.updated_on).datetime
pull_request.updated_by = get_user_for_activity(
activity[0])['display_name']
# Get author
pull_request.author = pr.author.display_name
# Get task count
pull_request.task_count = pr.task_count
# Get last build
statuses = list(pr.statuses())
if 'pagelen' in statuses[0]:
statuses.pop()
statuses = sorted(
statuses,
key=lambda s: arrow.get(s['updated_on']).datetime,
reverse=True)
print("Build count:", len(statuses))
pull_request.build_count = len(statuses)
pull_request.last_build = None
if len(statuses):
pull_request.last_build = statuses[0]['state']
pull_request.url = pr.links['html']['href']
pull_request.save()
if pull_request.approvals.count():
pull_request.approvals.all().delete()
# Get approvals
approvals = filter(lambda a: 'approval' in a, activity)
for a in approvals:
approval = PullRequestApproval()
approval.display_name = a['approval']['user']['display_name']
approval.avatar = a['approval']['user']['links']['avatar'][
'href']
approval.pullrequest = pull_request
approval.save()
PullRequestModel.objects.filter(client=client).exclude(
key__in=existing_keys).all().delete()
def __init__(self, user, password):
self.client = Client(
BasicAuthenticator(user, password, '*****@*****.**'))
self.status_str = ''
from pybitbucket.repository import Repository
from pybitbucket.user import User
from requests import HTTPError
from ratelimit import limits, sleep_and_retry
import requests
from src.bitbucket.bitbucket_commit import get_commit_payload
from src.bitbucket.bitbucket_pull_request import get_pull_request_info
from src.bitbucket.bitbucket_ref import get_ref_info
from src.bitbucket.bitbucket_repository import get_repository_info
from src.bitbucket.bitbucket_user import get_user_info
from src.utils import get_logger, get_config
logger = get_logger("BitBucketAPI")
bitbucket_client = BasicAuthenticator(get_config("bitbucket.user_name"),
get_config("bitbucket.app_password"),
get_config("bitbucket.user_email"))
bitbucket_client = Client(bitbucket_client)
BATCH_SIZE = 100
ONE_HOUR_SEC = 3600
USER = {}
def add_remote_relationship_methods(self, data):
for name, url in BitbucketBase.links_from(data):
if (name not in [i.name for i in BitbucketSpecialAction]):
setattr(self, name,
partial(self.client.remote_relationship, template=url))
if not username:
username = raw_input('Enter your bitbucket username: '******'Enter your bitbucket teamname (leave empty if you don\'t need it): ')
if not teamname:
teamname = username
if not password:
password = getpass(prompt='Enter your bitbucket password: '******'Username and password must be provided.')
exit(0)
bitbucket = Client(
BasicAuthenticator(
username,
password,
''
)
)
for repo in Repository.find_repositories_by_owner_and_role(owner=teamname, role=RepositoryRole.MEMBER.value, client=bitbucket):
if repo.scm != RepositoryType.HG.value:
continue
if repo.name.startswith(hgprefix):
continue
hg_clone_https = ''
for clone in repo.links['clone']:
if clone['name'] == 'https':
hg_clone_https = clone['href']
break
repos = Repository.find_repositories_by_owner_and_role(client=client,
role="member")
path.mkdir(parents=True, exist_ok=True)
with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor:
future_to_repo = {
executor.submit(clone_repo, path, repo.clone['https']): repo
for repo in repos
}
for future in concurrent.futures.as_completed(future_to_repo):
repo: str = future_to_repo[future]
try:
data = future.result()
print(f'Cloned: {repo.name} ')
except Exception as exc:
if exc.status == 128:
print(f'{repo.name} already exists')
else:
print(data)
parser = build_arg_parser()
args = parser.parse_args()
bit_bucket = Client(
BasicAuthenticator(password=args.password,
username=args.user,
client_email=args.email))
clone_repos(client=bit_bucket, path=Path(args.target))
def setup_class(cls):
cls.auth = BasicAuthenticator(cls.username,
cls.password,
cls.email,
server_base_uri=cls.server_base_uri)