def repo_cat_file(self, repo_path, commit_hash, path):
(commit_hash, path) = self._all_to_utf8(commit_hash, path)
if not self._path_check_chdir(repo_path, commit_hash, path):
return ''
path = self._get_quote_path(path)
if path.startswith('./'):
path = path[2:]
file_type = path.split('.')[-1]
if file_type in BINARY_FILE_TYPE:
return u'二进制文件'
stage_file = self._get_stage_file(repo_path, commit_hash, path)
result = self._read_load_stage_file(stage_file)
if result is not None:
return result['blob']
command = '/usr/bin/git show %s:%s | /usr/bin/head -c 524288' % (commit_hash, path)
try:
signal.signal(signal.SIGPIPE, signal.SIG_DFL)
result = check_output(command, shell=True)
ud = UniversalDetector()
ud.feed(result)
ud.close()
if ud.result['encoding']:
encoding = ud.result['encoding']
if encoding != 'utf-8' or encoding != 'utf8':
result = result.decode(encoding).encode('utf-8')
self._dumps_write_stage_file({'blob': result}, stage_file)
return result
except Exception, e:
logger.exception(e)
def dropbox_request_token_pair():
dropbox_connection = None
try:
dropbox_connection = httplib.HTTPSConnection('api.dropbox.com',
443,
timeout=10)
params = urllib.urlencode({
'oauth_signature_method': 'PLAINTEXT',
'oauth_timestamp': _timestamp(),
'oauth_nonce': _oauth_nonce(),
'oauth_consumer_key': DROPBOX_APP_KEY,
'oauth_signature': DROPBOX_APP_SECRET + '&'
})
headers = {
'Host': 'api.dropbox.com',
'Content-type': 'application/x-www-form-urlencoded',
'User-Agent': USER_AGENT
}
dropbox_connection.request('POST', API_REQUEST_TOKEN_URL, params,
headers)
response = dropbox_connection.getresponse()
if response.status == 200:
result = response.read()
key_value_dict = urlparse.parse_qs(result)
if 'oauth_token' in key_value_dict and 'oauth_token_secret' in key_value_dict:
return (key_value_dict['oauth_token'][0],
key_value_dict['oauth_token_secret'][0])
except Exception, e:
logger.exception(e)
def team_create(request):
current = 'team'; title = u'设置 / 创建团队'
teamprofileForm = TeamprofileForm()
if request.method == 'POST':
teamprofileForm = TeamprofileForm(request.POST)
if teamprofileForm.is_valid():
username = teamprofileForm.cleaned_data['username']
email = '*****@*****.**' % username
password = '******' % random.getrandbits(64)
team = None
try:
team = User.objects.create_user(username, email, password)
except IntegrityError, e:
logger.exception(e)
teamprofile = teamprofileForm.save(commit=False)
teamprofile.id = team.id
teamprofile.email = email
teamprofile.imgurl = hashlib.md5(team.email.lower()).hexdigest()
teamprofile.is_team_account = 1
teamprofile.save()
teamMember = TeamMember(team_user_id = team.id, user_id = request.user.id, group_id = 0, permission = 2, is_admin = 1)
teamMember.save()
request.userprofile.has_joined_team = 1
request.userprofile.save()
return HttpResponseRedirect('/settings/team/')
def _repo_meta_refs_detail_commit(self, repo, meta, commit_hash_dict):
if not self._chdir(repo.get_abs_repopath()):
return
refs_detail_commit = {}
refses = commit_hash_dict.keys()
try:
last_commit_command = '/bin/bash -c \'for i in %s; do echo -n "$i|"; git log "$i" -1 --pretty="%%h|%%p|%%t|%%an|%%ae|%%at|%%cn|%%ce|%%ct|%%s" | /usr/bin/head -c 524288; done\'' % (' '.join(refses))
signal.signal(signal.SIGPIPE, signal.SIG_DFL)
last_commit_output = check_output(last_commit_command, shell=True)
for line in last_commit_output.split('\n'):
ars = line.split('|', 10)
if len(ars) != 11:
continue
(refs, commit_hash, parent_commit_hash, tree_hash, author_name, author_email, author_date, committer_name, committer_email, committer_date, commit_message) = (ars)
refs_commit = {
'commit_hash': commit_hash,
'parent_commit_hash': parent_commit_hash,
'tree_hash': tree_hash,
'author_name': author_name,
'author_email': author_email,
'author_date': author_date,
'committer_name': committer_name,
'committer_email': committer_email,
'committer_date': committer_date,
'commit_message': commit_message,
}
refs_commit['parent_commit_hash'] = parent_commit_hash.split(' ')
self._set_real_author_committer_name(refs_commit)
refs_detail_commit[refs] = refs_commit
except Exception, e:
logger.exception(e)
def github_oauth_access_token(code):
github_connection = None
try:
github_connection = httplib.HTTPSConnection('github.com',
443,
timeout=10)
params = urllib.urlencode({
'client_id': GITHUB_CLIENT_ID,
'client_secret': GITHUB_CLIENT_SECRET,
'code': code
})
headers = {
'Content-type': 'application/x-www-form-urlencoded',
'Accept': 'application/json',
'User-Agent': USER_AGENT
}
github_connection.request('POST', '/login/oauth/access_token', params,
headers)
response = github_connection.getresponse()
if response.status == 200:
json_str = response.read()
response = json.loads(json_str)
access_token = str(response['access_token'])
return access_token
except Exception, e:
logger.exception(e)
def _do_event(self, event):
username = event['username']
reponame = event['reponame']
remote_git_url = event['remote_git_url']
local_user = GsuserManager.get_user_by_name(username)
local_repo = RepoManager.get_repo_by_name(username, reponame)
if local_user is None or local_repo is None or local_repo.status == 0:
return
local_repo_path = local_repo.get_abs_repopath()
if os.path.exists(local_repo_path):
return
args = ['/bin/bash', '/opt/bin/git-import-remote-repo.sh'
] + [local_repo_path, remote_git_url]
try:
popen = Popen(args, stdout=PIPE, shell=False, close_fds=True)
output = popen.communicate()[0].strip()
returncode = popen.returncode
if returncode == 0:
RepoManager.check_export_ok_file(local_repo)
diff_size = long(output)
RepoManager.update_user_repo_quote(local_user, local_repo,
diff_size)
local_repo.status = 0
local_repo.save()
else:
local_repo.status = 500
local_repo.save()
except Exception, e:
local_repo.status = 500
local_repo.save()
logger.exception(e)
def github_authenticate(thirdpartyUser):
tp_id, tp_username, tp_email, github_user_info = thirdpartyUser.tp_id, thirdpartyUser.tp_username, thirdpartyUser.tp_email, thirdpartyUser.github_user_info
thirdpartyUser_find = GsuserManager.get_thirdpartyUser_by_type_tpId(ThirdpartyUser.GITHUB, tp_id)
if thirdpartyUser_find is not None:
if thirdpartyUser_find.access_token != thirdpartyUser.access_token:
thirdpartyUser_find.access_token = thirdpartyUser.access_token
thirdpartyUser_find.save()
user_id = thirdpartyUser_find.id
user = GsuserManager.get_user_by_id(user_id)
return user
username = __get_uniq_username(tp_username)
email = __get_uniq_email(tp_email)
password = __get_random_password()
if username is None or email is None or password is None:
return None
create_user = None
try:
create_user = User.objects.create_user(username, email, password)
if create_user is not None and create_user.is_active:
userprofile = Userprofile(username = create_user.username, email = create_user.email, imgurl = hashlib.md5(create_user.email.lower()).hexdigest())
_fill_github_user_info(userprofile, github_user_info)
userprofile.id = create_user.id
userprofile.save()
if username == tp_username and email == tp_email:
thirdpartyUser.init = 1
thirdpartyUser.user_type = ThirdpartyUser.GITHUB
thirdpartyUser.id = create_user.id
thirdpartyUser.save()
except IntegrityError, e:
logger.exception(e)
def team_create(request):
current = 'team'
title = u'设置 / 创建团队'
teamprofileForm = TeamprofileForm()
if request.method == 'POST':
teamprofileForm = TeamprofileForm(request.POST)
if teamprofileForm.is_valid():
username = teamprofileForm.cleaned_data['username']
email = '*****@*****.**' % username
password = '******' % random.getrandbits(64)
team = None
try:
team = User.objects.create_user(username, email, password)
except IntegrityError, e:
logger.exception(e)
teamprofile = teamprofileForm.save(commit=False)
teamprofile.id = team.id
teamprofile.email = email
teamprofile.imgurl = hashlib.md5(team.email.lower()).hexdigest()
teamprofile.is_team_account = 1
teamprofile.save()
teamMember = TeamMember(team_user_id=team.id,
user_id=request.user.id,
group_id=0,
permission=2,
is_admin=1)
teamMember.save()
request.userprofile.has_joined_team = 1
request.userprofile.save()
return HttpResponseRedirect('/settings/team/')
def github_get_thirdpartyUser(access_token):
github_connection = None
try:
thirdpartyUser = ThirdpartyUser()
github_connection = httplib.HTTPSConnection('api.github.com',
443,
timeout=10)
headers = {
'Host': 'api.github.com',
'Accept': 'application/json',
'User-Agent': USER_AGENT
}
github_connection.request('GET', '/user?access_token=' + access_token,
{}, headers)
response = github_connection.getresponse()
if response.status == 200:
json_str = response.read()
github_user_info = json.loads(json_str)
if 'login' in github_user_info:
thirdpartyUser.tp_username = github_user_info['login']
if 'id' in github_user_info:
thirdpartyUser.tp_id = github_user_info['id']
if 'email' in github_user_info:
thirdpartyUser.tp_email = github_user_info['email']
thirdpartyUser.init = 0
thirdpartyUser.access_token = access_token
thirdpartyUser.user_type = ThirdpartyUser.GITHUB
thirdpartyUser.github_user_info = github_user_info
return thirdpartyUser
except Exception, e:
logger.exception(e)
def github_authenticate(thirdpartyUser):
tp_id, tp_username, tp_email, github_user_info = thirdpartyUser.tp_id, thirdpartyUser.tp_username, thirdpartyUser.tp_email, thirdpartyUser.github_user_info
thirdpartyUser_find = GsuserManager.get_thirdpartyUser_by_type_tpId(
ThirdpartyUser.GITHUB, tp_id)
if thirdpartyUser_find is not None:
if thirdpartyUser_find.access_token != thirdpartyUser.access_token:
thirdpartyUser_find.access_token = thirdpartyUser.access_token
thirdpartyUser_find.save()
user_id = thirdpartyUser_find.id
user = GsuserManager.get_user_by_id(user_id)
return user
username = __get_uniq_username(tp_username)
email = __get_uniq_email(tp_email)
password = __get_random_password()
if username is None or email is None or password is None:
return None
create_user = None
try:
create_user = User.objects.create_user(username, email, password)
if create_user is not None and create_user.is_active:
userprofile = Userprofile(
username=create_user.username,
email=create_user.email,
imgurl=hashlib.md5(create_user.email.lower()).hexdigest())
_fill_github_user_info(userprofile, github_user_info)
userprofile.id = create_user.id
userprofile.save()
if username == tp_username and email == tp_email:
thirdpartyUser.init = 1
thirdpartyUser.user_type = ThirdpartyUser.GITHUB
thirdpartyUser.id = create_user.id
thirdpartyUser.save()
except IntegrityError, e:
logger.exception(e)
def post_payload(webHookURL, payload):
connection = None
url = webHookURL.url
try:
parseResult = urlparse(url)
scheme = parseResult.scheme
hostname = parseResult.hostname
port = parseResult.port
path = parseResult.path
if not port:
port = 80
if scheme == 'https':
connection = httplib.HTTPSConnection(hostname, port, timeout=10)
else:
connection = httplib.HTTPConnection(hostname, port, timeout=10)
params = urllib.urlencode({'payload': payload})
headers = {
'Content-type': 'application/x-www-form-urlencoded',
'Accept': ACCEPT,
'User-Agent': USER_AGENT
}
connection.request('POST', path, params, headers)
response = connection.getresponse()
if webHookURL.last_return_code != response.status:
webHookURL.last_return_code = response.status
webHookURL.save()
if response.status == 200:
return True
except Exception, e:
logger.exception(e)
def _do_event(self, event):
username = event['username']
reponame = event['reponame']
remote_git_url = event['remote_git_url']
local_user = GsuserManager.get_user_by_name(username)
local_repo = RepoManager.get_repo_by_name(username, reponame)
if local_user is None or local_repo is None or local_repo.status == 0:
return
local_repo_path = local_repo.get_abs_repopath()
if os.path.exists(local_repo_path):
return
args = ['/bin/bash', '/opt/bin/git-import-remote-repo.sh'] + [local_repo_path, remote_git_url]
try:
popen = Popen(args, stdout=PIPE, shell=False, close_fds=True)
output = popen.communicate()[0].strip()
returncode = popen.returncode
if returncode == 0:
RepoManager.check_export_ok_file(local_repo)
diff_size = long(output)
RepoManager.update_user_repo_quote(local_user, local_repo, diff_size)
local_repo.status = 0
local_repo.save()
else:
local_repo.status = 500
local_repo.save()
except Exception, e:
local_repo.status = 500
local_repo.save()
logger.exception(e)
def _ls_tree_check_output(self, commit_hash, path, origin_path):
command = 'git config --global core.quotepath false; /usr/bin/git ls-tree %s -- %s | /usr/bin/head -c 524288' % (commit_hash, path)
tree = {}; dirs = []; files = []; has_readme = False; readme_file = '';
try:
signal.signal(signal.SIGPIPE, signal.SIG_DFL)
raw_output = check_output(command, shell=True)
max = 500
for line in raw_output.split("\n"):
array = self.blank_p.split(line, 3)
if len(array) >= 4:
abs_path = array[3]; relative_path = abs_path
if path != '.':
relative_path = abs_path[len(origin_path):]
if array[1] == 'tree':
relative_path = relative_path + '/'
relative_path = self._oct_utf8_decode(relative_path)
if self._repo_file_path_check(relative_path):
tree[relative_path] = {}
tree[relative_path]['relative_path'] = relative_path
tree[relative_path]['type'] = array[1]
tree[relative_path]['commit_hash'] = array[2]
tree[relative_path]['abs_path'] = abs_path
filename_lower = relative_path.lower()
if filename_lower == 'readme.md' or filename_lower == 'readme.mkd':
has_readme = True
readme_file = abs_path
if array[1] == 'tree':
dirs.append(relative_path)
else:
files.append(relative_path)
if(max <= 0):
break
max = max - 1
quote_relative_paths = []
for relative_path in tree.keys():
quote_relative_paths.append(pipes.quote(relative_path))
if len(path.split('/')) < 50 and '"' not in origin_path:
pre_path = origin_path
if path == '.':
pre_path = ''
for i in range(0, 5):
sub_length = 1000
sub_quote_relative_paths = quote_relative_paths[i*sub_length:(i+1)*sub_length]
if len(sub_quote_relative_paths) == 0:
break
self._set_last_commit(tree, sub_quote_relative_paths, commit_hash, pre_path)
if len(sub_quote_relative_paths) < sub_length:
break
dirs.sort()
files.sort()
ordered_tree = []
for file_path in dirs + files:
tree_item = tree[file_path]
self._set_real_author_name(tree_item)
ordered_tree.append(tree_item)
return {'tree': ordered_tree, 'has_readme': has_readme, 'readme_file': readme_file}
except Exception, e:
logger.exception(e)
def _create_user_and_authenticate(request, username, email, password, ref_hash, is_verify):
user = None
try:
user = User.objects.create_user(username, email, password)
if user is None or not user.is_active:
return False
except IntegrityError, e:
logger.exception(e)
return False
def _read_load_stage_file(self, stage_file):
if not os.path.exists(stage_file):
return None
try:
with codecs.open(stage_file, encoding='utf-8', mode='r') as json_data:
result = json.load(json_data)
return result
except Exception, e:
logger.exception(e)
def _run_command_on_repo(self, repo, args):
abs_repopath = repo.get_abs_repopath()
if os.path.exists(abs_repopath):
self._chdir(abs_repopath)
try:
popen = Popen(args, stdout=PIPE, shell=False, close_fds=True)
output = popen.communicate()[0].strip()
return popen.returncode == 0
except Exception, e:
logger.exception(e)
def _create_user_and_authenticate(request, username, email, password, ref_hash,
is_verify):
user = None
try:
user = User.objects.create_user(username, email, password)
if user is None or not user.is_active:
return False
except IntegrityError, e:
logger.exception(e)
return False
def github_list_repo(access_token):
github_connection = None
try:
thirdpartyUser = ThirdpartyUser()
github_connection = httplib.HTTPSConnection('api.github.com', 443, timeout=10)
headers = {'Host': 'api.github.com', 'Accept': 'application/json', 'User-Agent': USER_AGENT}
github_connection.request('GET', '/user/repos?type=public&sort=pushed&access_token=' + access_token, {}, headers)
response = github_connection.getresponse()
if response.status == 200:
json_str = response.read()
return json_str
except Exception, e:
logger.exception(e)
def github_oauth_access_token(code):
github_connection = None
try:
github_connection = httplib.HTTPSConnection('github.com', 443, timeout=10)
params = urllib.urlencode({'client_id': GITHUB_CLIENT_ID, 'client_secret': GITHUB_CLIENT_SECRET, 'code': code})
headers = {'Content-type': 'application/x-www-form-urlencoded', 'Accept': 'application/json', 'User-Agent': USER_AGENT}
github_connection.request('POST', '/login/oauth/access_token', params, headers)
response = github_connection.getresponse()
if response.status == 200:
json_str = response.read()
response = json.loads(json_str)
access_token = str(response['access_token'])
return access_token
except Exception, e:
logger.exception(e)
def send_html_mail(self, header, body, sender, receivers):
if not sender:
sender = self.default_sender
smtp = SMTP('localhost', 25)
for receiver in receivers:
try:
msg = MIMEMultipart('alternative')
msg['Subject'] = header
msg['From'] = sender
msg['To'] = receiver
mIMEText = MIMEText(body, 'html', 'utf-8')
msg.attach(mIMEText)
smtp.sendmail(sender, receiver, msg.as_string())
except Exception, e:
logger.exception(e)
def dropbox_request_token_pair():
dropbox_connection = None
try:
dropbox_connection = httplib.HTTPSConnection('api.dropbox.com', 443, timeout=10)
params = urllib.urlencode({'oauth_signature_method': 'PLAINTEXT', 'oauth_timestamp': _timestamp(), 'oauth_nonce': _oauth_nonce(), 'oauth_consumer_key': DROPBOX_APP_KEY, 'oauth_signature': DROPBOX_APP_SECRET + '&'})
headers = {'Host': 'api.dropbox.com', 'Content-type': 'application/x-www-form-urlencoded', 'User-Agent': USER_AGENT}
dropbox_connection.request('POST', API_REQUEST_TOKEN_URL, params, headers)
response = dropbox_connection.getresponse()
if response.status == 200:
result = response.read()
key_value_dict = urlparse.parse_qs(result)
if 'oauth_token' in key_value_dict and 'oauth_token_secret' in key_value_dict:
return (key_value_dict['oauth_token'][0], key_value_dict['oauth_token_secret'][0])
except Exception, e:
logger.exception(e)
def send_html_mail(self, header, body, sender, receivers):
if not sender:
sender = self.default_sender
smtp = SMTP('localhost', 25)
for receiver in receivers:
try:
msg = MIMEMultipart('alternative')
msg['Subject'] = header
msg['From'] = sender
msg['To'] = receiver
mIMEText = MIMEText(body, 'html', 'utf-8')
msg.attach(mIMEText)
smtp.sendmail(sender, receiver, msg.as_string())
except Exception, e:
logger.exception(e)
def _dumps_write_stage_file(self, result, stage_file):
if result is None:
return
timenow = int(time.time())
stage_file_tmp = '%s.%s' % (stage_file, timenow)
stage_file_tmp_path = os.path.dirname(stage_file_tmp)
if not os.path.exists(stage_file_tmp_path):
os.makedirs(stage_file_tmp_path)
try:
with codecs.open(stage_file_tmp, encoding='utf-8', mode='w') as stage_file_w:
stage_file_w.write(json.dumps(result))
stage_file_w.flush()
shutil.move(stage_file_tmp, stage_file)
except Exception, e:
logger.exception(e)
def __get(model, pkid, only_visibly):
table = model._meta.db_table
id_key = __get_idkey(table, pkid)
obj = cache.get(id_key)
if obj is not None:
return obj
try:
obj = model.objects.get(id=pkid)
if not obj:
return None
if only_visibly and obj.visibly != 0:
return None
cache.add(id_key, obj)
return obj
except Exception, e:
logger.exception(e)
def run(self):
beanstalk = beanstalkc.Connection(host=BEANSTALK_HOST, port=BEANSTALK_PORT)
EventManager.switch(beanstalk, IMPORT_REPO_TUBE_NAME)
while True:
event_job = beanstalk.reserve()
try:
event = json.loads(event_job.body)
# exit signal
if event['type'] == -1:
event_job.delete()
sys.exit(0)
self._do_event(event)
except Exception, e:
logger.info('do_event catch except, event: %s' % event_job.body)
logger.exception(e)
event_job.delete()
def __get(model, pkid, only_visibly):
table = model._meta.db_table
id_key = __get_idkey(table, pkid)
obj = cache.get(id_key)
if obj is not None:
return obj
try:
obj = model.objects.get(id=pkid)
if not obj:
return None
if only_visibly and obj.visibly != 0:
return None
cache.add(id_key, obj)
return obj
except Exception, e:
logger.exception(e)
def _set_last_commit(self, tree, sub_quote_relative_paths, commit_hash, pre_path):
try:
signal.signal(signal.SIGPIPE, signal.SIG_DFL)
last_commit_command = 'for i in %s; do echo -n "$i|"; git log %s -1 --pretty="%%at|%%an|%%ae|%%s" -- "%s$i" | /usr/bin/head -c 524288; done' % (' '.join(sub_quote_relative_paths), commit_hash, pre_path)
last_commit_output = check_output(last_commit_command, shell=True)
for last_commit in last_commit_output.split('\n'):
splits = last_commit.split('|', 4)
if len(splits) < 5:
continue
relative_path = splits[0]
tree[relative_path]['author_time'] = splits[1]
tree[relative_path]['author_name'] = splits[2]
tree[relative_path]['author_email'] = splits[3]
tree[relative_path]['last_commit_message'] = splits[4]
except Exception, e:
logger.exception(e)
def merge_pull_request(self, source_repo, desc_repo, source_refs, desc_refs, merge_commit_message):
pullrequest_repo_path = '%s/%s/%s' % (PULLREQUEST_REPO_PATH, desc_repo.username, desc_repo.name)
source_abs_repopath = source_repo.get_abs_repopath()
source_remote_name = '%s-%s' % (source_repo.username, source_repo.name)
dest_abs_repopath = desc_repo.get_abs_repopath()
desc_remote_name = '%s-%s' % (desc_repo.username, desc_repo.name)
action = 'merge'
args = [pullrequest_repo_path, source_abs_repopath, source_remote_name, dest_abs_repopath, desc_remote_name, action, source_refs, desc_refs]
if not self._is_allowed_paths(args):
return (128, '合并失败，请检查是否存在冲突 或者 non-fast-forward')
args = ['/bin/bash', '/opt/bin/git-pullrequest.sh'] + args + [merge_commit_message]
try:
popen = Popen(args, stdout=PIPE, shell=False, close_fds=True)
output = popen.communicate()[0].strip()
return (popen.returncode, output)
except Exception, e:
logger.exception(e)
def start():
logger.info('==================== START forkworker ====================')
beanstalk = beanstalkc.Connection(host=BEANSTALK_HOST, port=BEANSTALK_PORT)
EventManager.switch(beanstalk, FORK_TUBE_NAME)
while True:
event_job = beanstalk.reserve()
try:
event = json.loads(event_job.body)
# exit signal
if event['type'] == -1:
event_job.delete()
sys.exit(0)
do_event(event)
except Exception, e:
logger.info('do_event catch except, event: %s' % event_job.body)
logger.exception(e)
event_job.delete()
def start():
logger.info('==================== START eventworker ====================')
beanstalk = beanstalkc.Connection(host=BEANSTALK_HOST, port=BEANSTALK_PORT)
EventManager.switch(beanstalk, HOOK_TUBE_NAME)
while True:
event_job = beanstalk.reserve()
try:
event = json.loads(event_job.body)
# exit signal
if event['type'] == -1:
event_job.delete()
sys.exit(0)
do_event(event)
except Exception, e:
logger.error('do_event catch except, event: %s' % event_job.body)
logger.exception(e)
event_job.delete()
def prepare_pull_request(self, source_repo, desc_repo):
pullrequest_repo_path = '%s/%s/%s' % (PULLREQUEST_REPO_PATH, desc_repo.username, desc_repo.name)
source_abs_repopath = source_repo.get_abs_repopath()
source_remote_name = '%s-%s' % (source_repo.username, source_repo.name)
dest_abs_repopath = desc_repo.get_abs_repopath()
desc_remote_name = '%s-%s' % (desc_repo.username, desc_repo.name)
action = 'prepare'
args = [pullrequest_repo_path, source_abs_repopath, source_remote_name, dest_abs_repopath, desc_remote_name, action]
if not self._is_allowed_paths(args):
return False
args = ['/bin/bash', '/opt/bin/git-pullrequest.sh'] + args
try:
popen = Popen(args, stdout=PIPE, shell=False, close_fds=True)
output = popen.communicate()[0].strip()
return popen.returncode == 0
except Exception, e:
logger.exception(e)
def repo_log_graph(self, repo, repo_path, commit_hash):
if not self._path_check_chdir(repo_path, commit_hash, '.') or not re.match('^\w+$', commit_hash):
return {}
stage_file = self._get_stage_file(repo_path, commit_hash, repo.last_push_time)
stage_file = stage_file + '.lg'
log_graph = self._read_load_stage_file(stage_file)
if log_graph is not None:
return log_graph
command = '/usr/bin/git log -100 --graph --abbrev-commit --date=relative --format=format:"%%h - (%%ar) %%s - %%an%%d" %s | /usr/bin/head -c 524288' % (commit_hash)
try:
signal.signal(signal.SIGPIPE, signal.SIG_DFL)
log_graph = check_output(command, shell=True)
log_graph_json = {'log_graph': log_graph}
self._dumps_write_stage_file(log_graph_json, stage_file)
return log_graph_json
except Exception, e:
logger.exception(e)
def dropbox_share(access_token, access_token_secret, share_path):
dropbox_connection = None
try:
dropbox_connection = httplib.HTTPSConnection('api.dropbox.com', 443, timeout=10)
headers = {'Host': 'api.dropbox.com', 'User-Agent': USER_AGENT}
params = urllib.urlencode({'oauth_signature_method': 'PLAINTEXT', 'oauth_timestamp': _timestamp(), 'oauth_nonce': _oauth_nonce(), 'oauth_consumer_key': DROPBOX_APP_KEY, 'oauth_token': access_token, 'oauth_signature': DROPBOX_APP_SECRET + '&' + access_token_secret, 'short_url': 'false'})
access_level = SANDBOX_ACCESS_LEVEL
urlencode_share_path = urllib.quote_plus(share_path)
share_url = ('/1/shares/%s/%s?' % (access_level, urlencode_share_path)) + params
dropbox_connection.request('GET', share_url, {}, headers)
response = dropbox_connection.getresponse()
if response.status == 200:
json_str = response.read()
response = json.loads(json_str)
dropbox_url = str(response['url'])
return dropbox_url
except Exception, e:
logger.exception(e)
def run(self):
beanstalk = beanstalkc.Connection(host=BEANSTALK_HOST,
port=BEANSTALK_PORT)
EventManager.switch(beanstalk, IMPORT_REPO_TUBE_NAME)
while True:
event_job = beanstalk.reserve()
try:
event = json.loads(event_job.body)
# exit signal
if event['type'] == -1:
event_job.delete()
sys.exit(0)
self._do_event(event)
except Exception, e:
logger.info('do_event catch except, event: %s' %
event_job.body)
logger.exception(e)
event_job.delete()
def dropbox_access_token_pair():
(request_token, request_token_secret) = dropbox_request_token_pair()
authorize_url = dropbox_authorize_url(request_token)
logger.info('open authorize url and click allow button: ' + authorize_url)
sys.stdin.readline()
dropbox_connection = None
try:
dropbox_connection = httplib.HTTPSConnection('api.dropbox.com', 443, timeout=10)
params = urllib.urlencode({'oauth_signature_method': 'PLAINTEXT', 'oauth_timestamp': _timestamp(), 'oauth_nonce': _oauth_nonce(), 'oauth_consumer_key': DROPBOX_APP_KEY, 'oauth_token': request_token, 'oauth_signature': DROPBOX_APP_SECRET + '&' + request_token_secret})
headers = {'Host': 'api.dropbox.com', 'Content-type': 'application/x-www-form-urlencoded', 'User-Agent': USER_AGENT}
dropbox_connection.request('POST', '/1/oauth/access_token', params, headers)
response = dropbox_connection.getresponse()
if response.status == 200:
result = response.read()
key_value_dict = urlparse.parse_qs(result)
if 'oauth_token' in key_value_dict and 'oauth_token_secret' in key_value_dict and 'uid' in key_value_dict:
return (key_value_dict['oauth_token'][0], key_value_dict['oauth_token_secret'][0])
except Exception, e:
logger.exception(e)
def repo_diff(self, repo_path, pre_commit_hash, commit_hash, context, path):
if not context:
context = 3
(pre_commit_hash, commit_hash, path) = self._all_to_utf8(pre_commit_hash, commit_hash, path)
if not self._path_check_chdir(repo_path, commit_hash, path) or not re.match('^\w+$', pre_commit_hash):
return {}
path = self._get_quote_path(path)
stage_file = self._get_diff_stage_file(repo_path, pre_commit_hash, commit_hash, context, path)
stage_file = stage_file + '.diff'
result = self._read_load_stage_file(stage_file)
if result is not None:
return result['diff']
command = '/usr/bin/git diff --numstat %s..%s -- %s | /usr/bin/head -c 524288 ; /usr/bin/git diff -U%s %s..%s -- %s | /usr/bin/head -c 524288' % (pre_commit_hash, commit_hash, path, context, pre_commit_hash, commit_hash, path)
try:
signal.signal(signal.SIGPIPE, signal.SIG_DFL)
result = check_output(command, shell=True)
diff = self._parse_diff_file_as_json(result)
self._dumps_write_stage_file({'diff': diff}, stage_file)
return diff
except Exception, e:
logger.exception(e)
def dropbox_access_token_pair():
(request_token, request_token_secret) = dropbox_request_token_pair()
authorize_url = dropbox_authorize_url(request_token)
logger.info('open authorize url and click allow button: ' + authorize_url)
sys.stdin.readline()
dropbox_connection = None
try:
dropbox_connection = httplib.HTTPSConnection('api.dropbox.com',
443,
timeout=10)
params = urllib.urlencode({
'oauth_signature_method':
'PLAINTEXT',
'oauth_timestamp':
_timestamp(),
'oauth_nonce':
_oauth_nonce(),
'oauth_consumer_key':
DROPBOX_APP_KEY,
'oauth_token':
request_token,
'oauth_signature':
DROPBOX_APP_SECRET + '&' + request_token_secret
})
headers = {
'Host': 'api.dropbox.com',
'Content-type': 'application/x-www-form-urlencoded',
'User-Agent': USER_AGENT
}
dropbox_connection.request('POST', '/1/oauth/access_token', params,
headers)
response = dropbox_connection.getresponse()
if response.status == 200:
result = response.read()
key_value_dict = urlparse.parse_qs(result)
if 'oauth_token' in key_value_dict and 'oauth_token_secret' in key_value_dict and 'uid' in key_value_dict:
return (key_value_dict['oauth_token'][0],
key_value_dict['oauth_token_secret'][0])
except Exception, e:
logger.exception(e)
def github_list_repo(access_token):
github_connection = None
try:
thirdpartyUser = ThirdpartyUser()
github_connection = httplib.HTTPSConnection('api.github.com',
443,
timeout=10)
headers = {
'Host': 'api.github.com',
'Accept': 'application/json',
'User-Agent': USER_AGENT
}
github_connection.request(
'GET',
'/user/repos?type=public&sort=pushed&access_token=' + access_token,
{}, headers)
response = github_connection.getresponse()
if response.status == 200:
json_str = response.read()
return json_str
except Exception, e:
logger.exception(e)
def post_payload(webHookURL, payload):
connection = None
url = webHookURL.url
try:
parseResult = urlparse(url)
scheme = parseResult.scheme; hostname = parseResult.hostname; port = parseResult.port; path = parseResult.path
if not port:
port = 80
if scheme == 'https':
connection = httplib.HTTPSConnection(hostname, port, timeout=10)
else:
connection = httplib.HTTPConnection(hostname, port, timeout=10)
params = urllib.urlencode({'payload': payload})
headers = {'Content-type': 'application/x-www-form-urlencoded', 'Accept': ACCEPT, 'User-Agent': USER_AGENT}
connection.request('POST', path, params, headers)
response = connection.getresponse()
if webHookURL.last_return_code != response.status:
webHookURL.last_return_code = response.status
webHookURL.save()
if response.status == 200:
return True
except Exception, e:
logger.exception(e)
def dropbox_share(access_token, access_token_secret, share_path):
dropbox_connection = None
try:
dropbox_connection = httplib.HTTPSConnection('api.dropbox.com',
443,
timeout=10)
headers = {'Host': 'api.dropbox.com', 'User-Agent': USER_AGENT}
params = urllib.urlencode({
'oauth_signature_method':
'PLAINTEXT',
'oauth_timestamp':
_timestamp(),
'oauth_nonce':
_oauth_nonce(),
'oauth_consumer_key':
DROPBOX_APP_KEY,
'oauth_token':
access_token,
'oauth_signature':
DROPBOX_APP_SECRET + '&' + access_token_secret,
'short_url':
'false'
})
access_level = SANDBOX_ACCESS_LEVEL
urlencode_share_path = urllib.quote_plus(share_path)
share_url = ('/1/shares/%s/%s?' %
(access_level, urlencode_share_path)) + params
dropbox_connection.request('GET', share_url, {}, headers)
response = dropbox_connection.getresponse()
if response.status == 200:
json_str = response.read()
response = json.loads(json_str)
dropbox_url = str(response['url'])
return dropbox_url
except Exception, e:
logger.exception(e)
def __get_many(model, pkids, only_visibly):
table = model._meta.db_table
if len(pkids) == 0:
return []
ids_key = __get_idskey(table, pkids)
cache_objs_map = cache.get_many(ids_key)
many_objects = [cache_objs_map[key] for key in cache_objs_map]
uncache_ids_key = list(set(ids_key) - set(cache_objs_map.keys()))
uncache_ids = __get_uncache_ids(uncache_ids_key)
if len(uncache_ids) > 0:
objects = []
try:
_objects = model.objects.filter(id__in=uncache_ids)
for _object in _objects:
if not _object:
continue
if only_visibly and _object.visibly != 0:
continue
objects.append(_object)
except Exception, e:
logger.exception(e)
__add_many(table, objects)
many_objects.extend(objects)
def _repo_load_log_file(self, from_commit_hash, commit_hash, log_size, path):
commits = []
between_commit_hash = commit_hash
path_in_git = ''
if path != '' and path != '.':
path_in_git = '-- ' + path
if from_commit_hash is not None and not from_commit_hash.startswith('0000000'):
between_commit_hash = from_commit_hash + '...' + commit_hash
command = '/usr/bin/git log -%s --pretty="%%h|%%p|%%t|%%an|%%ae|%%at|%%cn|%%ce|%%ct|%%s" %s %s | /usr/bin/head -c 524288' % (log_size, between_commit_hash, path_in_git)
try:
signal.signal(signal.SIGPIPE, signal.SIG_DFL)
raw_result = check_output(command, shell=True)
for line in raw_result.split('\n'):
ars = line.split('|', 9)
if len(ars) != 10:
continue
(commit_hash, parent_commit_hash, tree_hash, author_name, author_email, author_date, committer_name, committer_email, committer_date, commit_message) = (ars)
commit = {
'commit_hash': commit_hash,
'parent_commit_hash': parent_commit_hash,
'tree_hash': tree_hash,
'author_name': author_name,
'author_email': author_email,
'author_date': author_date,
'committer_name': committer_name,
'committer_email': committer_email,
'committer_date': committer_date,
'commit_message': commit_message,
}
commit['parent_commit_hash'] = parent_commit_hash.split(' ')
self._set_real_author_committer_name(commit)
commits.append(commit)
return commits
except Exception, e:
logger.exception(e)
def start():
logger.info(
'==================== START dropbox_sync_repo ====================')
gitshell_connection = None
try:
gitshell_connection = httplib.HTTPSConnection(REMOTE_HOST,
REMOTE_PORT,
timeout=10)
headers = {'Accept': 'application/json'}
gitshell_connection.request(
'GET',
'/gitshell/list_latest_push_repo/2M/?secret_key=' + SECRET_KEY, '',
headers)
response = gitshell_connection.getresponse()
if response.status == 200:
json_str = response.read()
json_response = json.loads(json_str)
result = json_response['result']
repos = json_response['latest_push_repos']
if result == 'success':
for repo in repos:
do_repo(repo)
except Exception, e:
logger.exception(e)
def github_get_thirdpartyUser(access_token):
github_connection = None
try:
thirdpartyUser = ThirdpartyUser()
github_connection = httplib.HTTPSConnection('api.github.com', 443, timeout=10)
headers = {'Host': 'api.github.com', 'Accept': 'application/json', 'User-Agent': USER_AGENT}
github_connection.request('GET', '/user?access_token=' + access_token, {}, headers)
response = github_connection.getresponse()
if response.status == 200:
json_str = response.read()
github_user_info = json.loads(json_str)
if 'login' in github_user_info:
thirdpartyUser.tp_username = github_user_info['login']
if 'id' in github_user_info:
thirdpartyUser.tp_id = github_user_info['id']
if 'email' in github_user_info:
thirdpartyUser.tp_email = github_user_info['email']
thirdpartyUser.init = 0
thirdpartyUser.access_token = access_token
thirdpartyUser.user_type = ThirdpartyUser.GITHUB
thirdpartyUser.github_user_info = github_user_info
return thirdpartyUser
except Exception, e:
logger.exception(e)
def __get_many(model, pkids, only_visibly):
table = model._meta.db_table
if len(pkids) == 0:
return []
ids_key = __get_idskey(table, pkids)
cache_objs_map = cache.get_many(ids_key)
many_objects = [cache_objs_map[key] for key in cache_objs_map]
uncache_ids_key = list( set(ids_key) - set(cache_objs_map.keys()) )
uncache_ids = __get_uncache_ids(uncache_ids_key)
if len(uncache_ids) > 0:
objects = []
try:
_objects = model.objects.filter(id__in=uncache_ids)
for _object in _objects:
if not _object:
continue
if only_visibly and _object.visibly != 0:
continue
objects.append(_object)
except Exception, e:
logger.exception(e)
__add_many(table, objects)
many_objects.extend(objects)
def queryraw(model, rawsql_id, parameters):
try:
return list(model.objects.raw(rawsql[rawsql_id], parameters))
except Exception, e:
logger.exception(e)
def send_mail(self, header, body, sender, receivers):
try:
send_mail(header, body, sender, receivers, fail_silently=False)
except Exception, e:
logger.exception(e)
def chdir(path):
try:
os.chdir(path)
return True
except Exception, e:
logger.exception(e)
def queryraw(model, rawsql_id, parameters):
try:
return list(model.objects.raw(rawsql[rawsql_id], parameters))
except Exception, e:
logger.exception(e)
def _chdir(self, path):
try:
os.chdir(path)
return True
except Exception, e:
logger.exception(e)