def repo_cat_file(self, repo_path, commit_hash, path): (commit_hash, path) = self._all_to_utf8(commit_hash, path) if not self._path_check_chdir(repo_path, commit_hash, path): return '' path = self._get_quote_path(path) if path.startswith('./'): path = path[2:] file_type = path.split('.')[-1] if file_type in BINARY_FILE_TYPE: return u'二进制文件' stage_file = self._get_stage_file(repo_path, commit_hash, path) result = self._read_load_stage_file(stage_file) if result is not None: return result['blob'] command = '/usr/bin/git show %s:%s | /usr/bin/head -c 524288' % (commit_hash, path) try: signal.signal(signal.SIGPIPE, signal.SIG_DFL) result = check_output(command, shell=True) ud = UniversalDetector() ud.feed(result) ud.close() if ud.result['encoding']: encoding = ud.result['encoding'] if encoding != 'utf-8' or encoding != 'utf8': result = result.decode(encoding).encode('utf-8') self._dumps_write_stage_file({'blob': result}, stage_file) return result except Exception, e: logger.exception(e)
def dropbox_request_token_pair(): dropbox_connection = None try: dropbox_connection = httplib.HTTPSConnection('api.dropbox.com', 443, timeout=10) params = urllib.urlencode({ 'oauth_signature_method': 'PLAINTEXT', 'oauth_timestamp': _timestamp(), 'oauth_nonce': _oauth_nonce(), 'oauth_consumer_key': DROPBOX_APP_KEY, 'oauth_signature': DROPBOX_APP_SECRET + '&' }) headers = { 'Host': 'api.dropbox.com', 'Content-type': 'application/x-www-form-urlencoded', 'User-Agent': USER_AGENT } dropbox_connection.request('POST', API_REQUEST_TOKEN_URL, params, headers) response = dropbox_connection.getresponse() if response.status == 200: result = response.read() key_value_dict = urlparse.parse_qs(result) if 'oauth_token' in key_value_dict and 'oauth_token_secret' in key_value_dict: return (key_value_dict['oauth_token'][0], key_value_dict['oauth_token_secret'][0]) except Exception, e: logger.exception(e)
def team_create(request): current = 'team'; title = u'设置 / 创建团队' teamprofileForm = TeamprofileForm() if request.method == 'POST': teamprofileForm = TeamprofileForm(request.POST) if teamprofileForm.is_valid(): username = teamprofileForm.cleaned_data['username'] email = '*****@*****.**' % username password = '******' % random.getrandbits(64) team = None try: team = User.objects.create_user(username, email, password) except IntegrityError, e: logger.exception(e) teamprofile = teamprofileForm.save(commit=False) teamprofile.id = team.id teamprofile.email = email teamprofile.imgurl = hashlib.md5(team.email.lower()).hexdigest() teamprofile.is_team_account = 1 teamprofile.save() teamMember = TeamMember(team_user_id = team.id, user_id = request.user.id, group_id = 0, permission = 2, is_admin = 1) teamMember.save() request.userprofile.has_joined_team = 1 request.userprofile.save() return HttpResponseRedirect('/settings/team/')
def _repo_meta_refs_detail_commit(self, repo, meta, commit_hash_dict): if not self._chdir(repo.get_abs_repopath()): return refs_detail_commit = {} refses = commit_hash_dict.keys() try: last_commit_command = '/bin/bash -c \'for i in %s; do echo -n "$i|"; git log "$i" -1 --pretty="%%h|%%p|%%t|%%an|%%ae|%%at|%%cn|%%ce|%%ct|%%s" | /usr/bin/head -c 524288; done\'' % (' '.join(refses)) signal.signal(signal.SIGPIPE, signal.SIG_DFL) last_commit_output = check_output(last_commit_command, shell=True) for line in last_commit_output.split('\n'): ars = line.split('|', 10) if len(ars) != 11: continue (refs, commit_hash, parent_commit_hash, tree_hash, author_name, author_email, author_date, committer_name, committer_email, committer_date, commit_message) = (ars) refs_commit = { 'commit_hash': commit_hash, 'parent_commit_hash': parent_commit_hash, 'tree_hash': tree_hash, 'author_name': author_name, 'author_email': author_email, 'author_date': author_date, 'committer_name': committer_name, 'committer_email': committer_email, 'committer_date': committer_date, 'commit_message': commit_message, } refs_commit['parent_commit_hash'] = parent_commit_hash.split(' ') self._set_real_author_committer_name(refs_commit) refs_detail_commit[refs] = refs_commit except Exception, e: logger.exception(e)
def github_oauth_access_token(code): github_connection = None try: github_connection = httplib.HTTPSConnection('github.com', 443, timeout=10) params = urllib.urlencode({ 'client_id': GITHUB_CLIENT_ID, 'client_secret': GITHUB_CLIENT_SECRET, 'code': code }) headers = { 'Content-type': 'application/x-www-form-urlencoded', 'Accept': 'application/json', 'User-Agent': USER_AGENT } github_connection.request('POST', '/login/oauth/access_token', params, headers) response = github_connection.getresponse() if response.status == 200: json_str = response.read() response = json.loads(json_str) access_token = str(response['access_token']) return access_token except Exception, e: logger.exception(e)
def _do_event(self, event): username = event['username'] reponame = event['reponame'] remote_git_url = event['remote_git_url'] local_user = GsuserManager.get_user_by_name(username) local_repo = RepoManager.get_repo_by_name(username, reponame) if local_user is None or local_repo is None or local_repo.status == 0: return local_repo_path = local_repo.get_abs_repopath() if os.path.exists(local_repo_path): return args = ['/bin/bash', '/opt/bin/git-import-remote-repo.sh' ] + [local_repo_path, remote_git_url] try: popen = Popen(args, stdout=PIPE, shell=False, close_fds=True) output = popen.communicate()[0].strip() returncode = popen.returncode if returncode == 0: RepoManager.check_export_ok_file(local_repo) diff_size = long(output) RepoManager.update_user_repo_quote(local_user, local_repo, diff_size) local_repo.status = 0 local_repo.save() else: local_repo.status = 500 local_repo.save() except Exception, e: local_repo.status = 500 local_repo.save() logger.exception(e)
def github_authenticate(thirdpartyUser): tp_id, tp_username, tp_email, github_user_info = thirdpartyUser.tp_id, thirdpartyUser.tp_username, thirdpartyUser.tp_email, thirdpartyUser.github_user_info thirdpartyUser_find = GsuserManager.get_thirdpartyUser_by_type_tpId(ThirdpartyUser.GITHUB, tp_id) if thirdpartyUser_find is not None: if thirdpartyUser_find.access_token != thirdpartyUser.access_token: thirdpartyUser_find.access_token = thirdpartyUser.access_token thirdpartyUser_find.save() user_id = thirdpartyUser_find.id user = GsuserManager.get_user_by_id(user_id) return user username = __get_uniq_username(tp_username) email = __get_uniq_email(tp_email) password = __get_random_password() if username is None or email is None or password is None: return None create_user = None try: create_user = User.objects.create_user(username, email, password) if create_user is not None and create_user.is_active: userprofile = Userprofile(username = create_user.username, email = create_user.email, imgurl = hashlib.md5(create_user.email.lower()).hexdigest()) _fill_github_user_info(userprofile, github_user_info) userprofile.id = create_user.id userprofile.save() if username == tp_username and email == tp_email: thirdpartyUser.init = 1 thirdpartyUser.user_type = ThirdpartyUser.GITHUB thirdpartyUser.id = create_user.id thirdpartyUser.save() except IntegrityError, e: logger.exception(e)
def team_create(request): current = 'team' title = u'设置 / 创建团队' teamprofileForm = TeamprofileForm() if request.method == 'POST': teamprofileForm = TeamprofileForm(request.POST) if teamprofileForm.is_valid(): username = teamprofileForm.cleaned_data['username'] email = '*****@*****.**' % username password = '******' % random.getrandbits(64) team = None try: team = User.objects.create_user(username, email, password) except IntegrityError, e: logger.exception(e) teamprofile = teamprofileForm.save(commit=False) teamprofile.id = team.id teamprofile.email = email teamprofile.imgurl = hashlib.md5(team.email.lower()).hexdigest() teamprofile.is_team_account = 1 teamprofile.save() teamMember = TeamMember(team_user_id=team.id, user_id=request.user.id, group_id=0, permission=2, is_admin=1) teamMember.save() request.userprofile.has_joined_team = 1 request.userprofile.save() return HttpResponseRedirect('/settings/team/')
def github_get_thirdpartyUser(access_token): github_connection = None try: thirdpartyUser = ThirdpartyUser() github_connection = httplib.HTTPSConnection('api.github.com', 443, timeout=10) headers = { 'Host': 'api.github.com', 'Accept': 'application/json', 'User-Agent': USER_AGENT } github_connection.request('GET', '/user?access_token=' + access_token, {}, headers) response = github_connection.getresponse() if response.status == 200: json_str = response.read() github_user_info = json.loads(json_str) if 'login' in github_user_info: thirdpartyUser.tp_username = github_user_info['login'] if 'id' in github_user_info: thirdpartyUser.tp_id = github_user_info['id'] if 'email' in github_user_info: thirdpartyUser.tp_email = github_user_info['email'] thirdpartyUser.init = 0 thirdpartyUser.access_token = access_token thirdpartyUser.user_type = ThirdpartyUser.GITHUB thirdpartyUser.github_user_info = github_user_info return thirdpartyUser except Exception, e: logger.exception(e)
def github_authenticate(thirdpartyUser): tp_id, tp_username, tp_email, github_user_info = thirdpartyUser.tp_id, thirdpartyUser.tp_username, thirdpartyUser.tp_email, thirdpartyUser.github_user_info thirdpartyUser_find = GsuserManager.get_thirdpartyUser_by_type_tpId( ThirdpartyUser.GITHUB, tp_id) if thirdpartyUser_find is not None: if thirdpartyUser_find.access_token != thirdpartyUser.access_token: thirdpartyUser_find.access_token = thirdpartyUser.access_token thirdpartyUser_find.save() user_id = thirdpartyUser_find.id user = GsuserManager.get_user_by_id(user_id) return user username = __get_uniq_username(tp_username) email = __get_uniq_email(tp_email) password = __get_random_password() if username is None or email is None or password is None: return None create_user = None try: create_user = User.objects.create_user(username, email, password) if create_user is not None and create_user.is_active: userprofile = Userprofile( username=create_user.username, email=create_user.email, imgurl=hashlib.md5(create_user.email.lower()).hexdigest()) _fill_github_user_info(userprofile, github_user_info) userprofile.id = create_user.id userprofile.save() if username == tp_username and email == tp_email: thirdpartyUser.init = 1 thirdpartyUser.user_type = ThirdpartyUser.GITHUB thirdpartyUser.id = create_user.id thirdpartyUser.save() except IntegrityError, e: logger.exception(e)
def post_payload(webHookURL, payload): connection = None url = webHookURL.url try: parseResult = urlparse(url) scheme = parseResult.scheme hostname = parseResult.hostname port = parseResult.port path = parseResult.path if not port: port = 80 if scheme == 'https': connection = httplib.HTTPSConnection(hostname, port, timeout=10) else: connection = httplib.HTTPConnection(hostname, port, timeout=10) params = urllib.urlencode({'payload': payload}) headers = { 'Content-type': 'application/x-www-form-urlencoded', 'Accept': ACCEPT, 'User-Agent': USER_AGENT } connection.request('POST', path, params, headers) response = connection.getresponse() if webHookURL.last_return_code != response.status: webHookURL.last_return_code = response.status webHookURL.save() if response.status == 200: return True except Exception, e: logger.exception(e)
def _do_event(self, event): username = event['username'] reponame = event['reponame'] remote_git_url = event['remote_git_url'] local_user = GsuserManager.get_user_by_name(username) local_repo = RepoManager.get_repo_by_name(username, reponame) if local_user is None or local_repo is None or local_repo.status == 0: return local_repo_path = local_repo.get_abs_repopath() if os.path.exists(local_repo_path): return args = ['/bin/bash', '/opt/bin/git-import-remote-repo.sh'] + [local_repo_path, remote_git_url] try: popen = Popen(args, stdout=PIPE, shell=False, close_fds=True) output = popen.communicate()[0].strip() returncode = popen.returncode if returncode == 0: RepoManager.check_export_ok_file(local_repo) diff_size = long(output) RepoManager.update_user_repo_quote(local_user, local_repo, diff_size) local_repo.status = 0 local_repo.save() else: local_repo.status = 500 local_repo.save() except Exception, e: local_repo.status = 500 local_repo.save() logger.exception(e)
def _ls_tree_check_output(self, commit_hash, path, origin_path): command = 'git config --global core.quotepath false; /usr/bin/git ls-tree %s -- %s | /usr/bin/head -c 524288' % (commit_hash, path) tree = {}; dirs = []; files = []; has_readme = False; readme_file = ''; try: signal.signal(signal.SIGPIPE, signal.SIG_DFL) raw_output = check_output(command, shell=True) max = 500 for line in raw_output.split("\n"): array = self.blank_p.split(line, 3) if len(array) >= 4: abs_path = array[3]; relative_path = abs_path if path != '.': relative_path = abs_path[len(origin_path):] if array[1] == 'tree': relative_path = relative_path + '/' relative_path = self._oct_utf8_decode(relative_path) if self._repo_file_path_check(relative_path): tree[relative_path] = {} tree[relative_path]['relative_path'] = relative_path tree[relative_path]['type'] = array[1] tree[relative_path]['commit_hash'] = array[2] tree[relative_path]['abs_path'] = abs_path filename_lower = relative_path.lower() if filename_lower == 'readme.md' or filename_lower == 'readme.mkd': has_readme = True readme_file = abs_path if array[1] == 'tree': dirs.append(relative_path) else: files.append(relative_path) if(max <= 0): break max = max - 1 quote_relative_paths = [] for relative_path in tree.keys(): quote_relative_paths.append(pipes.quote(relative_path)) if len(path.split('/')) < 50 and '"' not in origin_path: pre_path = origin_path if path == '.': pre_path = '' for i in range(0, 5): sub_length = 1000 sub_quote_relative_paths = quote_relative_paths[i*sub_length:(i+1)*sub_length] if len(sub_quote_relative_paths) == 0: break self._set_last_commit(tree, sub_quote_relative_paths, commit_hash, pre_path) if len(sub_quote_relative_paths) < sub_length: break dirs.sort() files.sort() ordered_tree = [] for file_path in dirs + files: tree_item = tree[file_path] self._set_real_author_name(tree_item) ordered_tree.append(tree_item) return {'tree': ordered_tree, 'has_readme': has_readme, 'readme_file': readme_file} except Exception, e: logger.exception(e)
def _create_user_and_authenticate(request, username, email, password, ref_hash, is_verify): user = None try: user = User.objects.create_user(username, email, password) if user is None or not user.is_active: return False except IntegrityError, e: logger.exception(e) return False
def _read_load_stage_file(self, stage_file): if not os.path.exists(stage_file): return None try: with codecs.open(stage_file, encoding='utf-8', mode='r') as json_data: result = json.load(json_data) return result except Exception, e: logger.exception(e)
def _run_command_on_repo(self, repo, args): abs_repopath = repo.get_abs_repopath() if os.path.exists(abs_repopath): self._chdir(abs_repopath) try: popen = Popen(args, stdout=PIPE, shell=False, close_fds=True) output = popen.communicate()[0].strip() return popen.returncode == 0 except Exception, e: logger.exception(e)
def github_list_repo(access_token): github_connection = None try: thirdpartyUser = ThirdpartyUser() github_connection = httplib.HTTPSConnection('api.github.com', 443, timeout=10) headers = {'Host': 'api.github.com', 'Accept': 'application/json', 'User-Agent': USER_AGENT} github_connection.request('GET', '/user/repos?type=public&sort=pushed&access_token=' + access_token, {}, headers) response = github_connection.getresponse() if response.status == 200: json_str = response.read() return json_str except Exception, e: logger.exception(e)
def github_oauth_access_token(code): github_connection = None try: github_connection = httplib.HTTPSConnection('github.com', 443, timeout=10) params = urllib.urlencode({'client_id': GITHUB_CLIENT_ID, 'client_secret': GITHUB_CLIENT_SECRET, 'code': code}) headers = {'Content-type': 'application/x-www-form-urlencoded', 'Accept': 'application/json', 'User-Agent': USER_AGENT} github_connection.request('POST', '/login/oauth/access_token', params, headers) response = github_connection.getresponse() if response.status == 200: json_str = response.read() response = json.loads(json_str) access_token = str(response['access_token']) return access_token except Exception, e: logger.exception(e)
def send_html_mail(self, header, body, sender, receivers): if not sender: sender = self.default_sender smtp = SMTP('localhost', 25) for receiver in receivers: try: msg = MIMEMultipart('alternative') msg['Subject'] = header msg['From'] = sender msg['To'] = receiver mIMEText = MIMEText(body, 'html', 'utf-8') msg.attach(mIMEText) smtp.sendmail(sender, receiver, msg.as_string()) except Exception, e: logger.exception(e)
def dropbox_request_token_pair(): dropbox_connection = None try: dropbox_connection = httplib.HTTPSConnection('api.dropbox.com', 443, timeout=10) params = urllib.urlencode({'oauth_signature_method': 'PLAINTEXT', 'oauth_timestamp': _timestamp(), 'oauth_nonce': _oauth_nonce(), 'oauth_consumer_key': DROPBOX_APP_KEY, 'oauth_signature': DROPBOX_APP_SECRET + '&'}) headers = {'Host': 'api.dropbox.com', 'Content-type': 'application/x-www-form-urlencoded', 'User-Agent': USER_AGENT} dropbox_connection.request('POST', API_REQUEST_TOKEN_URL, params, headers) response = dropbox_connection.getresponse() if response.status == 200: result = response.read() key_value_dict = urlparse.parse_qs(result) if 'oauth_token' in key_value_dict and 'oauth_token_secret' in key_value_dict: return (key_value_dict['oauth_token'][0], key_value_dict['oauth_token_secret'][0]) except Exception, e: logger.exception(e)
def _dumps_write_stage_file(self, result, stage_file): if result is None: return timenow = int(time.time()) stage_file_tmp = '%s.%s' % (stage_file, timenow) stage_file_tmp_path = os.path.dirname(stage_file_tmp) if not os.path.exists(stage_file_tmp_path): os.makedirs(stage_file_tmp_path) try: with codecs.open(stage_file_tmp, encoding='utf-8', mode='w') as stage_file_w: stage_file_w.write(json.dumps(result)) stage_file_w.flush() shutil.move(stage_file_tmp, stage_file) except Exception, e: logger.exception(e)
def __get(model, pkid, only_visibly): table = model._meta.db_table id_key = __get_idkey(table, pkid) obj = cache.get(id_key) if obj is not None: return obj try: obj = model.objects.get(id=pkid) if not obj: return None if only_visibly and obj.visibly != 0: return None cache.add(id_key, obj) return obj except Exception, e: logger.exception(e)
def run(self): beanstalk = beanstalkc.Connection(host=BEANSTALK_HOST, port=BEANSTALK_PORT) EventManager.switch(beanstalk, IMPORT_REPO_TUBE_NAME) while True: event_job = beanstalk.reserve() try: event = json.loads(event_job.body) # exit signal if event['type'] == -1: event_job.delete() sys.exit(0) self._do_event(event) except Exception, e: logger.info('do_event catch except, event: %s' % event_job.body) logger.exception(e) event_job.delete()
def _set_last_commit(self, tree, sub_quote_relative_paths, commit_hash, pre_path): try: signal.signal(signal.SIGPIPE, signal.SIG_DFL) last_commit_command = 'for i in %s; do echo -n "$i|"; git log %s -1 --pretty="%%at|%%an|%%ae|%%s" -- "%s$i" | /usr/bin/head -c 524288; done' % (' '.join(sub_quote_relative_paths), commit_hash, pre_path) last_commit_output = check_output(last_commit_command, shell=True) for last_commit in last_commit_output.split('\n'): splits = last_commit.split('|', 4) if len(splits) < 5: continue relative_path = splits[0] tree[relative_path]['author_time'] = splits[1] tree[relative_path]['author_name'] = splits[2] tree[relative_path]['author_email'] = splits[3] tree[relative_path]['last_commit_message'] = splits[4] except Exception, e: logger.exception(e)
def merge_pull_request(self, source_repo, desc_repo, source_refs, desc_refs, merge_commit_message): pullrequest_repo_path = '%s/%s/%s' % (PULLREQUEST_REPO_PATH, desc_repo.username, desc_repo.name) source_abs_repopath = source_repo.get_abs_repopath() source_remote_name = '%s-%s' % (source_repo.username, source_repo.name) dest_abs_repopath = desc_repo.get_abs_repopath() desc_remote_name = '%s-%s' % (desc_repo.username, desc_repo.name) action = 'merge' args = [pullrequest_repo_path, source_abs_repopath, source_remote_name, dest_abs_repopath, desc_remote_name, action, source_refs, desc_refs] if not self._is_allowed_paths(args): return (128, '合并失败,请检查是否存在冲突 或者 non-fast-forward') args = ['/bin/bash', '/opt/bin/git-pullrequest.sh'] + args + [merge_commit_message] try: popen = Popen(args, stdout=PIPE, shell=False, close_fds=True) output = popen.communicate()[0].strip() return (popen.returncode, output) except Exception, e: logger.exception(e)
def start(): logger.info('==================== START forkworker ====================') beanstalk = beanstalkc.Connection(host=BEANSTALK_HOST, port=BEANSTALK_PORT) EventManager.switch(beanstalk, FORK_TUBE_NAME) while True: event_job = beanstalk.reserve() try: event = json.loads(event_job.body) # exit signal if event['type'] == -1: event_job.delete() sys.exit(0) do_event(event) except Exception, e: logger.info('do_event catch except, event: %s' % event_job.body) logger.exception(e) event_job.delete()
def start(): logger.info('==================== START eventworker ====================') beanstalk = beanstalkc.Connection(host=BEANSTALK_HOST, port=BEANSTALK_PORT) EventManager.switch(beanstalk, HOOK_TUBE_NAME) while True: event_job = beanstalk.reserve() try: event = json.loads(event_job.body) # exit signal if event['type'] == -1: event_job.delete() sys.exit(0) do_event(event) except Exception, e: logger.error('do_event catch except, event: %s' % event_job.body) logger.exception(e) event_job.delete()
def prepare_pull_request(self, source_repo, desc_repo): pullrequest_repo_path = '%s/%s/%s' % (PULLREQUEST_REPO_PATH, desc_repo.username, desc_repo.name) source_abs_repopath = source_repo.get_abs_repopath() source_remote_name = '%s-%s' % (source_repo.username, source_repo.name) dest_abs_repopath = desc_repo.get_abs_repopath() desc_remote_name = '%s-%s' % (desc_repo.username, desc_repo.name) action = 'prepare' args = [pullrequest_repo_path, source_abs_repopath, source_remote_name, dest_abs_repopath, desc_remote_name, action] if not self._is_allowed_paths(args): return False args = ['/bin/bash', '/opt/bin/git-pullrequest.sh'] + args try: popen = Popen(args, stdout=PIPE, shell=False, close_fds=True) output = popen.communicate()[0].strip() return popen.returncode == 0 except Exception, e: logger.exception(e)
def repo_log_graph(self, repo, repo_path, commit_hash): if not self._path_check_chdir(repo_path, commit_hash, '.') or not re.match('^\w+$', commit_hash): return {} stage_file = self._get_stage_file(repo_path, commit_hash, repo.last_push_time) stage_file = stage_file + '.lg' log_graph = self._read_load_stage_file(stage_file) if log_graph is not None: return log_graph command = '/usr/bin/git log -100 --graph --abbrev-commit --date=relative --format=format:"%%h - (%%ar) %%s - %%an%%d" %s | /usr/bin/head -c 524288' % (commit_hash) try: signal.signal(signal.SIGPIPE, signal.SIG_DFL) log_graph = check_output(command, shell=True) log_graph_json = {'log_graph': log_graph} self._dumps_write_stage_file(log_graph_json, stage_file) return log_graph_json except Exception, e: logger.exception(e)
def dropbox_share(access_token, access_token_secret, share_path): dropbox_connection = None try: dropbox_connection = httplib.HTTPSConnection('api.dropbox.com', 443, timeout=10) headers = {'Host': 'api.dropbox.com', 'User-Agent': USER_AGENT} params = urllib.urlencode({'oauth_signature_method': 'PLAINTEXT', 'oauth_timestamp': _timestamp(), 'oauth_nonce': _oauth_nonce(), 'oauth_consumer_key': DROPBOX_APP_KEY, 'oauth_token': access_token, 'oauth_signature': DROPBOX_APP_SECRET + '&' + access_token_secret, 'short_url': 'false'}) access_level = SANDBOX_ACCESS_LEVEL urlencode_share_path = urllib.quote_plus(share_path) share_url = ('/1/shares/%s/%s?' % (access_level, urlencode_share_path)) + params dropbox_connection.request('GET', share_url, {}, headers) response = dropbox_connection.getresponse() if response.status == 200: json_str = response.read() response = json.loads(json_str) dropbox_url = str(response['url']) return dropbox_url except Exception, e: logger.exception(e)
def dropbox_access_token_pair(): (request_token, request_token_secret) = dropbox_request_token_pair() authorize_url = dropbox_authorize_url(request_token) logger.info('open authorize url and click allow button: ' + authorize_url) sys.stdin.readline() dropbox_connection = None try: dropbox_connection = httplib.HTTPSConnection('api.dropbox.com', 443, timeout=10) params = urllib.urlencode({'oauth_signature_method': 'PLAINTEXT', 'oauth_timestamp': _timestamp(), 'oauth_nonce': _oauth_nonce(), 'oauth_consumer_key': DROPBOX_APP_KEY, 'oauth_token': request_token, 'oauth_signature': DROPBOX_APP_SECRET + '&' + request_token_secret}) headers = {'Host': 'api.dropbox.com', 'Content-type': 'application/x-www-form-urlencoded', 'User-Agent': USER_AGENT} dropbox_connection.request('POST', '/1/oauth/access_token', params, headers) response = dropbox_connection.getresponse() if response.status == 200: result = response.read() key_value_dict = urlparse.parse_qs(result) if 'oauth_token' in key_value_dict and 'oauth_token_secret' in key_value_dict and 'uid' in key_value_dict: return (key_value_dict['oauth_token'][0], key_value_dict['oauth_token_secret'][0]) except Exception, e: logger.exception(e)
def repo_diff(self, repo_path, pre_commit_hash, commit_hash, context, path): if not context: context = 3 (pre_commit_hash, commit_hash, path) = self._all_to_utf8(pre_commit_hash, commit_hash, path) if not self._path_check_chdir(repo_path, commit_hash, path) or not re.match('^\w+$', pre_commit_hash): return {} path = self._get_quote_path(path) stage_file = self._get_diff_stage_file(repo_path, pre_commit_hash, commit_hash, context, path) stage_file = stage_file + '.diff' result = self._read_load_stage_file(stage_file) if result is not None: return result['diff'] command = '/usr/bin/git diff --numstat %s..%s -- %s | /usr/bin/head -c 524288 ; /usr/bin/git diff -U%s %s..%s -- %s | /usr/bin/head -c 524288' % (pre_commit_hash, commit_hash, path, context, pre_commit_hash, commit_hash, path) try: signal.signal(signal.SIGPIPE, signal.SIG_DFL) result = check_output(command, shell=True) diff = self._parse_diff_file_as_json(result) self._dumps_write_stage_file({'diff': diff}, stage_file) return diff except Exception, e: logger.exception(e)
def dropbox_access_token_pair(): (request_token, request_token_secret) = dropbox_request_token_pair() authorize_url = dropbox_authorize_url(request_token) logger.info('open authorize url and click allow button: ' + authorize_url) sys.stdin.readline() dropbox_connection = None try: dropbox_connection = httplib.HTTPSConnection('api.dropbox.com', 443, timeout=10) params = urllib.urlencode({ 'oauth_signature_method': 'PLAINTEXT', 'oauth_timestamp': _timestamp(), 'oauth_nonce': _oauth_nonce(), 'oauth_consumer_key': DROPBOX_APP_KEY, 'oauth_token': request_token, 'oauth_signature': DROPBOX_APP_SECRET + '&' + request_token_secret }) headers = { 'Host': 'api.dropbox.com', 'Content-type': 'application/x-www-form-urlencoded', 'User-Agent': USER_AGENT } dropbox_connection.request('POST', '/1/oauth/access_token', params, headers) response = dropbox_connection.getresponse() if response.status == 200: result = response.read() key_value_dict = urlparse.parse_qs(result) if 'oauth_token' in key_value_dict and 'oauth_token_secret' in key_value_dict and 'uid' in key_value_dict: return (key_value_dict['oauth_token'][0], key_value_dict['oauth_token_secret'][0]) except Exception, e: logger.exception(e)
def github_list_repo(access_token): github_connection = None try: thirdpartyUser = ThirdpartyUser() github_connection = httplib.HTTPSConnection('api.github.com', 443, timeout=10) headers = { 'Host': 'api.github.com', 'Accept': 'application/json', 'User-Agent': USER_AGENT } github_connection.request( 'GET', '/user/repos?type=public&sort=pushed&access_token=' + access_token, {}, headers) response = github_connection.getresponse() if response.status == 200: json_str = response.read() return json_str except Exception, e: logger.exception(e)
def post_payload(webHookURL, payload): connection = None url = webHookURL.url try: parseResult = urlparse(url) scheme = parseResult.scheme; hostname = parseResult.hostname; port = parseResult.port; path = parseResult.path if not port: port = 80 if scheme == 'https': connection = httplib.HTTPSConnection(hostname, port, timeout=10) else: connection = httplib.HTTPConnection(hostname, port, timeout=10) params = urllib.urlencode({'payload': payload}) headers = {'Content-type': 'application/x-www-form-urlencoded', 'Accept': ACCEPT, 'User-Agent': USER_AGENT} connection.request('POST', path, params, headers) response = connection.getresponse() if webHookURL.last_return_code != response.status: webHookURL.last_return_code = response.status webHookURL.save() if response.status == 200: return True except Exception, e: logger.exception(e)
def dropbox_share(access_token, access_token_secret, share_path): dropbox_connection = None try: dropbox_connection = httplib.HTTPSConnection('api.dropbox.com', 443, timeout=10) headers = {'Host': 'api.dropbox.com', 'User-Agent': USER_AGENT} params = urllib.urlencode({ 'oauth_signature_method': 'PLAINTEXT', 'oauth_timestamp': _timestamp(), 'oauth_nonce': _oauth_nonce(), 'oauth_consumer_key': DROPBOX_APP_KEY, 'oauth_token': access_token, 'oauth_signature': DROPBOX_APP_SECRET + '&' + access_token_secret, 'short_url': 'false' }) access_level = SANDBOX_ACCESS_LEVEL urlencode_share_path = urllib.quote_plus(share_path) share_url = ('/1/shares/%s/%s?' % (access_level, urlencode_share_path)) + params dropbox_connection.request('GET', share_url, {}, headers) response = dropbox_connection.getresponse() if response.status == 200: json_str = response.read() response = json.loads(json_str) dropbox_url = str(response['url']) return dropbox_url except Exception, e: logger.exception(e)
def __get_many(model, pkids, only_visibly): table = model._meta.db_table if len(pkids) == 0: return [] ids_key = __get_idskey(table, pkids) cache_objs_map = cache.get_many(ids_key) many_objects = [cache_objs_map[key] for key in cache_objs_map] uncache_ids_key = list(set(ids_key) - set(cache_objs_map.keys())) uncache_ids = __get_uncache_ids(uncache_ids_key) if len(uncache_ids) > 0: objects = [] try: _objects = model.objects.filter(id__in=uncache_ids) for _object in _objects: if not _object: continue if only_visibly and _object.visibly != 0: continue objects.append(_object) except Exception, e: logger.exception(e) __add_many(table, objects) many_objects.extend(objects)
def _repo_load_log_file(self, from_commit_hash, commit_hash, log_size, path): commits = [] between_commit_hash = commit_hash path_in_git = '' if path != '' and path != '.': path_in_git = '-- ' + path if from_commit_hash is not None and not from_commit_hash.startswith('0000000'): between_commit_hash = from_commit_hash + '...' + commit_hash command = '/usr/bin/git log -%s --pretty="%%h|%%p|%%t|%%an|%%ae|%%at|%%cn|%%ce|%%ct|%%s" %s %s | /usr/bin/head -c 524288' % (log_size, between_commit_hash, path_in_git) try: signal.signal(signal.SIGPIPE, signal.SIG_DFL) raw_result = check_output(command, shell=True) for line in raw_result.split('\n'): ars = line.split('|', 9) if len(ars) != 10: continue (commit_hash, parent_commit_hash, tree_hash, author_name, author_email, author_date, committer_name, committer_email, committer_date, commit_message) = (ars) commit = { 'commit_hash': commit_hash, 'parent_commit_hash': parent_commit_hash, 'tree_hash': tree_hash, 'author_name': author_name, 'author_email': author_email, 'author_date': author_date, 'committer_name': committer_name, 'committer_email': committer_email, 'committer_date': committer_date, 'commit_message': commit_message, } commit['parent_commit_hash'] = parent_commit_hash.split(' ') self._set_real_author_committer_name(commit) commits.append(commit) return commits except Exception, e: logger.exception(e)
def start(): logger.info( '==================== START dropbox_sync_repo ====================') gitshell_connection = None try: gitshell_connection = httplib.HTTPSConnection(REMOTE_HOST, REMOTE_PORT, timeout=10) headers = {'Accept': 'application/json'} gitshell_connection.request( 'GET', '/gitshell/list_latest_push_repo/2M/?secret_key=' + SECRET_KEY, '', headers) response = gitshell_connection.getresponse() if response.status == 200: json_str = response.read() json_response = json.loads(json_str) result = json_response['result'] repos = json_response['latest_push_repos'] if result == 'success': for repo in repos: do_repo(repo) except Exception, e: logger.exception(e)
def github_get_thirdpartyUser(access_token): github_connection = None try: thirdpartyUser = ThirdpartyUser() github_connection = httplib.HTTPSConnection('api.github.com', 443, timeout=10) headers = {'Host': 'api.github.com', 'Accept': 'application/json', 'User-Agent': USER_AGENT} github_connection.request('GET', '/user?access_token=' + access_token, {}, headers) response = github_connection.getresponse() if response.status == 200: json_str = response.read() github_user_info = json.loads(json_str) if 'login' in github_user_info: thirdpartyUser.tp_username = github_user_info['login'] if 'id' in github_user_info: thirdpartyUser.tp_id = github_user_info['id'] if 'email' in github_user_info: thirdpartyUser.tp_email = github_user_info['email'] thirdpartyUser.init = 0 thirdpartyUser.access_token = access_token thirdpartyUser.user_type = ThirdpartyUser.GITHUB thirdpartyUser.github_user_info = github_user_info return thirdpartyUser except Exception, e: logger.exception(e)
def __get_many(model, pkids, only_visibly): table = model._meta.db_table if len(pkids) == 0: return [] ids_key = __get_idskey(table, pkids) cache_objs_map = cache.get_many(ids_key) many_objects = [cache_objs_map[key] for key in cache_objs_map] uncache_ids_key = list( set(ids_key) - set(cache_objs_map.keys()) ) uncache_ids = __get_uncache_ids(uncache_ids_key) if len(uncache_ids) > 0: objects = [] try: _objects = model.objects.filter(id__in=uncache_ids) for _object in _objects: if not _object: continue if only_visibly and _object.visibly != 0: continue objects.append(_object) except Exception, e: logger.exception(e) __add_many(table, objects) many_objects.extend(objects)
def queryraw(model, rawsql_id, parameters): try: return list(model.objects.raw(rawsql[rawsql_id], parameters)) except Exception, e: logger.exception(e)
def send_mail(self, header, body, sender, receivers): try: send_mail(header, body, sender, receivers, fail_silently=False) except Exception, e: logger.exception(e)
def chdir(path): try: os.chdir(path) return True except Exception, e: logger.exception(e)
def _chdir(self, path): try: os.chdir(path) return True except Exception, e: logger.exception(e)