def start(): logger.info('==================== START importrepoworker ====================') import_repo_threads = [] for i in range(0, TOTAL_THREAD_COUNT): import_repo_threads.append(ImportRepoThread()) for import_repo_thread in import_repo_threads: import_repo_thread.start() for import_repo_thread in import_repo_threads: import_repo_thread.join() logger.info('==================== STOP importrepoworker ====================')
def start(): logger.info( '==================== START importrepoworker ====================') import_repo_threads = [] for i in range(0, TOTAL_THREAD_COUNT): import_repo_threads.append(ImportRepoThread()) for import_repo_thread in import_repo_threads: import_repo_thread.start() for import_repo_thread in import_repo_threads: import_repo_thread.join() logger.info( '==================== STOP importrepoworker ====================')
def run(self): beanstalk = beanstalkc.Connection(host=BEANSTALK_HOST, port=BEANSTALK_PORT) EventManager.switch(beanstalk, IMPORT_REPO_TUBE_NAME) while True: event_job = beanstalk.reserve() try: event = json.loads(event_job.body) # exit signal if event['type'] == -1: event_job.delete() sys.exit(0) self._do_event(event) except Exception, e: logger.info('do_event catch except, event: %s' % event_job.body) logger.exception(e) event_job.delete()
def start(): logger.info('==================== START forkworker ====================') beanstalk = beanstalkc.Connection(host=BEANSTALK_HOST, port=BEANSTALK_PORT) EventManager.switch(beanstalk, FORK_TUBE_NAME) while True: event_job = beanstalk.reserve() try: event = json.loads(event_job.body) # exit signal if event['type'] == -1: event_job.delete() sys.exit(0) do_event(event) except Exception, e: logger.info('do_event catch except, event: %s' % event_job.body) logger.exception(e) event_job.delete()
def start(): logger.info('==================== START eventworker ====================') beanstalk = beanstalkc.Connection(host=BEANSTALK_HOST, port=BEANSTALK_PORT) EventManager.switch(beanstalk, HOOK_TUBE_NAME) while True: event_job = beanstalk.reserve() try: event = json.loads(event_job.body) # exit signal if event['type'] == -1: event_job.delete() sys.exit(0) do_event(event) except Exception, e: logger.error('do_event catch except, event: %s' % event_job.body) logger.exception(e) event_job.delete()
def dropbox_access_token_pair(): (request_token, request_token_secret) = dropbox_request_token_pair() authorize_url = dropbox_authorize_url(request_token) logger.info('open authorize url and click allow button: ' + authorize_url) sys.stdin.readline() dropbox_connection = None try: dropbox_connection = httplib.HTTPSConnection('api.dropbox.com', 443, timeout=10) params = urllib.urlencode({'oauth_signature_method': 'PLAINTEXT', 'oauth_timestamp': _timestamp(), 'oauth_nonce': _oauth_nonce(), 'oauth_consumer_key': DROPBOX_APP_KEY, 'oauth_token': request_token, 'oauth_signature': DROPBOX_APP_SECRET + '&' + request_token_secret}) headers = {'Host': 'api.dropbox.com', 'Content-type': 'application/x-www-form-urlencoded', 'User-Agent': USER_AGENT} dropbox_connection.request('POST', '/1/oauth/access_token', params, headers) response = dropbox_connection.getresponse() if response.status == 200: result = response.read() key_value_dict = urlparse.parse_qs(result) if 'oauth_token' in key_value_dict and 'oauth_token_secret' in key_value_dict and 'uid' in key_value_dict: return (key_value_dict['oauth_token'][0], key_value_dict['oauth_token_secret'][0]) except Exception, e: logger.exception(e)
def do_event(event): from_repo_id = event['from_repo_id'] to_repo_id = event['to_repo_id'] from_repo = RepoManager.get_repo_by_id(from_repo_id) to_repo = RepoManager.get_repo_by_id(to_repo_id) copy_from_bare = False if to_repo is None: return if from_repo is None: copy_from_bare = True from_repo_path = GIT_BARE_REPO_PATH if not copy_from_bare: from_repo_path = from_repo.get_abs_repopath() to_repo_path = to_repo.get_abs_repopath() if not os.path.exists(from_repo_path): logger.info('from_repo_path: %s is not exists, clone failed' % from_repo_path) return if chdir(from_repo_path) is False: logger.info('chdir to from_repo_path: %s is False, clone failed' % from_repo_path) return if os.path.exists(to_repo_path): logger.info('to_repo_path: %s already exists, clone failed' % to_repo_path) return args = ['/usr/bin/git', 'gc'] popen = Popen(args, stdout=PIPE, shell=False, close_fds=True) result = popen.communicate()[0].strip() to_repo_dirname = os.path.dirname(to_repo_path) if not os.path.exists(to_repo_dirname): os.makedirs(to_repo_dirname) shutil.copytree(from_repo_path, to_repo_path) update_repo_status(from_repo, to_repo)
def dropbox_access_token_pair(): (request_token, request_token_secret) = dropbox_request_token_pair() authorize_url = dropbox_authorize_url(request_token) logger.info('open authorize url and click allow button: ' + authorize_url) sys.stdin.readline() dropbox_connection = None try: dropbox_connection = httplib.HTTPSConnection('api.dropbox.com', 443, timeout=10) params = urllib.urlencode({ 'oauth_signature_method': 'PLAINTEXT', 'oauth_timestamp': _timestamp(), 'oauth_nonce': _oauth_nonce(), 'oauth_consumer_key': DROPBOX_APP_KEY, 'oauth_token': request_token, 'oauth_signature': DROPBOX_APP_SECRET + '&' + request_token_secret }) headers = { 'Host': 'api.dropbox.com', 'Content-type': 'application/x-www-form-urlencoded', 'User-Agent': USER_AGENT } dropbox_connection.request('POST', '/1/oauth/access_token', params, headers) response = dropbox_connection.getresponse() if response.status == 200: result = response.read() key_value_dict = urlparse.parse_qs(result) if 'oauth_token' in key_value_dict and 'oauth_token_secret' in key_value_dict and 'uid' in key_value_dict: return (key_value_dict['oauth_token'][0], key_value_dict['oauth_token_secret'][0]) except Exception, e: logger.exception(e)
def start(): logger.info( '==================== START dropbox_sync_repo ====================') gitshell_connection = None try: gitshell_connection = httplib.HTTPSConnection(REMOTE_HOST, REMOTE_PORT, timeout=10) headers = {'Accept': 'application/json'} gitshell_connection.request( 'GET', '/gitshell/list_latest_push_repo/2M/?secret_key=' + SECRET_KEY, '', headers) response = gitshell_connection.getresponse() if response.status == 200: json_str = response.read() json_response = json.loads(json_str) result = json_response['result'] repos = json_response['latest_push_repos'] if result == 'success': for repo in repos: do_repo(repo) except Exception, e: logger.exception(e)
def start(): logger.info('==================== START notifworker ====================') while True: expect_notif_time = datetime.now() #print expect_notif_time if os.path.exists(STOP_FILE_FLAG): os.remove(STOP_FILE_FLAG) break for i in range(0, 1000): notifSettings = FeedManager.list_notifsetting_by_expectNotifTime(expect_notif_time, ROW_COUNT*i, ROW_COUNT) #print len(notifSettings) for notifSetting in notifSettings: #print notifSetting.id from_time = notifSetting.last_notif_time to_time = expect_notif_time #print from_time, to_time if from_time >= to_time: notifSetting.expect_notif_time = TIME_NEVER_COME notifSetting.save() continue notifMessages = FeedManager.list_notifmessage_by_userId_betweenTime_notifTypes(notifSetting.user_id, from_time, to_time, notifSetting.notif_types, 0, 1000) #print len(notifMessages) _send_notifMessages(notifMessages, notifSetting) notifSetting.last_notif_time = expect_notif_time notifSetting.expect_notif_time = TIME_NEVER_COME notifSetting.save() if len(notifSettings) < ROW_COUNT: break now = int(time.time()) next_minute_left = 60 - now%60 if next_minute_left == 0: next_minute_left = 60 #print next_minute_left time.sleep(next_minute_left) logger.info('==================== STOP notifworker ====================')
beanstalk = beanstalkc.Connection(host=BEANSTALK_HOST, port=BEANSTALK_PORT) EventManager.switch(beanstalk, HOOK_TUBE_NAME) while True: event_job = beanstalk.reserve() try: event = json.loads(event_job.body) # exit signal if event['type'] == -1: event_job.delete() sys.exit(0) do_event(event) except Exception, e: logger.error('do_event catch except, event: %s' % event_job.body) logger.exception(e) event_job.delete() logger.info('==================== STOP eventworker ====================') # abspath is the repo hooks directory def do_event(event): etype = event['type'] if etype == 0: abspath = event['abspath'].strip() (username, reponame) = get_username_reponame(abspath) if username == '' or reponame == '': return (user, userprofile, repo, abs_repo_path) = get_attrs(username, reponame) if user is None or userprofile is None or repo is None or abs_repo_path is None or not os.path.exists( abs_repo_path): return
'/gitshell/list_latest_push_repo/2M/?secret_key=' + SECRET_KEY, '', headers) response = gitshell_connection.getresponse() if response.status == 200: json_str = response.read() json_response = json.loads(json_str) result = json_response['result'] repos = json_response['latest_push_repos'] if result == 'success': for repo in repos: do_repo(repo) except Exception, e: logger.exception(e) finally: if gitshell_connection: gitshell_connection.close() logger.info( '==================== END dropbox_sync_repo ====================') def do_repo(repo): id = str(repo['id']) username = repo['username'] name = repo['name'] dropbox_sync = str(repo['dropbox_sync']) visibly = str(repo['visibly']) args = [ '/bin/bash', '/opt/bin/dropbox_sync_repo.sh', id, username, name, dropbox_sync, visibly ] popen = Popen(args, stdout=PIPE, shell=False, close_fds=True) result = popen.communicate()[0].strip()
def stop(): for i in range(0, TOTAL_THREAD_COUNT): EventManager.send_stop_event(IMPORT_REPO_TUBE_NAME) logger.info('send stop event message...')
beanstalk = beanstalkc.Connection(host=BEANSTALK_HOST, port=BEANSTALK_PORT) EventManager.switch(beanstalk, FORK_TUBE_NAME) while True: event_job = beanstalk.reserve() try: event = json.loads(event_job.body) # exit signal if event['type'] == -1: event_job.delete() sys.exit(0) do_event(event) except Exception, e: logger.info('do_event catch except, event: %s' % event_job.body) logger.exception(e) event_job.delete() logger.info('==================== STOP forkworker ====================') # git gc and file copy, nothing more def do_event(event): from_repo_id = event['from_repo_id'] to_repo_id = event['to_repo_id'] from_repo = RepoManager.get_repo_by_id(from_repo_id) to_repo = RepoManager.get_repo_by_id(to_repo_id) copy_from_bare = False if to_repo is None: return if from_repo is None: copy_from_bare = True from_repo_path = GIT_BARE_REPO_PATH if not copy_from_bare: from_repo_path = from_repo.get_abs_repopath()
def stop(): EventManager.send_stop_event(FORK_TUBE_NAME) logger.info('send stop event message...')
route_beanstalk_dict = init_route_beanstalk_dict() while True: event_job = beanstalk.reserve() try: event = json.loads(event_job.body) # exit signal if event['type'] == -1: event_job.delete() sys.exit(0) do_event(event) do_route_event(route_beanstalk_dict, event_job) except Exception, e: logger.error('do_event catch except, event: %s' % event_job.body) logger.exception(e) event_job.delete() logger.info('==================== STOP eventworker ====================') def init_route_beanstalk_dict(): route_beanstalk_dict = {} for tube_name in [HOOK_TUBE_NAME]: beanstalk = beanstalkc.Connection(host=BEANSTALK_HOST, port=BEANSTALK_PORT) EventManager.switch(beanstalk, tube_name) route_beanstalk_dict[tube_name] = beanstalk return route_beanstalk_dict def do_route_event(route_beanstalk_dict, event_job): for tube_name, beanstalk in route_beanstalk_dict.iteritems(): beanstalk.put(event_job.body) # abspath is the repo hooks directory def do_event(event):
beanstalk = beanstalkc.Connection(host=BEANSTALK_HOST, port=BEANSTALK_PORT) EventManager.switch(beanstalk, FORK_TUBE_NAME) while True: event_job = beanstalk.reserve() try: event = json.loads(event_job.body) # exit signal if event['type'] == -1: event_job.delete() sys.exit(0) do_event(event) except Exception, e: logger.info('do_event catch except, event: %s' % event_job.body) logger.exception(e) event_job.delete() logger.info('==================== STOP forkworker ====================') # git gc and file copy, nothing more def do_event(event): from_repo_id = event['from_repo_id'] to_repo_id = event['to_repo_id'] from_repo = RepoManager.get_repo_by_id(from_repo_id) to_repo = RepoManager.get_repo_by_id(to_repo_id) copy_from_bare = False if to_repo is None: return if from_repo is None: copy_from_bare = True from_repo_path = GIT_BARE_REPO_PATH if not copy_from_bare: