def start():
logger.info('==================== START importrepoworker ====================')
import_repo_threads = []
for i in range(0, TOTAL_THREAD_COUNT):
import_repo_threads.append(ImportRepoThread())
for import_repo_thread in import_repo_threads:
import_repo_thread.start()
for import_repo_thread in import_repo_threads:
import_repo_thread.join()
logger.info('==================== STOP importrepoworker ====================')
def start():
logger.info(
'==================== START importrepoworker ====================')
import_repo_threads = []
for i in range(0, TOTAL_THREAD_COUNT):
import_repo_threads.append(ImportRepoThread())
for import_repo_thread in import_repo_threads:
import_repo_thread.start()
for import_repo_thread in import_repo_threads:
import_repo_thread.join()
logger.info(
'==================== STOP importrepoworker ====================')
def run(self):
beanstalk = beanstalkc.Connection(host=BEANSTALK_HOST, port=BEANSTALK_PORT)
EventManager.switch(beanstalk, IMPORT_REPO_TUBE_NAME)
while True:
event_job = beanstalk.reserve()
try:
event = json.loads(event_job.body)
# exit signal
if event['type'] == -1:
event_job.delete()
sys.exit(0)
self._do_event(event)
except Exception, e:
logger.info('do_event catch except, event: %s' % event_job.body)
logger.exception(e)
event_job.delete()
def start():
logger.info('==================== START forkworker ====================')
beanstalk = beanstalkc.Connection(host=BEANSTALK_HOST, port=BEANSTALK_PORT)
EventManager.switch(beanstalk, FORK_TUBE_NAME)
while True:
event_job = beanstalk.reserve()
try:
event = json.loads(event_job.body)
# exit signal
if event['type'] == -1:
event_job.delete()
sys.exit(0)
do_event(event)
except Exception, e:
logger.info('do_event catch except, event: %s' % event_job.body)
logger.exception(e)
event_job.delete()
def start():
logger.info('==================== START eventworker ====================')
beanstalk = beanstalkc.Connection(host=BEANSTALK_HOST, port=BEANSTALK_PORT)
EventManager.switch(beanstalk, HOOK_TUBE_NAME)
while True:
event_job = beanstalk.reserve()
try:
event = json.loads(event_job.body)
# exit signal
if event['type'] == -1:
event_job.delete()
sys.exit(0)
do_event(event)
except Exception, e:
logger.error('do_event catch except, event: %s' % event_job.body)
logger.exception(e)
event_job.delete()
def run(self):
beanstalk = beanstalkc.Connection(host=BEANSTALK_HOST,
port=BEANSTALK_PORT)
EventManager.switch(beanstalk, IMPORT_REPO_TUBE_NAME)
while True:
event_job = beanstalk.reserve()
try:
event = json.loads(event_job.body)
# exit signal
if event['type'] == -1:
event_job.delete()
sys.exit(0)
self._do_event(event)
except Exception, e:
logger.info('do_event catch except, event: %s' %
event_job.body)
logger.exception(e)
event_job.delete()
def dropbox_access_token_pair():
(request_token, request_token_secret) = dropbox_request_token_pair()
authorize_url = dropbox_authorize_url(request_token)
logger.info('open authorize url and click allow button: ' + authorize_url)
sys.stdin.readline()
dropbox_connection = None
try:
dropbox_connection = httplib.HTTPSConnection('api.dropbox.com', 443, timeout=10)
params = urllib.urlencode({'oauth_signature_method': 'PLAINTEXT', 'oauth_timestamp': _timestamp(), 'oauth_nonce': _oauth_nonce(), 'oauth_consumer_key': DROPBOX_APP_KEY, 'oauth_token': request_token, 'oauth_signature': DROPBOX_APP_SECRET + '&' + request_token_secret})
headers = {'Host': 'api.dropbox.com', 'Content-type': 'application/x-www-form-urlencoded', 'User-Agent': USER_AGENT}
dropbox_connection.request('POST', '/1/oauth/access_token', params, headers)
response = dropbox_connection.getresponse()
if response.status == 200:
result = response.read()
key_value_dict = urlparse.parse_qs(result)
if 'oauth_token' in key_value_dict and 'oauth_token_secret' in key_value_dict and 'uid' in key_value_dict:
return (key_value_dict['oauth_token'][0], key_value_dict['oauth_token_secret'][0])
except Exception, e:
logger.exception(e)
def do_event(event):
from_repo_id = event['from_repo_id']
to_repo_id = event['to_repo_id']
from_repo = RepoManager.get_repo_by_id(from_repo_id)
to_repo = RepoManager.get_repo_by_id(to_repo_id)
copy_from_bare = False
if to_repo is None:
return
if from_repo is None:
copy_from_bare = True
from_repo_path = GIT_BARE_REPO_PATH
if not copy_from_bare:
from_repo_path = from_repo.get_abs_repopath()
to_repo_path = to_repo.get_abs_repopath()
if not os.path.exists(from_repo_path):
logger.info('from_repo_path: %s is not exists, clone failed' % from_repo_path)
return
if chdir(from_repo_path) is False:
logger.info('chdir to from_repo_path: %s is False, clone failed' % from_repo_path)
return
if os.path.exists(to_repo_path):
logger.info('to_repo_path: %s already exists, clone failed' % to_repo_path)
return
args = ['/usr/bin/git', 'gc']
popen = Popen(args, stdout=PIPE, shell=False, close_fds=True)
result = popen.communicate()[0].strip()
to_repo_dirname = os.path.dirname(to_repo_path)
if not os.path.exists(to_repo_dirname):
os.makedirs(to_repo_dirname)
shutil.copytree(from_repo_path, to_repo_path)
update_repo_status(from_repo, to_repo)
def do_event(event):
from_repo_id = event['from_repo_id']
to_repo_id = event['to_repo_id']
from_repo = RepoManager.get_repo_by_id(from_repo_id)
to_repo = RepoManager.get_repo_by_id(to_repo_id)
copy_from_bare = False
if to_repo is None:
return
if from_repo is None:
copy_from_bare = True
from_repo_path = GIT_BARE_REPO_PATH
if not copy_from_bare:
from_repo_path = from_repo.get_abs_repopath()
to_repo_path = to_repo.get_abs_repopath()
if not os.path.exists(from_repo_path):
logger.info('from_repo_path: %s is not exists, clone failed' %
from_repo_path)
return
if chdir(from_repo_path) is False:
logger.info('chdir to from_repo_path: %s is False, clone failed' %
from_repo_path)
return
if os.path.exists(to_repo_path):
logger.info('to_repo_path: %s already exists, clone failed' %
to_repo_path)
return
args = ['/usr/bin/git', 'gc']
popen = Popen(args, stdout=PIPE, shell=False, close_fds=True)
result = popen.communicate()[0].strip()
to_repo_dirname = os.path.dirname(to_repo_path)
if not os.path.exists(to_repo_dirname):
os.makedirs(to_repo_dirname)
shutil.copytree(from_repo_path, to_repo_path)
update_repo_status(from_repo, to_repo)
def dropbox_access_token_pair():
(request_token, request_token_secret) = dropbox_request_token_pair()
authorize_url = dropbox_authorize_url(request_token)
logger.info('open authorize url and click allow button: ' + authorize_url)
sys.stdin.readline()
dropbox_connection = None
try:
dropbox_connection = httplib.HTTPSConnection('api.dropbox.com',
443,
timeout=10)
params = urllib.urlencode({
'oauth_signature_method':
'PLAINTEXT',
'oauth_timestamp':
_timestamp(),
'oauth_nonce':
_oauth_nonce(),
'oauth_consumer_key':
DROPBOX_APP_KEY,
'oauth_token':
request_token,
'oauth_signature':
DROPBOX_APP_SECRET + '&' + request_token_secret
})
headers = {
'Host': 'api.dropbox.com',
'Content-type': 'application/x-www-form-urlencoded',
'User-Agent': USER_AGENT
}
dropbox_connection.request('POST', '/1/oauth/access_token', params,
headers)
response = dropbox_connection.getresponse()
if response.status == 200:
result = response.read()
key_value_dict = urlparse.parse_qs(result)
if 'oauth_token' in key_value_dict and 'oauth_token_secret' in key_value_dict and 'uid' in key_value_dict:
return (key_value_dict['oauth_token'][0],
key_value_dict['oauth_token_secret'][0])
except Exception, e:
logger.exception(e)
def start():
logger.info(
'==================== START dropbox_sync_repo ====================')
gitshell_connection = None
try:
gitshell_connection = httplib.HTTPSConnection(REMOTE_HOST,
REMOTE_PORT,
timeout=10)
headers = {'Accept': 'application/json'}
gitshell_connection.request(
'GET',
'/gitshell/list_latest_push_repo/2M/?secret_key=' + SECRET_KEY, '',
headers)
response = gitshell_connection.getresponse()
if response.status == 200:
json_str = response.read()
json_response = json.loads(json_str)
result = json_response['result']
repos = json_response['latest_push_repos']
if result == 'success':
for repo in repos:
do_repo(repo)
except Exception, e:
logger.exception(e)
def start():
logger.info('==================== START notifworker ====================')
while True:
expect_notif_time = datetime.now()
#print expect_notif_time
if os.path.exists(STOP_FILE_FLAG):
os.remove(STOP_FILE_FLAG)
break
for i in range(0, 1000):
notifSettings = FeedManager.list_notifsetting_by_expectNotifTime(expect_notif_time, ROW_COUNT*i, ROW_COUNT)
#print len(notifSettings)
for notifSetting in notifSettings:
#print notifSetting.id
from_time = notifSetting.last_notif_time
to_time = expect_notif_time
#print from_time, to_time
if from_time >= to_time:
notifSetting.expect_notif_time = TIME_NEVER_COME
notifSetting.save()
continue
notifMessages = FeedManager.list_notifmessage_by_userId_betweenTime_notifTypes(notifSetting.user_id, from_time, to_time, notifSetting.notif_types, 0, 1000)
#print len(notifMessages)
_send_notifMessages(notifMessages, notifSetting)
notifSetting.last_notif_time = expect_notif_time
notifSetting.expect_notif_time = TIME_NEVER_COME
notifSetting.save()
if len(notifSettings) < ROW_COUNT:
break
now = int(time.time())
next_minute_left = 60 - now%60
if next_minute_left == 0:
next_minute_left = 60
#print next_minute_left
time.sleep(next_minute_left)
logger.info('==================== STOP notifworker ====================')
beanstalk = beanstalkc.Connection(host=BEANSTALK_HOST, port=BEANSTALK_PORT)
EventManager.switch(beanstalk, HOOK_TUBE_NAME)
while True:
event_job = beanstalk.reserve()
try:
event = json.loads(event_job.body)
# exit signal
if event['type'] == -1:
event_job.delete()
sys.exit(0)
do_event(event)
except Exception, e:
logger.error('do_event catch except, event: %s' % event_job.body)
logger.exception(e)
event_job.delete()
logger.info('==================== STOP eventworker ====================')
# abspath is the repo hooks directory
def do_event(event):
etype = event['type']
if etype == 0:
abspath = event['abspath'].strip()
(username, reponame) = get_username_reponame(abspath)
if username == '' or reponame == '':
return
(user, userprofile, repo,
abs_repo_path) = get_attrs(username, reponame)
if user is None or userprofile is None or repo is None or abs_repo_path is None or not os.path.exists(
abs_repo_path):
return
'/gitshell/list_latest_push_repo/2M/?secret_key=' + SECRET_KEY, '',
headers)
response = gitshell_connection.getresponse()
if response.status == 200:
json_str = response.read()
json_response = json.loads(json_str)
result = json_response['result']
repos = json_response['latest_push_repos']
if result == 'success':
for repo in repos:
do_repo(repo)
except Exception, e:
logger.exception(e)
finally:
if gitshell_connection: gitshell_connection.close()
logger.info(
'==================== END dropbox_sync_repo ====================')
def do_repo(repo):
id = str(repo['id'])
username = repo['username']
name = repo['name']
dropbox_sync = str(repo['dropbox_sync'])
visibly = str(repo['visibly'])
args = [
'/bin/bash', '/opt/bin/dropbox_sync_repo.sh', id, username, name,
dropbox_sync, visibly
]
popen = Popen(args, stdout=PIPE, shell=False, close_fds=True)
result = popen.communicate()[0].strip()
def stop():
for i in range(0, TOTAL_THREAD_COUNT):
EventManager.send_stop_event(IMPORT_REPO_TUBE_NAME)
logger.info('send stop event message...')
beanstalk = beanstalkc.Connection(host=BEANSTALK_HOST, port=BEANSTALK_PORT)
EventManager.switch(beanstalk, FORK_TUBE_NAME)
while True:
event_job = beanstalk.reserve()
try:
event = json.loads(event_job.body)
# exit signal
if event['type'] == -1:
event_job.delete()
sys.exit(0)
do_event(event)
except Exception, e:
logger.info('do_event catch except, event: %s' % event_job.body)
logger.exception(e)
event_job.delete()
logger.info('==================== STOP forkworker ====================')
# git gc and file copy, nothing more
def do_event(event):
from_repo_id = event['from_repo_id']
to_repo_id = event['to_repo_id']
from_repo = RepoManager.get_repo_by_id(from_repo_id)
to_repo = RepoManager.get_repo_by_id(to_repo_id)
copy_from_bare = False
if to_repo is None:
return
if from_repo is None:
copy_from_bare = True
from_repo_path = GIT_BARE_REPO_PATH
if not copy_from_bare:
from_repo_path = from_repo.get_abs_repopath()
def stop():
for i in range(0, TOTAL_THREAD_COUNT):
EventManager.send_stop_event(IMPORT_REPO_TUBE_NAME)
logger.info('send stop event message...')
def stop():
EventManager.send_stop_event(FORK_TUBE_NAME)
logger.info('send stop event message...')
def stop():
EventManager.send_stop_event(FORK_TUBE_NAME)
logger.info('send stop event message...')
route_beanstalk_dict = init_route_beanstalk_dict()
while True:
event_job = beanstalk.reserve()
try:
event = json.loads(event_job.body)
# exit signal
if event['type'] == -1:
event_job.delete()
sys.exit(0)
do_event(event)
do_route_event(route_beanstalk_dict, event_job)
except Exception, e:
logger.error('do_event catch except, event: %s' % event_job.body)
logger.exception(e)
event_job.delete()
logger.info('==================== STOP eventworker ====================')
def init_route_beanstalk_dict():
route_beanstalk_dict = {}
for tube_name in [HOOK_TUBE_NAME]:
beanstalk = beanstalkc.Connection(host=BEANSTALK_HOST, port=BEANSTALK_PORT)
EventManager.switch(beanstalk, tube_name)
route_beanstalk_dict[tube_name] = beanstalk
return route_beanstalk_dict
def do_route_event(route_beanstalk_dict, event_job):
for tube_name, beanstalk in route_beanstalk_dict.iteritems():
beanstalk.put(event_job.body)
# abspath is the repo hooks directory
def do_event(event):
beanstalk = beanstalkc.Connection(host=BEANSTALK_HOST, port=BEANSTALK_PORT)
EventManager.switch(beanstalk, FORK_TUBE_NAME)
while True:
event_job = beanstalk.reserve()
try:
event = json.loads(event_job.body)
# exit signal
if event['type'] == -1:
event_job.delete()
sys.exit(0)
do_event(event)
except Exception, e:
logger.info('do_event catch except, event: %s' % event_job.body)
logger.exception(e)
event_job.delete()
logger.info('==================== STOP forkworker ====================')
# git gc and file copy, nothing more
def do_event(event):
from_repo_id = event['from_repo_id']
to_repo_id = event['to_repo_id']
from_repo = RepoManager.get_repo_by_id(from_repo_id)
to_repo = RepoManager.get_repo_by_id(to_repo_id)
copy_from_bare = False
if to_repo is None:
return
if from_repo is None:
copy_from_bare = True
from_repo_path = GIT_BARE_REPO_PATH
if not copy_from_bare:
