def pick_attack_write(r):
"""choose write system call containing 'attack' string"""
if r.name == "write" and \
str(r).find("attack") != -1:
dbg.info("pick:#R<%s#> %s" % (r.tac, r))
return PICK_ATTACK|PICK_STOP
def repair(db, attack_edges, dryrun):
global rollbacks
global queued_actions
rollbacks = []
q = PriorityQueue()
while True:
try:
queued_actions = []
repair_loop(db, q, attack_edges, dryrun)
return
except TimeLoopException as tle:
dbg.ctrl("time loop: %s" % tle.n)
dbg.info("node list: %s" % str(rollbacks))
for a in queued_actions:
if hasattr(a, 'todo'): delattr(a, 'todo')
if hasattr(a, 'redone'): delattr(a, 'redone')
if hasattr(a, 'new_action'): delattr(a, 'new_action')
rbsave = rollbacks
rollbacks = []
q = PriorityQueue()
for (n, tic) in rbsave:
if hasattr(n, 'rollback_ts'): delattr(n, 'rollback_ts')
for (n, tic) in rbsave:
ne = rollback_nodes([n], tic)
add_queue(q, ne)
def _authorize(self):
dbg.info('Request access token from Dropbox')
flow = DropboxOAuth2FlowNoRedirect(APP_KEY, APP_SECRET)
authorize_url = flow.start()
# print 'Open auth url:', authorize_url
#browser = webdriver.PhantomJS(service_log_path=os.path.join(tempfile.gettempdir(), 'ghostdriver.log'))
#browser = webdriver.PhantomJS(service_log_path=os.path.join(tempfile.gettempdir(), 'ghostdriver.log'), service_args=['--ignore-ssl-errors=true', '--ssl-protocol=tlsv1'])
# Change to rely on browser
print(
"We need to authorize access to Dropbox. Please visit the following URL and authorize the access:"
)
print(authorize_url)
print("")
code = raw_input("Input the code you got: ").strip()
#code = #raw_input("Enter the authorization code here: ").strip()
try:
oauth_result = flow.finish(code)
with open(self.auth_file, 'w') as file:
file.write(oauth_result.access_token + "\n")
file.write(oauth_result.user_id + "\n")
dbg.info('Authentication successful')
return (oauth_result.access_token, oauth_result.user_id)
except Exception, e:
print('Error: %s' % (e, ))
return
def __search_checkpoints(self, n, x):
for p in glob(
os.path.join(kutil.fsget(x.dev), ".inodes", str(x.ino)) + "*"):
dbg.info("found inodes: %s" % p)
info = p.split(".")
tic = (int(info[-2]) * 10**9 + int(info[-1]), 0)
n.checkpoints.add(fsmgr.FileCheckpoint(tic, tic, x.dev, x.ino, p))
def repair2(repair_ckpts, gui=False):
dbg.info('repairing with', repair_ckpts)
if gui:
ctrl = GuiRepairCtrl(repair_ckpts)
else:
ctrl = BasicRepairCtrl(repair_ckpts)
ctrl.repair()
def refresh_token(refresh_token):
dbg.info('Refresh access token from Baidu')
if not refresh_token:
raise Exception('Refresh token is null')
token = OAuth2._token_request('refresh_token', refresh_token=refresh_token)
dbg.info('Refresh successful')
return token
def cmd_update(self):
master = self.get_uptodate_master(False)
# already up-to-date
prev = self.get_prev_value()
if (master == prev):
self.update_head_and_prev(master)
return True
head = self.get_head_and_config()
# XXX: need to check if non-checked in but modified files.
if (head == prev):
headstr = head.split(".")[0]
if headstr == '':
self.update_head_and_prev(master)
return self.restore_from_master()
masterstr = master.split(".")[0]
self.update_changed(headstr, masterstr)
else:
### need to merge
raise Exception('Merge required')
self.update_head_and_prev(master)
self.blobstore.rootblob = None
dbg.info("update done %s" % time.ctime())
return True
def cmd_repair(fd, args):
"""repairing attacks: hint attacker"""
if len(args) < 1:
cmd_help(fd)
return
hint = args[0]
pick = "pick_attack_execve" if len(args) < 2 else args[1]
name = attacker.find_attack_node(logd, pick, hint)
attack = mgrapi.RegisteredObject.by_name_load(name)
if attack is None:
dbg.error('missing attack node:', name)
return
chkpt = max(c for c in attack.checkpoints if c < min(attack.actions))
assert chkpt
assert len(attack.actions) == 1
for a in attack.actions:
dbg.info("cancel: %s" % a.argsnode)
a.argsnode.data = None
a.cancel = True
dbg.info("pick:", chkpt)
The(CtrlFSM).set_state("repairing")
ctrl.repair(attack, chkpt)
The(CtrlFSM).set_state("init")
rtn_cmd(fd, "done")
def pick_execve(r):
"""choose first successful execve"""
if r.name == "execve" and \
hasattr(r, "ret") and \
r.ret == 0:
dbg.info("pick:#R<%s#> %s" % (r.tac, r))
return PICK_ATTACK|PICK_STOP
def rollback(self, n, cp):
#code.interact(local=locals())
dbg.info('#G<rollback#> %s -> %s' % (prettify(n), (prettify(cp))))
n.rollback(cp)
self.set_cs(n, cp)
self.mod_state[n] = cp
self.set_dirty(n, False)
self.set_color(cp, self.colors['redone'])
def refresh(self):
if 'refresh_token' in self._token:
token = OAuth2.refresh_token(self._token['refresh_token'])
else:
dbg.info('No refresh token in the access token')
token = OAuth2.request_token()
self.set_token(token)
def cmd_test(metasync, args, opts):
"quick test (e.g., metasync test {%s})"
# invoke pdb when failed in testing
util.install_pdb()
tmpdir = tempfile.mkdtemp()
root = os.path.join(tmpdir, "repo")
util.mkdirs(root)
metasync = MetaSync(root)
# opts for sub test routines
opts.root = root
opts.tmpdir = tmpdir
opts.encrypt_key = "testkey" if opts.encrypt else ""
dbg.info("root: %s" % root)
dbg.info("args: %s" % args)
dbg.info("opts: ")
for (k, v) in vars(opts).iteritems():
dbg.info(" %-12s = %s" % (k, v))
alltests = dict(test.get_all_tests())
if any(case not in alltests for case in args):
dbg.err("no such a test case: %s" % args)
alltests["help"](metasync, opts)
exit(1)
# print help if no test case is provided
if len(args) == 0:
args = ["help"]
for case in args:
dbg.info("#R<testing %s#> (%s)" % (case, alltests[case].__doc__))
alltests[case](metasync, opts)
# poorman's tree
def tree(path):
for root, dirs, files in os.walk(path):
base = os.path.basename(root)
idnt = ' ' * (root.replace(path, '').count(os.sep))
print('%s%s/' % (idnt, base))
for f in files:
pn = os.path.join(root, f)
print(' %s%s [%s]' % (idnt, f, os.stat(pn).st_size))
# dump some content of blobs
if opts.dump and "objects" == base:
print(util.hexdump(util.read_file(pn, 32*2)))
print
# dump root
if not opts.notree:
tree(tmpdir)
# cleanup tmpdir
if not opts.keep:
shutil.rmtree(tmpdir)
def repair(self):
"""Process all actions until finished."""
while (self.has_next()):
self.next()
pi = mgrapi.PlusInfinity()
self.save_ckpts(self.mod_state.keys(), mgrapi.TimeInterval(pi, pi))
dbg.info('repair done')
PathidTable.dumptables()
def test_all(metasync, opts):
"run all test cases"
opts.notree = True
for (n, f) in get_all_tests():
if n == "all" or n.startswith("bench_"):
continue
dbg.info("#R<testing %s#> (%s)" % (n, f.__doc__))
f(metasync, opts)
def test_all(metasync, opts):
"run all test cases"
opts.notree = True
for (n, f) in get_all_tests():
if n == "all" or n.startswith("bench_"):
continue
dbg.info("#R<testing %s#> (%s)" % (n, f.__doc__))
f(metasync, opts)
def _authorize(self, flow, storage, flags):
# copied and edited from tools.run_flow()
import getpass, time
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
dbg.info('Request access token from Google Drive')
oauth_callback = client.OOB_CALLBACK_URN
flow.redirect_uri = oauth_callback
authorize_url = flow.step1_get_authorize_url()
#print 'Open auth url:', authorize_url
import tempfile
browser = webdriver.PhantomJS(service_log_path=os.path.join(tempfile.gettempdir(), 'ghostdriver.log'))
browser.get(authorize_url)
try:
wait = WebDriverWait(browser, 30)
email = wait.until(EC.element_to_be_clickable((By.ID, "Email")))
except:
print(browser.title)
print(browser.page_source)
browser.quit()
raise Exception("timeout for authorization")
email.send_keys(raw_input("Enter your Google Drive email:"))
pwd = browser.find_element_by_id("Passwd")
pwd.send_keys(getpass.getpass("Enter your Google Drive password:"******"signIn")
btn.click()
try:
wait = WebDriverWait(browser, 30)
btn = wait.until(EC.element_to_be_clickable((By.ID, "submit_approve_access")))
except:
print(browser.title)
print(browser.page_source)
browser.quit()
raise Exception("timeout for authorization")
btn.click()
try:
wait = WebDriverWait(browser, 30)
wait.until(EC.title_contains("Success code"))
except:
print(browser.title)
print(browser.page_source)
browser.quit()
raise Exception("timeout for authorization")
code = browser.title.split("=")[1]
browser.quit()
try:
credential = flow.step2_exchange(code)
except client.FlowExchangeError, e:
sys.exit('Authentication has failed: %s' % e)
def _authorize(self):
dbg.info('Request access token from Dropbox')
flow = DropboxOAuth2FlowNoRedirect(APP_KEY, APP_SECRET)
authorize_url = flow.start()
# print 'Open auth url:', authorize_url
#browser = webdriver.PhantomJS(service_log_path=os.path.join(tempfile.gettempdir(), 'ghostdriver.log'))
browser = webdriver.PhantomJS(service_log_path=os.path.join(tempfile.gettempdir(), 'ghostdriver.log'), service_args=['--ignore-ssl-errors=true', '--ssl-protocol=tlsv1'])
browser.get(authorize_url)
try:
wait = WebDriverWait(browser, 30)
email = wait.until(EC.element_to_be_clickable((By.XPATH, "//input[@name='login_email']")))
except:
print(browser.title)
print(browser.page_source)
browser.quit()
raise Exception("timeout for authorization")
email.send_keys(raw_input("Enter your Dropbox email:"))
pwd = browser.find_element_by_xpath("//input[@name='login_password']")
pwd.send_keys(getpass.getpass("Enter your Dropbox password:"******"allow_access")))
except:
print(browser.title)
print(browser.page_source)
browser.quit()
raise Exception("timeout for authorization")
btn.click()
try:
wait = WebDriverWait(browser, 30)
auth_code = wait.until(EC.presence_of_element_located((By.ID, "auth-code")))
except:
print(browser.title)
print(browser.page_source)
browser.quit()
raise Exception("timeout for authorization")
print(browser.title)
#auth_code = browser.find_element_by_id("auth-code")
code = auth_code.text
browser.quit()
#code = #raw_input("Enter the authorization code here: ").strip()
access_token, user_id = flow.finish(code)
with open(self.auth_file, 'w') as file:
file.write(access_token + "\n")
file.write(user_id + "\n")
dbg.info('Authentication successful')
return (access_token, user_id)
def cmd_test(metasync, args, opts):
"quick test (e.g., metasync test {%s})"
# invoke pdb when failed in testing
util.install_pdb()
tmpdir = tempfile.mkdtemp()
root = os.path.join(tmpdir, "repo")
util.mkdirs(root)
metasync = MetaSync(root)
# opts for sub test routines
opts.root = root
opts.tmpdir = tmpdir
opts.encrypt_key = "testkey" if opts.encrypt else ""
dbg.info("root: %s" % root)
dbg.info("args: %s" % args)
dbg.info("opts: ")
for (k, v) in vars(opts).iteritems():
dbg.info(" %-12s = %s" % (k, v))
alltests = dict(test.get_all_tests())
if any(case not in alltests for case in args):
dbg.err("no such a test case: %s" % args)
alltests["help"](metasync, opts)
exit(1)
def __do_check(m):
assert len([n for n in m.distrib if n == 1]) == len(m.distrib)/10*2
assert len([n for n in m.distrib if n == 2]) == len(m.distrib)/10*5
assert len([n for n in m.distrib if n == 3]) == len(m.distrib)/10*3
copyset = set()
for i in range(len(m.distrib)):
c = m.get_mapping(i)
assert len(c) == m.replica
copyset.add(tuple(c))
assert len(copyset) < len(m.distrib)
dbg.info("mapping: %s ..." % str(m.distrib)[:30])
dbg.info("copyset: %s" % str(list(copyset)))
def __do_check(m):
assert len([n for n in m.distrib if n == 1]) == len(m.distrib) / 10 * 2
assert len([n for n in m.distrib if n == 2]) == len(m.distrib) / 10 * 5
assert len([n for n in m.distrib if n == 3]) == len(m.distrib) / 10 * 3
copyset = set()
for i in range(len(m.distrib)):
c = m.get_mapping(i)
assert len(c) == m.replica
copyset.add(tuple(c))
assert len(copyset) < len(m.distrib)
dbg.info("mapping: %s ..." % str(m.distrib)[:30])
dbg.info("copyset: %s" % str(list(copyset)))
def test_merge(metasync, opts):
clone = test_clone(metasync, opts)
new_files = [3072, 4096]
metasyncs = [metasync, clone]
for i in range(2):
dbg.info("checkin %d" % i)
pn = os.path.join(metasyncs[i].path_root, "file-%s" % new_files[i])
util.create_random_file(pn, new_files[i])
metasyncs[i].cmd_checkin(pn)
metasync.cmd_push()
clone.cmd_fetch()
assert not clone.cmd_push()
clone.cmd_update()
assert clone.cmd_push()
def test_bstore_iterate(metasync, opts):
"walk over all files in a service, and check if correctly distributed"
test_init(metasync, opts)
hashes = set()
for srv in metasync.services:
for hv in metasync.bstore_iter_remote(srv):
dbg.info("%-10s: %s" % (srv, hv))
hashes.add(hv)
# covered by local's bstore?
for hv in metasync.bstore_iter():
hashes.remove(hv)
assert len(hashes) == 0
def test_merge(metasync, opts):
clone = test_clone(metasync, opts)
new_files = [3072, 4096]
metasyncs = [metasync, clone]
for i in range(2):
dbg.info("checkin %d" % i)
pn = os.path.join(metasyncs[i].path_root, "file-%s" % new_files[i])
util.create_random_file(pn, new_files[i])
metasyncs[i].cmd_checkin(pn)
metasync.cmd_push()
clone.cmd_fetch()
assert not clone.cmd_push()
clone.cmd_update()
assert clone.cmd_push()
def test_bstore_iterate(metasync, opts):
"walk over all files in a service, and check if correctly distributed"
test_init(metasync, opts)
hashes = set()
for srv in metasync.services:
for hv in metasync.bstore_iter_remote(srv):
dbg.info("%-10s: %s" % (srv, hv))
hashes.add(hv)
# covered by local's bstore?
for hv in metasync.bstore_iter():
hashes.remove(hv)
assert len(hashes) == 0
def test_mapping_reconfig(metasync, opts):
"test reconfiguration/versioning of mapping strategies"
opts.notree = True
m = DetMap([(1, 2*GB), (2, 5*GB), (3, 2*GB)])
m.reconfig([(1, 2*GB), (2, 5*GB)])
# two version of mapping info
assert len(m.mapinfo) == 2
for (ver, conf) in m.mapinfo.iteritems():
dbg.info("%s -> %s" % (ver, conf.store()))
# where to remap of hash value: 0
# 0: [1,2] -> [1,2]
# 1: [2,3] -> [2,1]
assert m.get_remmaping(0) == []
assert m.get_remmaping(1) == [1]
def test_mapping_reconfig(metasync, opts):
"test reconfiguration/versioning of mapping strategies"
opts.notree = True
m = DetMap([(1, 2 * GB), (2, 5 * GB), (3, 2 * GB)])
m.reconfig([(1, 2 * GB), (2, 5 * GB)])
# two version of mapping info
assert len(m.mapinfo) == 2
for (ver, conf) in m.mapinfo.iteritems():
dbg.info("%s -> %s" % (ver, conf.store()))
# where to remap of hash value: 0
# 0: [1,2] -> [1,2]
# 1: [2,3] -> [2,1]
assert m.get_remmaping(0) == []
assert m.get_remmaping(1) == [1]
def cmd_update(self):
master = self.get_uptodate_master()
# already up-to-date
prev = self.get_prev_value()
if (master == prev):
return True
head = self.get_head_and_config()
# XXX: need to check if non-checked in but modified files.
if (head == prev):
self.update_changed(head.split(".")[0], master.split(".")[0])
else:
### need to merge
raise Exception('Merge required')
self.update_head_and_prev(master)
self.blobstore.rootblob = None
dbg.info("update done %s" % time.ctime())
return True
def _authorize(self):
dbg.info('Request access token from Dropbox')
flow = DropboxOAuth2FlowNoRedirect(APP_KEY, APP_SECRET)
authorize_url = flow.start()
# print 'Open auth url:', authorize_url
#browser = webdriver.PhantomJS(service_log_path=os.path.join(tempfile.gettempdir(), 'ghostdriver.log'))
#browser = webdriver.PhantomJS(service_log_path=os.path.join(tempfile.gettempdir(), 'ghostdriver.log'), service_args=['--ignore-ssl-errors=true', '--ssl-protocol=tlsv1'])
# Change to rely on browser
print("We need to authorize access to Dropbox. Please visit the following URL and authorize the access:")
print(authorize_url)
print("")
code = raw_input("Input the code you got: ").strip()
#code = #raw_input("Enter the authorization code here: ").strip()
access_token, user_id = flow.finish(code)
with open(self.auth_file, 'w') as file:
file.write(access_token + "\n")
file.write(user_id + "\n")
dbg.info('Authentication successful')
return (access_token, user_id)
def _authorize(self, flow, storage, flags):
# copied and edited from tools.run_flow()
import getpass, time
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
dbg.info('Request access token from Google Drive')
oauth_callback = client.OOB_CALLBACK_URN
flow.redirect_uri = oauth_callback
authorize_url = flow.step1_get_authorize_url()
print("We need to authorize access to Google. Please visit the following URL and authorize the access:")
print(authorize_url)
code = raw_input("Input the code you got: ").strip()
try:
credential = flow.step2_exchange(code)
except client.FlowExchangeError, e:
sys.exit('Authentication has failed: %s' % e)
def process_action(self):
self.next = self.pick_action
dbg.infom(
'*', '%s:%s' % (prettify(self.action),
"same" if self.action.equiv() else "#R<changed#>"))
# dbg.infom('*', '[%s]%s:%s' % (prettify(self.action.tic),
# prettify(self.action),
# "S" if self.action.equiv() else "#R<D#>"))
# ipopov: conjecture:
# The purpose of this if/elif/else chain is to prevent the redo
# from continuing if any of the statements in the elif's return to
# the effect that other nodes have been updated during this pass
# through process_action(). Why do we care? Because this class is
# implemented as a state machine, and processing maintains the
# invariant that the action being redone now is the earliest one.
# But connect() and prepare_redo() may affect the invariant.
if (not self.action.cancel and self.action.equiv() and all([
self.cur_state[n] >= self.action
for n in self.action.outputs | set([self.action.actor])
])):
dbg.info('skipping', self.action)
for n in self.action.inputs:
self.set_cs(n, max(self.cur_state[n], self.action))
self.set_color(self.action, self.colors['skipped'])
elif not self.action.connect():
dbg.debug('retrying after connect')
elif not self.prepare_redo():
dbg.debug('retrying after prepare')
elif self.action.cancel:
if hasattr(self.action, 'cancel_hook'):
self.action.cancel_hook()
dbg.info('cancelled action', self.action)
self.next = self.update_state
self.set_color(self.action, self.colors['cancelled'])
else:
dbg.info('#G<redoing#> ', prettify(self.action))
self.save_ckpts(self.action.outputs, self.action)
try:
self.action.redo()
except mgrapi.ReschedulePendingException:
dbg.info('rescheduling %s' % prettify(self.action))
return
else:
self.next = self.update_state
self.set_color(self.action, self.colors['redone'])
class GoogleAPI(StorageAPI, AppendOnlyLog):
"google@auth : drive.google.com account with auth info"
UPLOAD_URL = 'https://www.googleapis.com/upload/drive/v2'
def __init__(self):
self.service = self._get_service()
self._num_retries = 0 # retry times in uploading
def copy(self):
return GoogleAPI()
def info_storage(self):
return 15 * GB
def sid(self):
return util.md5("google") % 10000
def _authorize(self, flow, storage, flags):
# copied and edited from tools.run_flow()
import getpass, time
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
dbg.info('Request access token from Google Drive')
oauth_callback = client.OOB_CALLBACK_URN
flow.redirect_uri = oauth_callback
authorize_url = flow.step1_get_authorize_url()
print(
"We need to authorize access to Google. Please visit the following URL and authorize the access:"
)
print(authorize_url)
code = raw_input("Input the code you got: ").strip()
try:
credential = flow.step2_exchange(code)
except client.FlowExchangeError, e:
sys.exit('Authentication has failed: %s' % e)
storage.put(credential)
credential.set_store(storage)
dbg.info('Authentication successful')
return credential
def _authorize(self, flow, storage, flags):
# copied and edited from tools.run_flow()
import getpass, time
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
dbg.info('Request access token from Google Drive')
oauth_callback = client.OOB_CALLBACK_URN
flow.redirect_uri = oauth_callback
authorize_url = flow.step1_get_authorize_url()
print(
"We need to authorize access to Google. Please visit the following URL and authorize the access:"
)
print(authorize_url)
code = raw_input("Input the code you got: ").strip()
try:
credential = flow.step2_exchange(code)
except client.FlowExchangeError, e:
sys.exit('Authentication has failed: %s' % e)
def test_bench_paxos(metasync, opts):
"bencmark latency of paxos with backends"
def new_index(srv, folder, prefix):
if services.slug(srv) == 'onedrive':
folder = '/Public' + folder
if not srv.exists(folder):
return 0
files = srv.listdir(folder)
cnt = 0
for fn in files:
if fn.startswith(prefix):
cnt += 1
return cnt
from paxos import PPaxosWorker
repeat = 5
client_num = [1, 2, 3, 4, 5]
backend_list = [["google"], ["dropbox"], ["onedrive"], ["box"],
["google", "dropbox", "onedrive"]]
results = [['Clients'] + [','.join(x) for x in backend_list]]
# start to test
for num in client_num:
for _ in range(repeat):
row = ['%d clients' % (num)]
for backend in backend_list:
dbg.info('Test paxos for %d clients and %s' %
(num, ','.join(backend)))
srvs = map(services.factory, backend)
# init log file
prefix = 'test-%d-%d' % (num, len(backend))
index = new_index(srvs[0], '/ppaxos', prefix)
path = '/ppaxos/%s.%d' % (prefix, index)
dbg.info(path)
for srv in srvs:
srv.init_log(path)
clients = []
for i in range(num):
storages = map(services.factory, backend)
worker = PPaxosWorker(storages, path)
clients.append(worker)
for worker in clients:
worker.start()
latency = []
master_latency = None
for worker in clients:
worker.join()
latency.append(worker.latency)
if (worker.master):
assert master_latency is None
master_latency = worker.latency
for worker in clients:
worker.join()
summary = ",".join(
map(str, [
min(latency),
max(latency),
util.median(latency), master_latency
]))
dbg.info("Result: %s" % summary)
row.append(summary)
results.append(row)
# tabularize
print "Item Format: min,max,median,master"
for row in results:
for e in row:
print "%s \t" % e,
print
def test_bstore_reconfig(metasync, opts):
"rebalancing all blobs when conf changes"
test_init(metasync, opts)
dbg.info("old config: %s" % metasync.mapping)
if __name__ == '__main__':
parser = optparse.OptionParser()
parser.add_option("--promo", "-p",
help="list of promo log",
default="data/lock-promo.log")
(opts, args) = parser.parse_args()
utils.install_pdb()
# pick the latest one from /tmp
log_d = utils.get_latest_file("/tmp/fss-*/")
if log_d is None:
# second choice
log_d = os.path.join(ROOT, "data", "sample-fss-output")
# perfered one
if len(args) > 1:
log_d = args[1]
# enable trace output on unittest
if not "unit" in log_d:
dbg.quiet(["trace"])
promo = load_promo_log(opts.promo)
dbg.info("# %d loaded, %s", len(promo), log_d)
pool = mp.Pool(mp.cpu_count())
for pn in Parser(log_d)._get_files():
pool.apply_async(explore_file, args=(pn, promo))
pool.close()
pool.join()
def cmd_reconfigure(self, backends, replica):
srvmap = {}
for tok in backends.split(","):
srv = services.factory(tok)
srvmap[srv.sid()] = srv
lst_services = srvmap.values()
mapconfig = []
lock_dic = {}
for srv in lst_services:
mapconfig.append((srv.sid(), srv.info_storage()/GB))
for srv in srvmap:
lock_dic[srv] = threading.Lock()
if srv not in self.srvmap:
srvmap[srv].putdir(self.get_remote_path("objects"))
for srv in self.srvmap:
if srv not in lock_dic:
lock_dic[srv] = threading.Lock()
beg = time.time()
self.mapping.reconfig(mapconfig, eval(replica))
end = time.time()
dbg.info("remap: %.3fs" % (end-beg))
beg = time.time()
lst_objs = self.blobstore.list()
added, removed = self.mapping.get_remapping(lst_objs)
nthreads = self.options.nthreads if self.options is not None else 2
#REFACTOR
def __put_next(srv, lst, lock):
dbg.job("submitted to: %s" % srv)
while True:
lock.acquire()
if(len(lst) == 0):
lock.release()
break
next = lst.pop()
lock.release()
if next is not None:
with open(self.get_local_obj_path(next), "rb") as f:
blob = f.read()
for tr in self.translators:
blob = tr.put(blob)
# XXX HACK
backoff = 0.5
remote_path = self.get_remote_obj_path(next)
while not srv.put(remote_path, blob):
time.sleep(backoff)
backoff *= 2
def __rm_next(srv, lst, lock):
dbg.job("submitted to: %s" % srv)
while True:
lock.acquire()
if(len(lst) == 0):
lock.release()
break
next = lst.pop()
lock.release()
if next is not None:
remote_path = self.get_remote_obj_path(next)
srv.rm(remote_path)
cnt_added = 0
for srv in added:
if(len(added[srv]) == 0): continue
cnt_added += len(added[srv])
for i in range(nthreads):
self.scheduler.submit(srvmap[srv], False, __put_next, added[srv], lock_dic[srv])
self._join()
end = time.time()
dbg.info("remap put: %.3fs" % (end-beg))
beg = time.time()
cnt_removed = 0
for srv in removed:
if(len(removed[srv]) == 0): continue
cnt_removed += len(removed[srv])
for i in range(nthreads):
self.scheduler.submit(self.srvmap[srv], False, __rm_next, removed[srv], lock_dic[srv])
self._join()
end = time.time()
dbg.info("remap rm: %.3fs" % (end-beg))
dbg.info("added %d, removed %d" % (cnt_added, cnt_removed))
def main():
atexit.register(retroctl.disable)
parser = optparse.OptionParser(usage="%prog [options] LOG-DIRECTORY")
parser.add_option("-d",
"--dry",
default=False,
action="store_true",
dest="dryrun",
help="dry run for repair")
parser.add_option("-a",
"--attk",
dest="attack",
metavar="NAME",
default="pick_attack_execve",
help="algorithm to pick attacker's node")
parser.add_option("-p",
"--profile",
default=False,
action="store_true",
dest="profile",
help="profiling execution time")
# hints
parser.add_option("-i",
"--inode",
dest="inode_hint",
metavar="FILE",
default=None,
help="attacker's binary")
(opts, args) = parser.parse_args()
if opts.profile:
runopts.set_profile()
if len(args) != 1 or not opts.inode_hint:
parser.print_usage()
exit(1)
# pick attack
attack = attacker.find_attack_node(args[0], opts.attack, opts.inode_hint)
osloader.set_logdir(args[0])
attack_node = mgrapi.RegisteredObject.by_name_load(attack)
print_tics_tacs()
if attack_node is None:
raise Exception('missing attack node', attack)
chkpt = max(c for c in attack_node.checkpoints
if c < min(attack_node.actions))
assert chkpt
assert len(attack_node.actions) == 1
#code.interact(local=locals())
#pdb.set_trace()
# cancel exec syscall actor
for a in attack_node.actions:
dbg.info("cancel: %s" % a.argsnode)
a.argsnode.data = None
a.cancel = True
dbg.info("pick:", chkpt)
runopts.set_dryrun(opts.dryrun)
ctrl.repair(attack_node, chkpt)
== "(S64 # $A3__struct_dentry_P->d_inode)")
assert(filter_out_non_args("(E # S_ISDIR($A1__struct_dentry_P->d_inode->i_mode))")
== "(E # S_ISDIR($A1__struct_dentry_P->d_inode->i_mode))")
assert(filter_out_non_args("(S64 # new->inode)")
== "(S64 # -)")
assert(filter_out_non_args("$A1_XXX")=="$A1_XXX")
assert(filter_out_non_args("xyz")=="-")
assert(filter_out_non_args("ext2_set_attr($A1__xxx, $A2__yyy, local)")
=="ext2_set_attr($A1__xxx, $A2__yyy, -)")
assert(filter_out_non_args("ext2_set_attr($A1__xxx, malloc(x, y, z), xxx)")
=="ext2_set_attr($A1__xxx, malloc(-, -, -), -)")
assert(filter_out_non_args("(S64 # $A4__unsigned_int) & (I # 4294967294)")
== "(S64 # $A4__unsigned_int) & (I # 4294967294)")
return
if __name__ == '__main__':
utils.install_pdb()
# get log dir
log_d = os.path.join(ROOT, "data", "sample-fss-output")
if len(sys.argv) > 1:
log_d = sys.argv[1]
dbg.info("> %s", log_d)
for path in Parser(log_d).parse():
print "<< Before normalization"
print(path)
npath = ArgNormalizer(path)
print ">> After normalization"
print(npath)
def test_mapping2(metasync, opts):
"test a new mapping scheme to tolerate rebalancing on node-failre"
from itertools import permutations
# goal
# 1. non-uniformly locating blobs, approximately reflecting storage size of each node
# 2. minimize realigning on a node failure
# design
# 0. node -> (node, storage)
# (e.g., (1,0), (1,1) if node 1 has 2G storage)
# 1. fixed hspace, where h(blob) % hspace = index
# (hspace any large number, NOT depending on len(nodes))
# 2. detmap[index] -> a group of nodes
# (a group includes all nodes, but different order see 3)
# 3. order nodes in a group, by hash(index, node)
# (so deterministic)
# 4. in each group, pick first #replication nodes
# failure
# node change
# replication
# => in all of above situations, only blobs in old/new node
# will be re-balanced
#
def uniq(lst, n):
rtn = []
for (i, _) in lst:
if i in rtn:
continue
rtn.append(i)
if len(rtn) == n:
break
return rtn
replication = 2
config = [(0, 2), (1, 4), (2, 4), (3, 2)]
hspace = 20
detmap = DetMap2(config, hspace, replication)
stats = [0] * len(config)
for (k, v) in enumerate(detmap.mapinfo[0].detmap):
"""
# doesn't it need to calculate over uniq?
for i in range(replication):
stats[v[i][0]] += 1
"""
for i in uniq(v, replication):
stats[i] += 1
if opts.slow:
dbg.info("%-2s: [" % k)
for (n, priority) in v:
dbg.info(" %s: %s" % (n, priority))
dbg.info("]")
else:
dbg.info("%-2s: %s -> %s" \
% (k, [e[0] for e in v], detmap.get_mapping(k)))
# approximately reflect storage?
for (i, k) in enumerate(stats):
dbg.info("%s (%s GB) -> #%s" % (i, config[i][1], k))
def test_bench_disk_paxos(metasync, opts):
"test disk paxos"
"bencmark latency of paxos with backends"
from disk_paxos import DiskPaxosWorker
repeat = 5
client_num = [1, 2, 3, 4, 5]
backend_list = [["google"], ["dropbox"], ["onedrive"], ["box"],
["google", "dropbox", "onedrive"]]
results = [['Clients'] + [','.join(x) for x in backend_list]]
# start to test
for num in client_num:
for num_prop in range(1, num + 1):
for _ in range(repeat):
row = ['%d/%d clients' % (num_prop, num)]
for backend in backend_list:
srvs = map(services.factory, backend)
dbg.info('Test paxos for %d/%d clients and %s' %
(num_prop, num, ','.join(backend)))
# initialize all disk blocks
blockList = []
for i in range(num):
path = '/diskpaxos/client%d' % i
for srv in srvs:
if not srv.exists(path):
srv.put(path, '')
else:
srv.update(path, '')
blockList.append(path)
clients = []
for i in range(num_prop):
storages = map(services.factory, backend)
worker = DiskPaxosWorker(storages, blockList[i],
blockList)
clients.append(worker)
#dbg.dbg('client %d %s' % (i, worker.clientid))
for worker in clients:
worker.start()
latency = []
master_latency = None
for worker in clients:
worker.join()
latency.append(worker.latency)
if (worker.master):
assert master_latency is None
master_latency = worker.latency
for worker in clients:
worker.join()
summary = ",".join(
map(str, [
min(latency),
max(latency),
util.median(latency), master_latency
]))
dbg.info("Result: %s" % summary)
row.append(summary)
results.append(row)
# tabularize
print "Item Format: min,max,median,master"
for row in results:
for e in row:
print "%s \t" % e,
print
def pick_attack_execve(r):
"""choose execve containing 'attacker' string"""
if r.name == "execve" and str(r).find("attacker") != -1:
dbg.info("#R<attack#>:%s" % r)
return PICK_ATTACK|PICK_STOP
def request_token():
dbg.info('Request access token from Baidu')
code = OAuth2._authorize()
token = OAuth2._token_request('authorization_code', code=code)
dbg.info('Authentication successful')
return token
def cmd_reconfigure(self, backends, replica):
srvmap = {}
for tok in backends.split(","):
srv = services.factory(tok)
srvmap[srv.sid()] = srv
lst_services = srvmap.values()
mapconfig = []
lock_dic = {}
for srv in lst_services:
mapconfig.append((srv.sid(), srv.info_storage() / GB))
for srv in srvmap:
lock_dic[srv] = threading.Lock()
if srv not in self.srvmap:
srvmap[srv].putdir(self.get_remote_path("objects"))
for srv in self.srvmap:
if srv not in lock_dic:
lock_dic[srv] = threading.Lock()
beg = time.time()
self.mapping.reconfig(mapconfig, eval(replica))
end = time.time()
dbg.info("remap: %.3fs" % (end - beg))
beg = time.time()
lst_objs = self.blobstore.list()
added, removed = self.mapping.get_remapping(lst_objs)
nthreads = self.options.nthreads if self.options is not None else 2
#REFACTOR
def __put_next(srv, lst, lock):
dbg.job("submitted to: %s" % srv)
while True:
lock.acquire()
if (len(lst) == 0):
lock.release()
break
next = lst.pop()
lock.release()
if next is not None:
with open(self.get_local_obj_path(next), "rb") as f:
blob = f.read()
for tr in self.translators:
blob = tr.put(blob)
# XXX HACK
backoff = 0.5
remote_path = self.get_remote_obj_path(next)
while not srv.put(remote_path, blob):
time.sleep(backoff)
backoff *= 2
def __rm_next(srv, lst, lock):
dbg.job("submitted to: %s" % srv)
while True:
lock.acquire()
if (len(lst) == 0):
lock.release()
break
next = lst.pop()
lock.release()
if next is not None:
remote_path = self.get_remote_obj_path(next)
srv.rm(remote_path)
cnt_added = 0
for srv in added:
if (len(added[srv]) == 0): continue
cnt_added += len(added[srv])
for i in range(nthreads):
self.scheduler.submit(srvmap[srv], False, __put_next,
added[srv], lock_dic[srv])
self._join()
end = time.time()
dbg.info("remap put: %.3fs" % (end - beg))
beg = time.time()
cnt_removed = 0
for srv in removed:
if (len(removed[srv]) == 0): continue
cnt_removed += len(removed[srv])
for i in range(nthreads):
self.scheduler.submit(self.srvmap[srv], False, __rm_next,
removed[srv], lock_dic[srv])
self._join()
end = time.time()
dbg.info("remap rm: %.3fs" % (end - beg))
dbg.info("added %d, removed %d" % (cnt_added, cnt_removed))
def test_bench_disk_paxos(metasync, opts):
"test disk paxos"
"bencmark latency of paxos with backends"
from disk_paxos import DiskPaxosWorker
repeat = 5
client_num = [1, 2, 3, 4, 5]
backend_list = [["google"], ["dropbox"], ["onedrive"], ["box"], ["google", "dropbox", "onedrive"]]
results = [['Clients'] + [','.join(x) for x in backend_list]]
# start to test
for num in client_num:
for num_prop in range(1, num + 1):
for _ in range(repeat):
row = ['%d/%d clients' % (num_prop, num)]
for backend in backend_list:
srvs = map(services.factory, backend)
dbg.info('Test paxos for %d/%d clients and %s' % (num_prop, num, ','.join(backend)))
# initialize all disk blocks
blockList = []
for i in range(num):
path = '/diskpaxos/client%d' % i
for srv in srvs:
if not srv.exists(path):
srv.put(path, '')
else:
srv.update(path, '')
blockList.append(path)
clients = []
for i in range(num_prop):
storages = map(services.factory, backend)
worker = DiskPaxosWorker(storages, blockList[i], blockList)
clients.append(worker)
#dbg.dbg('client %d %s' % (i, worker.clientid))
for worker in clients:
worker.start()
latency = []
master_latency = None
for worker in clients:
worker.join()
latency.append(worker.latency)
if (worker.master):
assert master_latency is None
master_latency = worker.latency
for worker in clients:
worker.join()
summary = ",".join(map(str,[min(latency), max(latency), util.median(latency), master_latency]))
dbg.info("Result: %s" % summary)
row.append(summary)
results.append(row)
# tabularize
print "Item Format: min,max,median,master"
for row in results:
for e in row:
print "%s \t" % e,
print
def test_mapping2(metasync, opts):
"test a new mapping scheme to tolerate rebalancing on node-failre"
from itertools import permutations
# goal
# 1. non-uniformly locating blobs, approximately reflecting storage size of each node
# 2. minimize realigning on a node failure
# design
# 0. node -> (node, storage)
# (e.g., (1,0), (1,1) if node 1 has 2G storage)
# 1. fixed hspace, where h(blob) % hspace = index
# (hspace any large number, NOT depending on len(nodes))
# 2. detmap[index] -> a group of nodes
# (a group includes all nodes, but different order see 3)
# 3. order nodes in a group, by hash(index, node)
# (so deterministic)
# 4. in each group, pick first #replication nodes
# failure
# node change
# replication
# => in all of above situations, only blobs in old/new node
# will be re-balanced
#
def uniq(lst, n):
rtn = []
for (i, _) in lst:
if i in rtn:
continue
rtn.append(i)
if len(rtn) == n:
break
return rtn
replication = 2
config = [(0,2), (1,4), (2,4), (3,2)]
hspace = 20
detmap = DetMap2(config, hspace, replication)
stats = [0] * len(config)
for (k, v) in enumerate(detmap.mapinfo[0].detmap):
"""
# doesn't it need to calculate over uniq?
for i in range(replication):
stats[v[i][0]] += 1
"""
for i in uniq(v, replication):
stats[i] += 1
if opts.slow:
dbg.info("%-2s: [" % k)
for (n, priority) in v:
dbg.info(" %s: %s" % (n, priority))
dbg.info("]")
else:
dbg.info("%-2s: %s -> %s" \
% (k, [e[0] for e in v], detmap.get_mapping(k)))
# approximately reflect storage?
for (i, k) in enumerate(stats):
dbg.info("%s (%s GB) -> #%s" % (i, config[i][1], k))
"(E # S_ISDIR($A1__struct_dentry_P->d_inode->i_mode))") ==
"(E # S_ISDIR($A1__struct_dentry_P->d_inode->i_mode))")
assert (filter_out_non_args("(S64 # new->inode)") == "(S64 # -)")
assert (filter_out_non_args("$A1_XXX") == "$A1_XXX")
assert (filter_out_non_args("xyz") == "-")
assert (filter_out_non_args("ext2_set_attr($A1__xxx, $A2__yyy, local)") ==
"ext2_set_attr($A1__xxx, $A2__yyy, -)")
assert (
filter_out_non_args("ext2_set_attr($A1__xxx, malloc(x, y, z), xxx)") ==
"ext2_set_attr($A1__xxx, malloc(-, -, -), -)")
assert (filter_out_non_args("(S64 # $A4__unsigned_int) & (I # 4294967294)")
== "(S64 # $A4__unsigned_int) & (I # 4294967294)")
return
if __name__ == '__main__':
utils.install_pdb()
# get log dir
log_d = os.path.join(ROOT, "data", "sample-fss-output")
if len(sys.argv) > 1:
log_d = sys.argv[1]
dbg.info("> %s", log_d)
for path in Parser(log_d).parse():
print "<< Before normalization"
print(path)
npath = ArgNormalizer(path)
print ">> After normalization"
print(npath)
def test_bstore_reconfig(metasync, opts):
"rebalancing all blobs when conf changes"
test_init(metasync, opts)
dbg.info("old config: %s" % metasync.mapping)
def test_bench_paxos2(metasync, opts):
"bencmark latency of paxos with backends"
def new_index(srv, folder, prefix):
if not srv.exists(folder):
return 0
files = srv.listdir(folder)
cnt = 0
for fn in files:
if fn.startswith(prefix):
cnt += 1
return cnt
from paxos import PPaxosWorker2
repeat = 5
client_num = [1, 2, 3, 4, 5]
backend_list = [["dropbox"], ["onedrive"]]
results = [['Clients'] + [','.join(x) for x in backend_list]]
# start to test
for num in client_num:
for _ in range(repeat):
row = ['%d clients' % (num)]
for backend in backend_list:
dbg.info('Test paxos for %d clients and %s' % (num, ','.join(backend)))
srvs = map(services.factory, backend)
# init log file
prefix = 'test2-%d-%d' % (num , len(backend))
index = new_index(srvs[0], '/ppaxos', prefix)
path = '/ppaxos/%s.%d' % (prefix, index)
dbg.info(path)
for srv in srvs:
srv.init_log2(path)
clients = []
for i in range(num):
storages = map(services.factory, backend)
worker = PPaxosWorker2(storages, path)
clients.append(worker)
for worker in clients:
worker.start()
for worker in clients:
worker.join()
latency = []
master_latency = None
for worker in clients:
latency.append(worker.latency)
if (worker.master):
assert master_latency is None
master_latency = worker.latency
assert master_latency is not None
summary = ",".join(map(str,[min(latency), max(latency), util.median(latency), master_latency]))
dbg.info("Result: %s" % summary)
row.append(summary)
results.append(row)
# tabularize
print "Item Format: min,max,median,master"
for row in results:
for e in row:
print "%s \t" % e,
print