Example #1
0
def pick_attack_write(r):
	"""choose write system call containing 'attack' string"""

	if r.name == "write" and		\
		    str(r).find("attack") != -1:
		dbg.info("pick:#R<%s#> %s" % (r.tac, r))
		return PICK_ATTACK|PICK_STOP
Example #2
0
def repair(db, attack_edges, dryrun):
    global rollbacks
    global queued_actions

    rollbacks = []
    q = PriorityQueue()

    while True:
        try:
            queued_actions = []
            repair_loop(db, q, attack_edges, dryrun)
            return
        except TimeLoopException as tle:
            dbg.ctrl("time loop: %s" % tle.n)
            dbg.info("node list: %s" % str(rollbacks))
            for a in queued_actions:
                if hasattr(a, 'todo'): delattr(a, 'todo')
                if hasattr(a, 'redone'): delattr(a, 'redone')
                if hasattr(a, 'new_action'): delattr(a, 'new_action')
            rbsave = rollbacks
            rollbacks = []
            q = PriorityQueue()
            for (n, tic) in rbsave:
                if hasattr(n, 'rollback_ts'): delattr(n, 'rollback_ts')
            for (n, tic) in rbsave:
                ne = rollback_nodes([n], tic)
                add_queue(q, ne)
Example #3
0
 def _authorize(self):
     dbg.info('Request access token from Dropbox')
     flow = DropboxOAuth2FlowNoRedirect(APP_KEY, APP_SECRET)
     authorize_url = flow.start()
     # print 'Open auth url:', authorize_url
     #browser = webdriver.PhantomJS(service_log_path=os.path.join(tempfile.gettempdir(), 'ghostdriver.log'))
     #browser = webdriver.PhantomJS(service_log_path=os.path.join(tempfile.gettempdir(), 'ghostdriver.log'), service_args=['--ignore-ssl-errors=true', '--ssl-protocol=tlsv1'])
     # Change to rely on browser
     print(
         "We need to authorize access to Dropbox. Please visit the following URL and authorize the access:"
     )
     print(authorize_url)
     print("")
     code = raw_input("Input the code you got: ").strip()
     #code = #raw_input("Enter the authorization code here: ").strip()
     try:
         oauth_result = flow.finish(code)
         with open(self.auth_file, 'w') as file:
             file.write(oauth_result.access_token + "\n")
             file.write(oauth_result.user_id + "\n")
         dbg.info('Authentication successful')
         return (oauth_result.access_token, oauth_result.user_id)
     except Exception, e:
         print('Error: %s' % (e, ))
         return
Example #4
0
 def __search_checkpoints(self, n, x):
     for p in glob(
             os.path.join(kutil.fsget(x.dev), ".inodes", str(x.ino)) + "*"):
         dbg.info("found inodes: %s" % p)
         info = p.split(".")
         tic = (int(info[-2]) * 10**9 + int(info[-1]), 0)
         n.checkpoints.add(fsmgr.FileCheckpoint(tic, tic, x.dev, x.ino, p))
Example #5
0
def repair2(repair_ckpts, gui=False):
    dbg.info('repairing with', repair_ckpts)
    if gui:
        ctrl = GuiRepairCtrl(repair_ckpts)
    else:
        ctrl = BasicRepairCtrl(repair_ckpts)
    ctrl.repair()
Example #6
0
 def refresh_token(refresh_token):
   dbg.info('Refresh access token from Baidu')
   if not refresh_token:
     raise Exception('Refresh token is null')
   token = OAuth2._token_request('refresh_token', refresh_token=refresh_token)
   dbg.info('Refresh successful')
   return token
Example #7
0
    def cmd_update(self):
        master = self.get_uptodate_master(False)
        # already up-to-date
        prev = self.get_prev_value()
        if (master == prev):
            self.update_head_and_prev(master)
            return True

        head = self.get_head_and_config()
        # XXX: need to check if non-checked in but modified files.
        if (head == prev):
            headstr = head.split(".")[0]
            if headstr == '':
                self.update_head_and_prev(master)
                return self.restore_from_master()

            masterstr = master.split(".")[0]
            self.update_changed(headstr, masterstr)
        else:
            ### need to merge
            raise Exception('Merge required')

        self.update_head_and_prev(master)
        self.blobstore.rootblob = None
        dbg.info("update done %s" % time.ctime())
        return True
Example #8
0
def cmd_repair(fd, args):
    """repairing attacks: hint attacker"""
    if len(args) < 1:
        cmd_help(fd)
        return

    hint = args[0]
    pick = "pick_attack_execve" if len(args) < 2 else args[1]
    name = attacker.find_attack_node(logd, pick, hint)

    attack = mgrapi.RegisteredObject.by_name_load(name)
    if attack is None:
        dbg.error('missing attack node:', name)
        return

    chkpt = max(c for c in attack.checkpoints if c < min(attack.actions))

    assert chkpt
    assert len(attack.actions) == 1

    for a in attack.actions:
        dbg.info("cancel: %s" % a.argsnode)
        a.argsnode.data = None
        a.cancel = True

    dbg.info("pick:", chkpt)

    The(CtrlFSM).set_state("repairing")
    ctrl.repair(attack, chkpt)
    The(CtrlFSM).set_state("init")

    rtn_cmd(fd, "done")
Example #9
0
def pick_execve(r):
	"""choose first successful execve"""

	if r.name == "execve" and		\
		    hasattr(r, "ret") and	\
		    r.ret == 0:
		dbg.info("pick:#R<%s#> %s" % (r.tac, r))
		return PICK_ATTACK|PICK_STOP
Example #10
0
 def rollback(self, n, cp):
     #code.interact(local=locals())
     dbg.info('#G<rollback#> %s -> %s' % (prettify(n), (prettify(cp))))
     n.rollback(cp)
     self.set_cs(n, cp)
     self.mod_state[n] = cp
     self.set_dirty(n, False)
     self.set_color(cp, self.colors['redone'])
Example #11
0
  def refresh(self):
    if 'refresh_token' in self._token:
      token = OAuth2.refresh_token(self._token['refresh_token'])
    else:
      dbg.info('No refresh token in the access token')
      token = OAuth2.request_token()

    self.set_token(token)
Example #12
0
def cmd_test(metasync, args, opts):
    "quick test (e.g., metasync test {%s})"

    # invoke pdb when failed in testing
    util.install_pdb()

    tmpdir = tempfile.mkdtemp()
    root = os.path.join(tmpdir, "repo")
    util.mkdirs(root)
    metasync = MetaSync(root)

    # opts for sub test routines
    opts.root = root
    opts.tmpdir = tmpdir
    opts.encrypt_key = "testkey" if opts.encrypt else ""

    dbg.info("root: %s" % root)
    dbg.info("args: %s" % args)
    dbg.info("opts: ")
    for (k, v) in vars(opts).iteritems():
        dbg.info("  %-12s = %s" % (k, v))

    alltests = dict(test.get_all_tests())
    if any(case not in alltests for case in args):
        dbg.err("no such a test case: %s" % args)
        alltests["help"](metasync, opts)
        exit(1)

    # print help if no test case is provided
    if len(args) == 0:
        args = ["help"]

    for case in args:
        dbg.info("#R<testing %s#> (%s)" % (case, alltests[case].__doc__))
        alltests[case](metasync, opts)

    # poorman's tree
    def tree(path):
        for root, dirs, files in os.walk(path):
            base = os.path.basename(root)
            idnt = '    ' * (root.replace(path, '').count(os.sep))
            print('%s%s/' % (idnt, base))
            for f in files:
                pn = os.path.join(root, f)
                print('    %s%s [%s]' % (idnt, f, os.stat(pn).st_size))

                # dump some content of blobs
                if opts.dump and "objects" == base:
                    print(util.hexdump(util.read_file(pn, 32*2)))
                    print

    # dump root
    if not opts.notree:
        tree(tmpdir)

    # cleanup tmpdir
    if not opts.keep:
        shutil.rmtree(tmpdir)
Example #13
0
    def repair(self):
        """Process all actions until finished."""
        while (self.has_next()):
            self.next()

        pi = mgrapi.PlusInfinity()
        self.save_ckpts(self.mod_state.keys(), mgrapi.TimeInterval(pi, pi))
        dbg.info('repair done')
        PathidTable.dumptables()
Example #14
0
def test_all(metasync, opts):
    "run all test cases"

    opts.notree = True
    for (n, f) in get_all_tests():
        if n == "all" or n.startswith("bench_"):
            continue
        dbg.info("#R<testing %s#> (%s)" % (n, f.__doc__))
        f(metasync, opts)
Example #15
0
def test_all(metasync, opts):
    "run all test cases"

    opts.notree = True
    for (n, f) in get_all_tests():
        if n == "all" or n.startswith("bench_"):
            continue
        dbg.info("#R<testing %s#> (%s)" % (n, f.__doc__))
        f(metasync, opts)
Example #16
0
  def _authorize(self, flow, storage, flags):
    # copied and edited from tools.run_flow()
    import getpass, time
    from selenium import webdriver 
    from selenium.webdriver.common.keys import Keys
    from selenium.webdriver.support.ui import WebDriverWait
    from selenium.webdriver.support import expected_conditions as EC
    from selenium.webdriver.common.by import By

    dbg.info('Request access token from Google Drive')

    oauth_callback = client.OOB_CALLBACK_URN
    flow.redirect_uri = oauth_callback
    authorize_url = flow.step1_get_authorize_url()
    #print 'Open auth url:', authorize_url
    import tempfile
    browser = webdriver.PhantomJS(service_log_path=os.path.join(tempfile.gettempdir(), 'ghostdriver.log'))
    browser.get(authorize_url)
    try:
      wait = WebDriverWait(browser, 30)
      email = wait.until(EC.element_to_be_clickable((By.ID, "Email")))
    except:
      print(browser.title)
      print(browser.page_source)
      browser.quit()
      raise Exception("timeout for authorization")
    email.send_keys(raw_input("Enter your Google Drive email:"))
    pwd = browser.find_element_by_id("Passwd")
    pwd.send_keys(getpass.getpass("Enter your Google Drive password:"******"signIn")
    btn.click()
    try:
      wait = WebDriverWait(browser, 30)
      btn = wait.until(EC.element_to_be_clickable((By.ID, "submit_approve_access")))
    except:
      print(browser.title)
      print(browser.page_source)
      browser.quit()
      raise Exception("timeout for authorization")
    btn.click()
    try:
      wait = WebDriverWait(browser, 30)
      wait.until(EC.title_contains("Success code"))
    except:
      print(browser.title)
      print(browser.page_source)
      browser.quit()
      raise Exception("timeout for authorization")
    code = browser.title.split("=")[1]
    browser.quit()

    try:
      credential = flow.step2_exchange(code)
    except client.FlowExchangeError, e:
      sys.exit('Authentication has failed: %s' % e)
Example #17
0
  def _authorize(self):
    dbg.info('Request access token from Dropbox')
    flow = DropboxOAuth2FlowNoRedirect(APP_KEY, APP_SECRET)
    authorize_url = flow.start()
    # print 'Open auth url:', authorize_url
    #browser = webdriver.PhantomJS(service_log_path=os.path.join(tempfile.gettempdir(), 'ghostdriver.log'))
    browser = webdriver.PhantomJS(service_log_path=os.path.join(tempfile.gettempdir(), 'ghostdriver.log'), service_args=['--ignore-ssl-errors=true', '--ssl-protocol=tlsv1'])
    browser.get(authorize_url)
    try:
      wait = WebDriverWait(browser, 30)
      email = wait.until(EC.element_to_be_clickable((By.XPATH, "//input[@name='login_email']")))
    except:
      print(browser.title)
      print(browser.page_source)
      browser.quit()
      raise Exception("timeout for authorization")
    email.send_keys(raw_input("Enter your Dropbox email:"))
    pwd = browser.find_element_by_xpath("//input[@name='login_password']") 
    pwd.send_keys(getpass.getpass("Enter your Dropbox password:"******"allow_access")))
    except:
      print(browser.title)
      print(browser.page_source)
      browser.quit()
      raise Exception("timeout for authorization")
    btn.click()
    try:
      wait = WebDriverWait(browser, 30)
      auth_code = wait.until(EC.presence_of_element_located((By.ID, "auth-code")))
    except:
      print(browser.title)
      print(browser.page_source)
      browser.quit()
      raise Exception("timeout for authorization")
    print(browser.title)
    #auth_code = browser.find_element_by_id("auth-code")
    code = auth_code.text

    browser.quit()

    #code = #raw_input("Enter the authorization code here: ").strip()
    access_token, user_id = flow.finish(code)
    with open(self.auth_file, 'w') as file:
      file.write(access_token + "\n")
      file.write(user_id + "\n")

    dbg.info('Authentication successful')

    return (access_token, user_id)
Example #18
0
def cmd_test(metasync, args, opts):
    "quick test (e.g., metasync test {%s})"

    # invoke pdb when failed in testing
    util.install_pdb()

    tmpdir = tempfile.mkdtemp()
    root = os.path.join(tmpdir, "repo")
    util.mkdirs(root)
    metasync = MetaSync(root)

    # opts for sub test routines
    opts.root = root
    opts.tmpdir = tmpdir
    opts.encrypt_key = "testkey" if opts.encrypt else ""

    dbg.info("root: %s" % root)
    dbg.info("args: %s" % args)
    dbg.info("opts: ")
    for (k, v) in vars(opts).iteritems():
        dbg.info("  %-12s = %s" % (k, v))

    alltests = dict(test.get_all_tests())
    if any(case not in alltests for case in args):
        dbg.err("no such a test case: %s" % args)
        alltests["help"](metasync, opts)
        exit(1)
Example #19
0
    def __do_check(m):
        assert len([n for n in m.distrib if n == 1]) == len(m.distrib)/10*2
        assert len([n for n in m.distrib if n == 2]) == len(m.distrib)/10*5
        assert len([n for n in m.distrib if n == 3]) == len(m.distrib)/10*3

        copyset = set()
        for i in range(len(m.distrib)):
            c = m.get_mapping(i)
            assert len(c) == m.replica
            copyset.add(tuple(c))

        assert len(copyset) < len(m.distrib)

        dbg.info("mapping: %s ..." % str(m.distrib)[:30])
        dbg.info("copyset: %s" % str(list(copyset)))
Example #20
0
    def __do_check(m):
        assert len([n for n in m.distrib if n == 1]) == len(m.distrib) / 10 * 2
        assert len([n for n in m.distrib if n == 2]) == len(m.distrib) / 10 * 5
        assert len([n for n in m.distrib if n == 3]) == len(m.distrib) / 10 * 3

        copyset = set()
        for i in range(len(m.distrib)):
            c = m.get_mapping(i)
            assert len(c) == m.replica
            copyset.add(tuple(c))

        assert len(copyset) < len(m.distrib)

        dbg.info("mapping: %s ..." % str(m.distrib)[:30])
        dbg.info("copyset: %s" % str(list(copyset)))
Example #21
0
def test_merge(metasync, opts):
    clone = test_clone(metasync, opts)
    new_files = [3072, 4096]
    metasyncs = [metasync, clone]

    for i in range(2):
        dbg.info("checkin %d" % i)
        pn = os.path.join(metasyncs[i].path_root, "file-%s" % new_files[i])
        util.create_random_file(pn, new_files[i])
        metasyncs[i].cmd_checkin(pn)

    metasync.cmd_push()
    clone.cmd_fetch()
    assert not clone.cmd_push()
    clone.cmd_update()
    assert clone.cmd_push()
Example #22
0
def test_bstore_iterate(metasync, opts):
    "walk over all files in a service, and check if correctly distributed"

    test_init(metasync, opts)

    hashes = set()
    for srv in metasync.services:
        for hv in metasync.bstore_iter_remote(srv):
            dbg.info("%-10s: %s" % (srv, hv))
            hashes.add(hv)

    # covered by local's bstore?
    for hv in metasync.bstore_iter():
        hashes.remove(hv)

    assert len(hashes) == 0
Example #23
0
def test_merge(metasync, opts):
    clone = test_clone(metasync, opts)
    new_files = [3072, 4096]
    metasyncs = [metasync, clone]

    for i in range(2):
        dbg.info("checkin %d" % i)
        pn = os.path.join(metasyncs[i].path_root, "file-%s" % new_files[i]) 
        util.create_random_file(pn, new_files[i])
        metasyncs[i].cmd_checkin(pn)

    metasync.cmd_push()
    clone.cmd_fetch()
    assert not clone.cmd_push()
    clone.cmd_update()
    assert clone.cmd_push()
Example #24
0
def test_bstore_iterate(metasync, opts):
    "walk over all files in a service, and check if correctly distributed"

    test_init(metasync, opts)

    hashes = set()
    for srv in metasync.services:
        for hv in metasync.bstore_iter_remote(srv):
            dbg.info("%-10s: %s" % (srv, hv))
            hashes.add(hv)

    # covered by local's bstore?
    for hv in metasync.bstore_iter():
        hashes.remove(hv)

    assert len(hashes) == 0
Example #25
0
def test_mapping_reconfig(metasync, opts):
    "test reconfiguration/versioning of mapping strategies"

    opts.notree = True

    m = DetMap([(1, 2*GB), (2, 5*GB), (3, 2*GB)])
    m.reconfig([(1, 2*GB), (2, 5*GB)])

    # two version of mapping info
    assert len(m.mapinfo) == 2
    for (ver, conf) in m.mapinfo.iteritems():
        dbg.info("%s -> %s"  % (ver, conf.store()))

    # where to remap of hash value: 0
    #  0: [1,2] -> [1,2]
    #  1: [2,3] -> [2,1]
    assert m.get_remmaping(0) == []
    assert m.get_remmaping(1) == [1]
Example #26
0
def test_mapping_reconfig(metasync, opts):
    "test reconfiguration/versioning of mapping strategies"

    opts.notree = True

    m = DetMap([(1, 2 * GB), (2, 5 * GB), (3, 2 * GB)])
    m.reconfig([(1, 2 * GB), (2, 5 * GB)])

    # two version of mapping info
    assert len(m.mapinfo) == 2
    for (ver, conf) in m.mapinfo.iteritems():
        dbg.info("%s -> %s" % (ver, conf.store()))

    # where to remap of hash value: 0
    #  0: [1,2] -> [1,2]
    #  1: [2,3] -> [2,1]
    assert m.get_remmaping(0) == []
    assert m.get_remmaping(1) == [1]
Example #27
0
    def cmd_update(self):
        master = self.get_uptodate_master()
        # already up-to-date
        prev = self.get_prev_value()
        if (master == prev):
            return True

        head = self.get_head_and_config()
        # XXX: need to check if non-checked in but modified files.
        if (head == prev):
            self.update_changed(head.split(".")[0], master.split(".")[0])
        else:
            ### need to merge
            raise Exception('Merge required')

        self.update_head_and_prev(master)
        self.blobstore.rootblob = None
        dbg.info("update done %s" % time.ctime())
        return True
Example #28
0
  def _authorize(self):
    dbg.info('Request access token from Dropbox')
    flow = DropboxOAuth2FlowNoRedirect(APP_KEY, APP_SECRET)
    authorize_url = flow.start()
    # print 'Open auth url:', authorize_url
    #browser = webdriver.PhantomJS(service_log_path=os.path.join(tempfile.gettempdir(), 'ghostdriver.log'))
    #browser = webdriver.PhantomJS(service_log_path=os.path.join(tempfile.gettempdir(), 'ghostdriver.log'), service_args=['--ignore-ssl-errors=true', '--ssl-protocol=tlsv1'])
    # Change to rely on browser
    print("We need to authorize access to Dropbox. Please visit the following URL and authorize the access:")
    print(authorize_url)
    print("")
    code = raw_input("Input the code you got: ").strip()
    #code = #raw_input("Enter the authorization code here: ").strip()
    access_token, user_id = flow.finish(code)
    with open(self.auth_file, 'w') as file:
      file.write(access_token + "\n")
      file.write(user_id + "\n")

    dbg.info('Authentication successful')

    return (access_token, user_id)
Example #29
0
  def _authorize(self, flow, storage, flags):
    # copied and edited from tools.run_flow()
    import getpass, time
    from selenium import webdriver 
    from selenium.webdriver.common.keys import Keys
    from selenium.webdriver.support.ui import WebDriverWait
    from selenium.webdriver.support import expected_conditions as EC
    from selenium.webdriver.common.by import By

    dbg.info('Request access token from Google Drive')

    oauth_callback = client.OOB_CALLBACK_URN
    flow.redirect_uri = oauth_callback
    authorize_url = flow.step1_get_authorize_url()
    print("We need to authorize access to Google. Please visit the following URL and authorize the access:")
    print(authorize_url)
    code = raw_input("Input the code you got: ").strip()

    try:
      credential = flow.step2_exchange(code)
    except client.FlowExchangeError, e:
      sys.exit('Authentication has failed: %s' % e)
Example #30
0
    def process_action(self):
        self.next = self.pick_action
        dbg.infom(
            '*', '%s:%s' % (prettify(self.action),
                            "same" if self.action.equiv() else "#R<changed#>"))

        # dbg.infom('*', '[%s]%s:%s' % (prettify(self.action.tic),
        #         prettify(self.action),
        #         "S" if self.action.equiv() else "#R<D#>"))

        # ipopov: conjecture:
        # The purpose of this if/elif/else chain is to prevent the redo
        # from continuing if any of the statements in the elif's return to
        # the effect that other nodes have been updated during this pass
        # through process_action(). Why do we care? Because this class is
        # implemented as a state machine, and processing maintains the
        # invariant that the action being redone now is the earliest one.
        # But connect() and prepare_redo() may affect the invariant.

        if (not self.action.cancel and self.action.equiv() and all([
                self.cur_state[n] >= self.action
                for n in self.action.outputs | set([self.action.actor])
        ])):
            dbg.info('skipping', self.action)
            for n in self.action.inputs:
                self.set_cs(n, max(self.cur_state[n], self.action))
            self.set_color(self.action, self.colors['skipped'])
        elif not self.action.connect():
            dbg.debug('retrying after connect')
        elif not self.prepare_redo():
            dbg.debug('retrying after prepare')
        elif self.action.cancel:
            if hasattr(self.action, 'cancel_hook'):
                self.action.cancel_hook()
            dbg.info('cancelled action', self.action)
            self.next = self.update_state
            self.set_color(self.action, self.colors['cancelled'])
        else:
            dbg.info('#G<redoing#> ', prettify(self.action))
            self.save_ckpts(self.action.outputs, self.action)
            try:
                self.action.redo()
            except mgrapi.ReschedulePendingException:
                dbg.info('rescheduling %s' % prettify(self.action))
                return
            else:
                self.next = self.update_state
                self.set_color(self.action, self.colors['redone'])
Example #31
0
class GoogleAPI(StorageAPI, AppendOnlyLog):
    "google@auth  : drive.google.com account with auth info"

    UPLOAD_URL = 'https://www.googleapis.com/upload/drive/v2'

    def __init__(self):
        self.service = self._get_service()
        self._num_retries = 0  # retry times in uploading

    def copy(self):
        return GoogleAPI()

    def info_storage(self):
        return 15 * GB

    def sid(self):
        return util.md5("google") % 10000

    def _authorize(self, flow, storage, flags):
        # copied and edited from tools.run_flow()
        import getpass, time
        from selenium import webdriver
        from selenium.webdriver.common.keys import Keys
        from selenium.webdriver.support.ui import WebDriverWait
        from selenium.webdriver.support import expected_conditions as EC
        from selenium.webdriver.common.by import By

        dbg.info('Request access token from Google Drive')

        oauth_callback = client.OOB_CALLBACK_URN
        flow.redirect_uri = oauth_callback
        authorize_url = flow.step1_get_authorize_url()
        print(
            "We need to authorize access to Google. Please visit the following URL and authorize the access:"
        )
        print(authorize_url)
        code = raw_input("Input the code you got: ").strip()

        try:
            credential = flow.step2_exchange(code)
        except client.FlowExchangeError, e:
            sys.exit('Authentication has failed: %s' % e)

        storage.put(credential)
        credential.set_store(storage)

        dbg.info('Authentication successful')

        return credential
Example #32
0
    def _authorize(self, flow, storage, flags):
        # copied and edited from tools.run_flow()
        import getpass, time
        from selenium import webdriver
        from selenium.webdriver.common.keys import Keys
        from selenium.webdriver.support.ui import WebDriverWait
        from selenium.webdriver.support import expected_conditions as EC
        from selenium.webdriver.common.by import By

        dbg.info('Request access token from Google Drive')

        oauth_callback = client.OOB_CALLBACK_URN
        flow.redirect_uri = oauth_callback
        authorize_url = flow.step1_get_authorize_url()
        print(
            "We need to authorize access to Google. Please visit the following URL and authorize the access:"
        )
        print(authorize_url)
        code = raw_input("Input the code you got: ").strip()

        try:
            credential = flow.step2_exchange(code)
        except client.FlowExchangeError, e:
            sys.exit('Authentication has failed: %s' % e)
Example #33
0
def test_bench_paxos(metasync, opts):
    "bencmark latency of paxos with backends"

    def new_index(srv, folder, prefix):
        if services.slug(srv) == 'onedrive':
            folder = '/Public' + folder
        if not srv.exists(folder):
            return 0
        files = srv.listdir(folder)
        cnt = 0
        for fn in files:
            if fn.startswith(prefix):
                cnt += 1
        return cnt

    from paxos import PPaxosWorker

    repeat = 5
    client_num = [1, 2, 3, 4, 5]
    backend_list = [["google"], ["dropbox"], ["onedrive"], ["box"],
                    ["google", "dropbox", "onedrive"]]
    results = [['Clients'] + [','.join(x) for x in backend_list]]

    # start to test
    for num in client_num:
        for _ in range(repeat):
            row = ['%d clients' % (num)]
            for backend in backend_list:
                dbg.info('Test paxos for %d clients and %s' %
                         (num, ','.join(backend)))
                srvs = map(services.factory, backend)
                # init log file
                prefix = 'test-%d-%d' % (num, len(backend))
                index = new_index(srvs[0], '/ppaxos', prefix)
                path = '/ppaxos/%s.%d' % (prefix, index)
                dbg.info(path)
                for srv in srvs:
                    srv.init_log(path)

                clients = []
                for i in range(num):
                    storages = map(services.factory, backend)
                    worker = PPaxosWorker(storages, path)
                    clients.append(worker)
                for worker in clients:
                    worker.start()

                latency = []
                master_latency = None
                for worker in clients:
                    worker.join()
                    latency.append(worker.latency)
                    if (worker.master):
                        assert master_latency is None
                        master_latency = worker.latency
                for worker in clients:
                    worker.join()
                summary = ",".join(
                    map(str, [
                        min(latency),
                        max(latency),
                        util.median(latency), master_latency
                    ]))
                dbg.info("Result: %s" % summary)
                row.append(summary)
            results.append(row)

    # tabularize
    print "Item Format: min,max,median,master"
    for row in results:
        for e in row:
            print "%s \t" % e,
        print
Example #34
0
def test_bstore_reconfig(metasync, opts):
    "rebalancing all blobs when conf changes"

    test_init(metasync, opts)

    dbg.info("old config: %s" % metasync.mapping)
Example #35
0
if __name__ == '__main__':
    parser = optparse.OptionParser()
    parser.add_option("--promo", "-p",
                      help="list of promo log",
                      default="data/lock-promo.log")
    (opts, args) = parser.parse_args()

    utils.install_pdb()

    # pick the latest one from /tmp
    log_d = utils.get_latest_file("/tmp/fss-*/")
    if log_d is None:
        # second choice
        log_d = os.path.join(ROOT, "data", "sample-fss-output")
    # perfered one
    if len(args) > 1:
        log_d = args[1]

    # enable trace output on unittest
    if not "unit" in log_d:
        dbg.quiet(["trace"])

    promo = load_promo_log(opts.promo)
    dbg.info("# %d loaded, %s", len(promo), log_d)

    pool = mp.Pool(mp.cpu_count())
    for pn in Parser(log_d)._get_files():
        pool.apply_async(explore_file, args=(pn, promo))
    pool.close()
    pool.join()
Example #36
0
    def cmd_reconfigure(self, backends, replica):
        srvmap = {}
        for tok in backends.split(","):
            srv = services.factory(tok)
            srvmap[srv.sid()] = srv
        lst_services = srvmap.values()
        mapconfig = []
        lock_dic = {}
        for srv in lst_services:
            mapconfig.append((srv.sid(), srv.info_storage()/GB))
        for srv in srvmap:
            lock_dic[srv] = threading.Lock()
            if srv not in self.srvmap:
                srvmap[srv].putdir(self.get_remote_path("objects"))
        for srv in self.srvmap:
            if srv not in lock_dic: 
                lock_dic[srv] = threading.Lock()

        beg = time.time()
        self.mapping.reconfig(mapconfig, eval(replica))
        end = time.time()
        dbg.info("remap: %.3fs" % (end-beg))
        beg = time.time()
        lst_objs = self.blobstore.list()
        added, removed = self.mapping.get_remapping(lst_objs)
        nthreads = self.options.nthreads if self.options is not None else 2

        #REFACTOR 
        def __put_next(srv, lst, lock):
            dbg.job("submitted to: %s" % srv)
            while True:
                lock.acquire()
                if(len(lst) == 0):
                    lock.release()
                    break
                next = lst.pop()
                lock.release()
                if next is not None:
                    with open(self.get_local_obj_path(next), "rb") as f:
                        blob = f.read()
                        for tr in self.translators:
                            blob = tr.put(blob)
                        # XXX HACK
                        backoff = 0.5
                        remote_path = self.get_remote_obj_path(next)
                        while not srv.put(remote_path, blob):
                            time.sleep(backoff)
                            backoff *= 2

        def __rm_next(srv, lst, lock):
            dbg.job("submitted to: %s" % srv)
            while True:
                lock.acquire()
                if(len(lst) == 0):
                    lock.release()
                    break
                next = lst.pop()
                lock.release()
                if next is not None:
                    remote_path = self.get_remote_obj_path(next)
                    srv.rm(remote_path)
        cnt_added = 0 
        for srv in added:
            if(len(added[srv]) == 0): continue
            cnt_added += len(added[srv])
            for i in range(nthreads):
                self.scheduler.submit(srvmap[srv], False, __put_next, added[srv], lock_dic[srv])   
        self._join()

        end = time.time()
        dbg.info("remap put: %.3fs" % (end-beg))

        beg = time.time()
        cnt_removed = 0
        for srv in removed:
            if(len(removed[srv]) == 0): continue
            cnt_removed += len(removed[srv])
            for i in range(nthreads):
                self.scheduler.submit(self.srvmap[srv], False, __rm_next, removed[srv], lock_dic[srv])   
        self._join()
        end = time.time()
        dbg.info("remap rm: %.3fs" % (end-beg))
        dbg.info("added %d, removed %d" % (cnt_added, cnt_removed))
Example #37
0
def main():
    atexit.register(retroctl.disable)

    parser = optparse.OptionParser(usage="%prog [options] LOG-DIRECTORY")
    parser.add_option("-d",
                      "--dry",
                      default=False,
                      action="store_true",
                      dest="dryrun",
                      help="dry run for repair")
    parser.add_option("-a",
                      "--attk",
                      dest="attack",
                      metavar="NAME",
                      default="pick_attack_execve",
                      help="algorithm to pick attacker's node")
    parser.add_option("-p",
                      "--profile",
                      default=False,
                      action="store_true",
                      dest="profile",
                      help="profiling execution time")

    # hints
    parser.add_option("-i",
                      "--inode",
                      dest="inode_hint",
                      metavar="FILE",
                      default=None,
                      help="attacker's binary")

    (opts, args) = parser.parse_args()

    if opts.profile:
        runopts.set_profile()

    if len(args) != 1 or not opts.inode_hint:
        parser.print_usage()
        exit(1)

    # pick attack
    attack = attacker.find_attack_node(args[0], opts.attack, opts.inode_hint)

    osloader.set_logdir(args[0])
    attack_node = mgrapi.RegisteredObject.by_name_load(attack)
    print_tics_tacs()
    if attack_node is None:
        raise Exception('missing attack node', attack)

    chkpt = max(c for c in attack_node.checkpoints
                if c < min(attack_node.actions))
    assert chkpt

    assert len(attack_node.actions) == 1
    #code.interact(local=locals())
    #pdb.set_trace()

    # cancel exec syscall actor
    for a in attack_node.actions:
        dbg.info("cancel: %s" % a.argsnode)
        a.argsnode.data = None
        a.cancel = True

    dbg.info("pick:", chkpt)

    runopts.set_dryrun(opts.dryrun)

    ctrl.repair(attack_node, chkpt)
Example #38
0
           == "(S64 # $A3__struct_dentry_P->d_inode)")
    assert(filter_out_non_args("(E # S_ISDIR($A1__struct_dentry_P->d_inode->i_mode))")
           == "(E # S_ISDIR($A1__struct_dentry_P->d_inode->i_mode))")
    assert(filter_out_non_args("(S64 # new->inode)")
           == "(S64 # -)")
    assert(filter_out_non_args("$A1_XXX")=="$A1_XXX")
    assert(filter_out_non_args("xyz")=="-")
    assert(filter_out_non_args("ext2_set_attr($A1__xxx, $A2__yyy, local)")
           =="ext2_set_attr($A1__xxx, $A2__yyy, -)")
    assert(filter_out_non_args("ext2_set_attr($A1__xxx, malloc(x, y, z), xxx)")
           =="ext2_set_attr($A1__xxx, malloc(-, -, -), -)")
    assert(filter_out_non_args("(S64 # $A4__unsigned_int) & (I # 4294967294)") 
           == "(S64 # $A4__unsigned_int) & (I # 4294967294)")
    return

if __name__ == '__main__':
    utils.install_pdb()

    # get log dir
    log_d = os.path.join(ROOT, "data", "sample-fss-output")
    if len(sys.argv) > 1:
        log_d = sys.argv[1]

    dbg.info("> %s", log_d)
    for path in Parser(log_d).parse():
        print "<< Before normalization"
        print(path)
        npath = ArgNormalizer(path)
        print ">> After normalization"
        print(npath)
Example #39
0
def test_mapping2(metasync, opts):
    "test a new mapping scheme to tolerate rebalancing on node-failre"

    from itertools import permutations

    # goal
    # 1. non-uniformly locating blobs, approximately reflecting storage size of each node
    # 2. minimize realigning on a node failure

    # design
    # 0. node -> (node, storage)
    #    (e.g., (1,0), (1,1) if node 1 has 2G storage)
    # 1. fixed hspace, where h(blob) % hspace = index
    #    (hspace any large number, NOT depending on len(nodes))
    # 2. detmap[index] -> a group of nodes
    #    (a group includes all nodes, but different order see 3)
    # 3. order nodes in a group, by hash(index, node)
    #    (so deterministic)
    # 4. in each group, pick first #replication nodes

    # failure
    # node change
    # replication
    #   => in all of above situations, only blobs in old/new node
    #      will be re-balanced
    #

    def uniq(lst, n):
        rtn = []
        for (i, _) in lst:
            if i in rtn:
                continue
            rtn.append(i)
            if len(rtn) == n:
                break
        return rtn

    replication = 2
    config = [(0, 2), (1, 4), (2, 4), (3, 2)]
    hspace = 20
    detmap = DetMap2(config, hspace, replication)

    stats = [0] * len(config)
    for (k, v) in enumerate(detmap.mapinfo[0].detmap):
        """
        # doesn't it need to calculate over uniq?
        for i in range(replication):
            stats[v[i][0]] += 1
        """
        for i in uniq(v, replication):
            stats[i] += 1
        if opts.slow:
            dbg.info("%-2s: [" % k)
            for (n, priority) in v:
                dbg.info("  %s: %s" % (n, priority))
            dbg.info("]")
        else:
            dbg.info("%-2s: %s -> %s" \
                       % (k, [e[0] for e in v], detmap.get_mapping(k)))

    # approximately reflect storage?
    for (i, k) in enumerate(stats):
        dbg.info("%s (%s GB) -> #%s" % (i, config[i][1], k))
Example #40
0
def test_bench_disk_paxos(metasync, opts):
    "test disk paxos"
    "bencmark latency of paxos with backends"

    from disk_paxos import DiskPaxosWorker

    repeat = 5
    client_num = [1, 2, 3, 4, 5]
    backend_list = [["google"], ["dropbox"], ["onedrive"], ["box"],
                    ["google", "dropbox", "onedrive"]]
    results = [['Clients'] + [','.join(x) for x in backend_list]]

    # start to test
    for num in client_num:
        for num_prop in range(1, num + 1):
            for _ in range(repeat):
                row = ['%d/%d clients' % (num_prop, num)]
                for backend in backend_list:
                    srvs = map(services.factory, backend)
                    dbg.info('Test paxos for %d/%d clients and %s' %
                             (num_prop, num, ','.join(backend)))
                    # initialize all disk blocks
                    blockList = []
                    for i in range(num):
                        path = '/diskpaxos/client%d' % i
                        for srv in srvs:
                            if not srv.exists(path):
                                srv.put(path, '')
                            else:
                                srv.update(path, '')
                        blockList.append(path)

                    clients = []
                    for i in range(num_prop):
                        storages = map(services.factory, backend)
                        worker = DiskPaxosWorker(storages, blockList[i],
                                                 blockList)
                        clients.append(worker)
                        #dbg.dbg('client %d %s' % (i, worker.clientid))
                    for worker in clients:
                        worker.start()

                    latency = []
                    master_latency = None
                    for worker in clients:
                        worker.join()
                        latency.append(worker.latency)
                        if (worker.master):
                            assert master_latency is None
                            master_latency = worker.latency
                    for worker in clients:
                        worker.join()

                    summary = ",".join(
                        map(str, [
                            min(latency),
                            max(latency),
                            util.median(latency), master_latency
                        ]))
                    dbg.info("Result: %s" % summary)
                    row.append(summary)
                results.append(row)

    # tabularize
    print "Item Format: min,max,median,master"
    for row in results:
        for e in row:
            print "%s \t" % e,
        print
Example #41
0
def pick_attack_execve(r):
	"""choose execve containing 'attacker' string"""

	if r.name == "execve" and str(r).find("attacker") != -1:
		dbg.info("#R<attack#>:%s" % r)
		return PICK_ATTACK|PICK_STOP
Example #42
0
 def request_token():
   dbg.info('Request access token from Baidu')
   code = OAuth2._authorize()
   token = OAuth2._token_request('authorization_code', code=code)
   dbg.info('Authentication successful')
   return token
Example #43
0
    def cmd_reconfigure(self, backends, replica):
        srvmap = {}
        for tok in backends.split(","):
            srv = services.factory(tok)
            srvmap[srv.sid()] = srv
        lst_services = srvmap.values()
        mapconfig = []
        lock_dic = {}
        for srv in lst_services:
            mapconfig.append((srv.sid(), srv.info_storage() / GB))
        for srv in srvmap:
            lock_dic[srv] = threading.Lock()
            if srv not in self.srvmap:
                srvmap[srv].putdir(self.get_remote_path("objects"))
        for srv in self.srvmap:
            if srv not in lock_dic:
                lock_dic[srv] = threading.Lock()

        beg = time.time()
        self.mapping.reconfig(mapconfig, eval(replica))
        end = time.time()
        dbg.info("remap: %.3fs" % (end - beg))
        beg = time.time()
        lst_objs = self.blobstore.list()
        added, removed = self.mapping.get_remapping(lst_objs)
        nthreads = self.options.nthreads if self.options is not None else 2

        #REFACTOR
        def __put_next(srv, lst, lock):
            dbg.job("submitted to: %s" % srv)
            while True:
                lock.acquire()
                if (len(lst) == 0):
                    lock.release()
                    break
                next = lst.pop()
                lock.release()
                if next is not None:
                    with open(self.get_local_obj_path(next), "rb") as f:
                        blob = f.read()
                        for tr in self.translators:
                            blob = tr.put(blob)
                        # XXX HACK
                        backoff = 0.5
                        remote_path = self.get_remote_obj_path(next)
                        while not srv.put(remote_path, blob):
                            time.sleep(backoff)
                            backoff *= 2

        def __rm_next(srv, lst, lock):
            dbg.job("submitted to: %s" % srv)
            while True:
                lock.acquire()
                if (len(lst) == 0):
                    lock.release()
                    break
                next = lst.pop()
                lock.release()
                if next is not None:
                    remote_path = self.get_remote_obj_path(next)
                    srv.rm(remote_path)

        cnt_added = 0
        for srv in added:
            if (len(added[srv]) == 0): continue
            cnt_added += len(added[srv])
            for i in range(nthreads):
                self.scheduler.submit(srvmap[srv], False, __put_next,
                                      added[srv], lock_dic[srv])
        self._join()

        end = time.time()
        dbg.info("remap put: %.3fs" % (end - beg))

        beg = time.time()
        cnt_removed = 0
        for srv in removed:
            if (len(removed[srv]) == 0): continue
            cnt_removed += len(removed[srv])
            for i in range(nthreads):
                self.scheduler.submit(self.srvmap[srv], False, __rm_next,
                                      removed[srv], lock_dic[srv])
        self._join()
        end = time.time()
        dbg.info("remap rm: %.3fs" % (end - beg))
        dbg.info("added %d, removed %d" % (cnt_added, cnt_removed))
Example #44
0
def test_bench_disk_paxos(metasync, opts):
    "test disk paxos"
    "bencmark latency of paxos with backends"

    from disk_paxos import DiskPaxosWorker

    repeat = 5
    client_num = [1, 2, 3, 4, 5]
    backend_list = [["google"], ["dropbox"], ["onedrive"], ["box"], ["google", "dropbox", "onedrive"]]
    results = [['Clients'] + [','.join(x) for x in backend_list]]

    # start to test
    for num in client_num:
        for num_prop in range(1, num + 1):
            for _ in range(repeat):
                row = ['%d/%d clients' % (num_prop, num)]
                for backend in backend_list:
                    srvs = map(services.factory, backend)
                    dbg.info('Test paxos for %d/%d clients and %s' % (num_prop, num, ','.join(backend)))
                    # initialize all disk blocks
                    blockList = []
                    for i in range(num):
                        path = '/diskpaxos/client%d' % i
                        for srv in srvs:
                            if not srv.exists(path):
                                srv.put(path, '')
                            else:
                                srv.update(path, '')
                        blockList.append(path)

                    clients = []
                    for i in range(num_prop):
                        storages = map(services.factory, backend)
                        worker = DiskPaxosWorker(storages, blockList[i], blockList)
                        clients.append(worker)
                        #dbg.dbg('client %d %s' % (i, worker.clientid))
                    for worker in clients:
                        worker.start()

                    latency = [] 
                    master_latency = None
                    for worker in clients:
                        worker.join()
                        latency.append(worker.latency)
                        if (worker.master):
                            assert master_latency is None
                            master_latency = worker.latency
                    for worker in clients:
                        worker.join()
                    
                    summary = ",".join(map(str,[min(latency), max(latency), util.median(latency), master_latency]))
                    dbg.info("Result: %s" % summary)
                    row.append(summary)
                results.append(row)

    # tabularize
    print "Item Format: min,max,median,master"
    for row in results:
        for e in row:
            print "%s \t" % e,
        print
Example #45
0
def test_mapping2(metasync, opts):
    "test a new mapping scheme to tolerate rebalancing on node-failre"

    from itertools import permutations

    # goal
    # 1. non-uniformly locating blobs, approximately reflecting storage size of each node
    # 2. minimize realigning on a node failure

    # design
    # 0. node -> (node, storage)
    #    (e.g., (1,0), (1,1) if node 1 has 2G storage)
    # 1. fixed hspace, where h(blob) % hspace = index
    #    (hspace any large number, NOT depending on len(nodes))
    # 2. detmap[index] -> a group of nodes
    #    (a group includes all nodes, but different order see 3)
    # 3. order nodes in a group, by hash(index, node)
    #    (so deterministic)
    # 4. in each group, pick first #replication nodes

    # failure
    # node change
    # replication
    #   => in all of above situations, only blobs in old/new node
    #      will be re-balanced
    #

    def uniq(lst, n):
        rtn = []
        for (i, _) in lst:
            if i in rtn:
                continue
            rtn.append(i)
            if len(rtn) == n:
                break
        return rtn

    replication = 2
    config = [(0,2), (1,4), (2,4), (3,2)]
    hspace = 20
    detmap = DetMap2(config, hspace, replication)

    stats = [0] * len(config)
    for (k, v) in enumerate(detmap.mapinfo[0].detmap):
        """
        # doesn't it need to calculate over uniq?
        for i in range(replication):
            stats[v[i][0]] += 1
        """
        for i in uniq(v, replication):
            stats[i] += 1
        if opts.slow:
            dbg.info("%-2s: [" % k)
            for (n, priority) in v:
                dbg.info("  %s: %s" % (n, priority))
            dbg.info("]")
        else:
            dbg.info("%-2s: %s -> %s" \
                       % (k, [e[0] for e in v], detmap.get_mapping(k)))

    # approximately reflect storage?
    for (i, k) in enumerate(stats):
        dbg.info("%s (%s GB) -> #%s" % (i, config[i][1], k))
Example #46
0
        "(E # S_ISDIR($A1__struct_dentry_P->d_inode->i_mode))") ==
            "(E # S_ISDIR($A1__struct_dentry_P->d_inode->i_mode))")
    assert (filter_out_non_args("(S64 # new->inode)") == "(S64 # -)")
    assert (filter_out_non_args("$A1_XXX") == "$A1_XXX")
    assert (filter_out_non_args("xyz") == "-")
    assert (filter_out_non_args("ext2_set_attr($A1__xxx, $A2__yyy, local)") ==
            "ext2_set_attr($A1__xxx, $A2__yyy, -)")
    assert (
        filter_out_non_args("ext2_set_attr($A1__xxx, malloc(x, y, z), xxx)") ==
        "ext2_set_attr($A1__xxx, malloc(-, -, -), -)")
    assert (filter_out_non_args("(S64 # $A4__unsigned_int) & (I # 4294967294)")
            == "(S64 # $A4__unsigned_int) & (I # 4294967294)")
    return


if __name__ == '__main__':
    utils.install_pdb()

    # get log dir
    log_d = os.path.join(ROOT, "data", "sample-fss-output")
    if len(sys.argv) > 1:
        log_d = sys.argv[1]

    dbg.info("> %s", log_d)
    for path in Parser(log_d).parse():
        print "<< Before normalization"
        print(path)
        npath = ArgNormalizer(path)
        print ">> After normalization"
        print(npath)
Example #47
0
def test_bstore_reconfig(metasync, opts):
    "rebalancing all blobs when conf changes"

    test_init(metasync, opts)

    dbg.info("old config: %s" % metasync.mapping)
Example #48
0
def test_bench_paxos2(metasync, opts):
    "bencmark latency of paxos with backends"

    def new_index(srv, folder, prefix):
        if not srv.exists(folder):
            return 0
        files = srv.listdir(folder)
        cnt = 0
        for fn in files:
            if fn.startswith(prefix):
                cnt += 1
        return cnt

    from paxos import PPaxosWorker2

    repeat = 5
    client_num = [1, 2, 3, 4, 5]
    backend_list = [["dropbox"], ["onedrive"]]
    results = [['Clients'] + [','.join(x) for x in backend_list]]

    # start to test
    for num in client_num:
        for _ in range(repeat):
            row = ['%d clients' % (num)]
            for backend in backend_list:
                dbg.info('Test paxos for %d clients and %s' % (num, ','.join(backend)))
                srvs = map(services.factory, backend)
                # init log file
                prefix = 'test2-%d-%d' % (num , len(backend))
                index = new_index(srvs[0], '/ppaxos', prefix)
                path = '/ppaxos/%s.%d' % (prefix, index)
                dbg.info(path)
                for srv in srvs:
                    srv.init_log2(path)

                clients = []
                for i in range(num):
                    storages = map(services.factory, backend)
                    worker = PPaxosWorker2(storages, path)
                    clients.append(worker)
                for worker in clients:
                    worker.start()
                for worker in clients:
                    worker.join()

                latency = []
                master_latency = None
                for worker in clients:
                    latency.append(worker.latency)
                    if (worker.master):
                        assert master_latency is None
                        master_latency = worker.latency
                assert master_latency is not None
                
                summary = ",".join(map(str,[min(latency), max(latency), util.median(latency), master_latency]))
                dbg.info("Result: %s" % summary)
                row.append(summary)
            results.append(row)

    # tabularize
    print "Item Format: min,max,median,master"
    for row in results:
        for e in row:
            print "%s \t" % e,
        print