コード例 #1
0
    def collect(self):

        self.pathsToCollect.append("/etc/profile")
        self.pathsToCollect.append("/etc/bashrc")
        self.pathsToCollect.append("/etc/sudoers")

        for user in self.userList:

            p = "/Users/{0}/.bash_profile".format(user)
            if os.path.isfile(p):
                self.pathsToCollect.append(p)

            p = "/Users/{0}/.bash_login".format(user)
            if os.path.isfile(p):
                self.pathsToCollect.append(p)

            p = "/Users/{0}/.profile".format(user)
            if os.path.isfile(p):
                self.pathsToCollect.append(p)

            p = "/Users/{0}/.bash_logout".format(user)
            if os.path.isfile(p):
                self.pathsToCollect.append(p)

        Collector.collect(self)
コード例 #2
0
ファイル: installs.py プロジェクト: thomasareed/pict
    def collect(self):

        output = ""
        receiptPlistPaths = glob.glob("/private/var/db/receipts/*.plist")
        for onePlistPath in receiptPlistPaths:
            if os.path.isfile(onePlistPath) and onePlistPath.endswith(
                    ".plist"):
                plistDict = FoundationPlist.readPlist(onePlistPath)
                if not plistDict:
                    continue
                output += str(plistDict["InstallDate"]) + "\t"
                output += plistDict["InstallPrefixPath"] + "\t"
                output += plistDict["InstallProcessName"] + "\t"
                output += plistDict["PackageFileName"] + "\t"
                output += plistDict["PackageIdentifier"] + "\t"
                output += plistDict["PackageVersion"] + "\n"

        output = "Install Date\tPrefix Path\tProcess Name\tFile Name\tIdentifier\tVersion\n" + "\n".join(
            sorted(output.split("\n")))

        filename = "installs.txt"
        filePath = self.collectionPath + filename
        f = open(filePath, "w+")
        f.write(output)
        f.close

        self.pathsToCollect.append("/Library/Receipts/InstallHistory.plist")

        Collector.collect(self)
コード例 #3
0
def metateleport(defaults):
    collector = Collector(**defaults)
    env = Game(**defaults)
    mover = Mover(env, _extra_dim=1, **defaults)
    teleporter1 = Teleporter(env, _extra_dim=1, **defaults)
    teleporter2 = MetaTeleporter(env, **defaults)
    buffer1 = ReplayBuffer(**defaults)
    buffer2 = ReplayBuffer(**defaults)

    with Save(env, collector, mover, teleporter1, teleporter2, **defaults) as save:
        intervention_idx2, modified_board2 = teleporter2.pre_process(env)
        intervention_idx1, _ = teleporter1.pre_process(env)
        for frame in loop(env, collector, save, teleporter1, teleporter2):
            modified_board2 = teleporter2.interveen(env.board, intervention_idx2, modified_board2)
            modified_board1 = teleporter1.interveen(env.board, intervention_idx1, modified_board2)
            actions = mover(modified_board1)
            observations, rewards, dones, info = env.step(actions)
            modified_board1, modified_board2, modified_rewards1, modified_rewards2, modified_dones1, modified_dones2, tele_rewards, intervention_idx1, intervention_idx2 = teleporter2.metamodify(observations, rewards, dones, info, teleporter1.interventions)
            buffer1.teleporter_save_data(teleporter1.boards, modified_board2, teleporter1.interventions, modified_rewards2, modified_dones2, intervention_idx1)
            buffer2.teleporter_save_data(teleporter2.boards, observations, teleporter2.interventions, tele_rewards, dones, intervention_idx2)
            mover.learn(modified_board1, actions, modified_rewards1, modified_dones1)
            board_before, board_after, intervention, tel_rewards, tele_dones = buffer1.sample_data()
            teleporter1.learn(board_after, intervention, tel_rewards, tele_dones, board_before)
            board_before, board_after, intervention, tel_rewards, tele_dones = buffer2.sample_data()
            teleporter2.learn(board_after, intervention, tel_rewards, tele_dones, board_before)
            collector.collect([rewards, modified_rewards1, modified_rewards2, tele_rewards], [dones, modified_dones1, modified_dones2])
コード例 #4
0
def CFagent(defaults):
    env = Game(**defaults)
    mover = Mover(env, _extra_dim=1, **defaults)
    teleporter = Teleporter(env, **defaults)
    buffer = ReplayBuffer(**defaults)
    CFagent = CFAgent(env, **defaults)
    CFbuffer = CFReplayBuffer(**defaults)
    collector = Collector(**defaults)

    with Save(env, collector, mover, teleporter, CFagent, **defaults) as save:
        intervention_idx, modified_board = teleporter.pre_process(env)
        dones = CFagent.pre_process(env)
        CF_dones, cfs = None, None
        for frame in loop(env, collector, save, teleporter):
            CFagent.counterfact(env, dones, teleporter, CF_dones, cfs)
            modified_board = teleporter.interveen(env.board, intervention_idx, modified_board)
            actions = mover(modified_board)
            observations, rewards, dones, info = env.step(actions)
            modified_board, modified_rewards, modified_dones, teleport_rewards, intervention_idx = teleporter.modify(observations, rewards, dones, info)
            buffer.teleporter_save_data(teleporter.boards, observations, teleporter.interventions, teleport_rewards, dones, intervention_idx)
            mover.learn(modified_board, actions, modified_rewards, modified_dones)
            board_before, board_after, intervention, tele_rewards, tele_dones = buffer.sample_data()
            teleporter.learn(board_after, intervention, tele_rewards, tele_dones, board_before)
            collector.collect([rewards, modified_rewards, teleport_rewards], [dones, modified_dones])
            CF_dones, cfs = CFagent.counterfact_check(dones, env, **defaults)
            CFbuffer.CF_save_data(CFagent.boards, observations, CFagent.counterfactuals, rewards, dones, CF_dones)
            CFboard, CFobs, cf, CFrewards, CFdones1 = CFbuffer.sample_data()
            CFagent.learn(CFobs, cf, CFrewards, CFdones1, CFboard)
コード例 #5
0
    def __init__(self, agent: Agent, env: UnityEnvironment, config: Dict[str,
                                                                         Any]):
        super().__init__(agent, env, config)

        default_config = {
            "steps": 2048,

            # Tensorboard settings
            "tensorboard_name": None,  # str, set explicitly

            # PPO
            "ppo_config": {
                # GD settings
                "optimizer": "adam",
                "optimizer_kwargs": {
                    "lr": 1e-4,
                    "betas": (0.9, 0.999),
                    "eps": 1e-7,
                    "weight_decay": 0,
                    "amsgrad": False
                },
                "gamma": .99,  # Discount factor

                # PPO settings
                "ppo_steps":
                25,  # How many max. gradient updates in one iterations
                "eps": 0.1,  # PPO clip parameter
                "target_kl": 0.01,  # KL divergence limit
                "value_loss_coeff": 0.1,
                "entropy_coeff": 0.1,
                "max_grad_norm": 0.5,

                # Backpropagation settings
                "use_gpu": False,
            }
        }

        self.config = with_default_config(config, default_config)

        self.collector = Collector(agent=self.agent, env=self.env)
        self.ppo = PPOptimizer(agent=agent, config=self.config["ppo_config"])

        # Setup tensorboard
        self.writer: SummaryWriter
        if self.config["tensorboard_name"]:
            dt_string = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
            self.path = Path.home(
            ) / "drlnd_logs" / f"{self.config['tensorboard_name']}_{dt_string}"
            self.writer = SummaryWriter(str(self.path))

            # Log the configs
            with open(str(self.path / "trainer_config.json"), "w") as f:
                json.dump(self.config, f)

            with open(str(self.path / f"agent_config.json"), "w") as f:
                json.dump(self.agent.model.config, f)

            self.path = str(self.path)
        else:
            self.writer = None
コード例 #6
0
def make_app():
    """Instantiates Flask app, attaches collector database, installs acl."""
    LOG.info('Starting API')
    app = flask.Flask(__name__)
    app.register_blueprint(v1.blueprint, url_prefix='/v1')

    collector = Collector()
    collector.clean()

    thread.start_new_thread(listen, (collector.add, ))

    @app.before_request
    def attach_config():
        flask.request.collector = collector
        collector.lock.acquire()

    @app.after_request
    def unlock(response):
        collector.lock.release()
        return response

    # Install the middleware wrapper
    if cfg.CONF.acl_enabled:
        acl.install(app, cfg.CONF)

    return app
コード例 #7
0
ファイル: browserhist.py プロジェクト: thomasareed/pict
    def collect(self):
        filename = "history_downloads.txt"
        filePath = self.collectionPath + filename
        f = open(filePath, "w+")
        f.write(
            "Downloads (QuarantineEventsV2)\n-----------------------------\n")
        output = self.collectDownloads()
        f.write(output)
        f.close

        filename = "history_safari.txt"
        filePath = self.collectionPath + filename
        f = open(filePath, "w+")
        f.write("Safari history\n-----------------------------\n")
        output = self.collectSafari()
        f.write(output)
        f.close

        filename = "history_chrome.txt"
        filePath = self.collectionPath + filename
        f = open(filePath, "w+")
        f.write("Chrome history\n-----------------------------\n")
        output = self.collectChrome()
        f.write(output)
        f.close

        filename = "history_firefox.txt"
        filePath = self.collectionPath + filename
        f = open(filePath, "w+")
        f.write("Firefox history\n-----------------------------\n")
        output = self.collectFirefox()
        f.write(output)
        f.close

        Collector.collect(self)
コード例 #8
0
class RankingCrawler():
    # flow capacity (MB)
    def __init__(self, cookie, capacity=1024):
        self.date = START_DATE
        self.domain = DOMAIN
        self.mode = PIXIV_MODES[PIXIV_MODE]
        # url sample: https://www.pixiv.net/ranking.php?mode=daily&date=20200801&p=1&format=json
        # ref url sample: https://www.pixiv.net/ranking.php?mode=daily&date=20200801
        self.url = 'https://www.pixiv.net/ranking.php?mode=' + self.mode
        self.cookie = cookie
        self.headers = {'x-requested-with': 'XMLHttpRequest'}
        self.collector = Collector(cookie, capacity)

    def __nextday(self):
        self.date += datetime.timedelta(days=1)

    # collect illust_id from daily json
    def collect(self):
        # note that 50 artworks per p=x
        page_num = (ARTWORKS_PER - 1) // 50 + 1  #ceil
        print("---start collecting " + self.mode + " ranking---")
        print("start with " + self.date.strftime("%Y-%m-%d"))
        print("end with " + (self.date + datetime.timedelta(
            days=self.domain - 1)).strftime("%Y-%m-%d" + '\n'))
        # store all jsons' url in self.group
        self.group = set()
        for _i in range(DOMAIN):
            for j in range(page_num):
                self.group.add(self.url + '&date=' +
                               self.date.strftime("%Y%m%d") + '&p=' +
                               str(j + 1) + '&format=json')
            self.__nextday()
        pool = ParallelPool(len(self.group))
        while len(self.group) or not pool.empty():
            time.sleep(THREAD_DELAY)
            # send ranking_json to parallel pool
            while not pool.full() and len(self.group):
                url = self.group.pop()
                ref = re.search('(.*)&p', url).group(1)
                headers = self.headers.update({'Referer': ref})
                pool.add(
                    CollectorUnit(url, self.cookie, ranking_selector, headers))
            # remove complete thread
            finished = pool.finished_item()
            while True:
                try:
                    ranking_json = next(finished)
                    self.collector.add(ranking_json.group)
                    if MOST_OUTPUT:
                        print("--send page " + ranking_json.url +
                              " to collector--")
                except StopIteration:
                    break

        print("\n---collect " + self.mode + " ranking complete---")

    def run(self):
        self.collect()
        self.collector.collect()
        return self.collector.download()
コード例 #9
0
 def __init__(self, cookie, maxnum=200, capacity=1024):
     self.num = maxnum
     self.cookie = cookie
     self.url = "https://www.pixiv.net/ajax/user/" + USER_ID + "/illusts"
     self.headers = BROWSER_HEADER
     self.collect_cnt = 0
     self.collector = Collector(cookie, capacity)
コード例 #10
0
ファイル: acq.py プロジェクト: nkrishnaswami/mids-w205-hw2
def main():
    creds = Credentials(os.path.expanduser('~/.tweepy'))
    today = datetime.date.today()
    week = datetime.timedelta(7)
    query_terms=['#NBAFinals2015', '#Warriors'] 
    query_ops={}
    query_ops['lang'] = 'en'
    stream = False;
    if stream:
        auth = tweepy.OAuthHandler(creds.consumer_key, creds.consumer_secret)
        auth.set_access_token(creds.access_token, creds.access_token_secret)
        query_ops['until'] = today+week
    else:
        #Per Ron Cordell's suggestion, use App Auth token for increased limits
        auth = tweepy.AppAuthHandler(creds.consumer_key, creds.consumer_secret)
        query_ops['since'] = today-week
        query_ops['until'] = today
        query_ops['result_type'] = 'recent'
    collector = Collector(auth)
    if stream:
        print("Starting streaming")
        collector.stream(query_terms=query_terms,
                  query_ops=query_ops)
    else:
        print("Starting search")
        collector.search(query_terms=query_terms,
                  query_ops=query_ops)
    print('All done!  Last ID processed={0}'.format(collector.last_id))
コード例 #11
0
ファイル: main.py プロジェクト: zzzz123321/autocomplete
class MainResource(object):
    def __init__(self):
        self._logger = logging.getLogger('gunicorn.error')
        self._collector = Collector()

    def on_post(self, req, resp):
        self._logger.debug(
            f'Handling request {req.url} with params {req.params}')

        try:
            self._collector.collect_phrase(req.params['phrase'])
            response_body = json.dumps({
                "status": "success",
                "message": "Phrase sent for collection"
            })
            resp.status = falcon.HTTP_200
            resp.body = response_body

        except Exception as e:
            self._logger.error('An error occurred when processing the request',
                               exc_info=e)
            response_body = json.dumps({
                "status":
                "error",
                "message":
                "An error occurred when processing the request"
            })
            resp.status = falcon.HTTP_500
            resp.body = response_body
コード例 #12
0
    def test_enhance_sibling_symbols(self):
        c = Collector()
        aeabi_drsub = {
            collector.ADDRESS: "0000009c",
            collector.SIZE: 8,
            collector.TYPE: collector.TYPE_FUNCTION,
        }
        aeabi_dsub = {
            collector.ADDRESS: "000000a4",
            collector.SIZE: 4,
            collector.TYPE: collector.TYPE_FUNCTION,
        }
        adddf3 = {
            collector.ADDRESS: "000000a8",
            collector.SIZE: 123,
            collector.TYPE: collector.TYPE_FUNCTION,
        }

        c.symbols = {int(f[collector.ADDRESS], 16): f for f in [aeabi_drsub, aeabi_dsub, adddf3]}
        c.enhance_sibling_symbols()

        self.assertFalse(aeabi_drsub.has_key(collector.PREV_FUNCTION))
        self.assertEqual(aeabi_dsub, aeabi_drsub.get(collector.NEXT_FUNCTION))

        self.assertEqual(aeabi_drsub, aeabi_dsub.get(collector.PREV_FUNCTION))
        self.assertEqual(adddf3, aeabi_dsub.get(collector.NEXT_FUNCTION))

        self.assertEqual(aeabi_dsub, adddf3.get(collector.PREV_FUNCTION))
        self.assertFalse(adddf3.has_key(collector.NEXT_FUNCTION))
コード例 #13
0
    def run(self):
        print("==== Checking hosts on ====")
        # Here we can block by online workers (only run if haver 4 workers),
        # but we can work with less then 4 workers, and in my test, I have only 1.
        num_hosts = self._get_hosts_on(1)
        print("Hosts ON: %d" % num_hosts)

        print("==== Creating random matrices ====")
        self.matrix_a = self._get_random_matrix(self.a_n, self.a_m)
        self.matrix_b = self._get_random_matrix(self.b_n, self.b_m)

        print("==== Created Matrices: ====")
        self._print_two_matrices(self.matrix_a, self.matrix_b)

        self.matrix_divide()
        distributor = Distributor("*", "50010")
        collector = Collector("*", "50012", 5000)
        distributor.send_jobs(self._create_jobs())

        # For test, check services in rasp's
        self._check_services(1)

        results = collector.collect(4)

        if 'err' in results:
            print("Error in some RasPI: %s" % results['err'])
            exit()

        print("==== Appending matrices ====")
        C1 = self._matrix_sum(results['A1B1'], results['A2B2'])
        C2 = self._matrix_sum(results['A3B1'], results['A4B2'])
        C = C1 + C2

        print("==== Final result: ====")
        self._print_matrix(C)
コード例 #14
0
    def test_parses_assembly_and_stops_after_function(self):
        assembly = """
000034fc <window_raw_click_subscribe>:
$t():
    34fc:	b40f      	push	{r0, r1, r2, r3}
    34fe:	4901      	ldr	r1, [pc, #4]	; (3504 <window_raw_click_subscribe+0x8>)
    3500:	f7fc bdc2 	b.w	88 <jump_to_pbl_function>
$d():
    3504:	000004c4 	.word	0x000004c4
    3508:	00040000 	.word	0x00040000
    350c:	000b008d 	.word	0x000b008d

00003510 <.LC1>:
.LC1():
    3510:	69727073 	.word	0x69727073
    3514:	42736574 	.word	0x42736574
    3518:	31647269 	.word	0x31647269
    351c:	0036      	.short	0x0036

"""

        c = Collector()
        self.assertEqual(2, c.parse_assembly_text(assembly))
        self.assertTrue(c.symbols.has_key(0x000034fc))
        self.assertEqual(c.symbols[0x000034fc]["name"], "window_raw_click_subscribe")
        # print "\n".join(c.symbols["000034fc"]["asm"])
        self.assertEqual(len(c.symbols[0x000034fc]["asm"]), 8)
コード例 #15
0
ファイル: app.py プロジェクト: openstack/kwapi
def make_app():
    """Instantiates Flask app, attaches collector database, installs acl."""
    LOG.info('Starting API')
    app = flask.Flask(__name__)
    app.register_blueprint(v1.blueprint, url_prefix='/v1')

    collector = Collector()
    collector.clean()

    thread.start_new_thread(listen, (collector.add,))

    @app.before_request
    def attach_config():
        flask.request.collector = collector
        collector.lock.acquire()

    @app.after_request
    def unlock(response):
        collector.lock.release()
        return response

    # Install the middleware wrapper
    if cfg.CONF.acl_enabled:
        acl.install(app, cfg.CONF)

    return app
コード例 #16
0
ファイル: processes.py プロジェクト: thomasareed/pict
    def collect(self):
        filename = "processes.txt"
        filePath = self.collectionPath + filename
        f = open(filePath, "w+")
        f.write("Processes\n-----------------------------\n")
        processData = os.popen(
            "ps axo user,pid,ppid,start,time,command").read().rstrip()
        f.write(processData + "\n\n")
        f.close

        filename = "processes_files.txt"
        filePath = self.collectionPath + filename
        f = open(filePath, "w+")
        f.write("Files open by process\n-----------------------------\n")
        output = os.popen("lsof").read().rstrip()
        f.write(output + "\n\n")
        f.close

        filename = "processes_network.txt"
        filePath = self.collectionPath + filename
        f = open(filePath, "w+")
        f.write(
            "Network connections open by process\n-----------------------------\n"
        )
        output = os.popen("lsof -i").read().rstrip()
        f.write(output + "\n\n")
        f.close

        # No file paths to collect

        Collector.collect(self)
コード例 #17
0
ファイル: certs.py プロジェクト: thomasareed/pict
    def collect(self):
        filename = "trusted_certificates.txt"
        filePath = self.collectionPath + filename
        f = open(filePath, "w+")

        f.write("Certificate trust settings\n-----------------------------\n")
        for user in self.userList:
            f.write("For user {0}:\n\n".format(user))
            output = os.popen(
                "sudo -u {0} security dump-trust-settings 2>&1".format(
                    user)).read().rstrip()
            f.write(output + "\n\n")

        f.write("Admin certificate info\n-----------------------------\n")
        output = os.popen("security dump-trust-settings -d").read().rstrip()
        f.write(output + "\n\n")

        f.write("System certificate info\n-----------------------------\n")
        output = os.popen("security dump-trust-settings -s").read().rstrip()
        f.write(output + "\n\n")

        f.write("All certificates\n-----------------------------\n")
        output = self.collectAllCerts()
        f.write(output)

        f.close

        # Add any paths to self.pathsToCollect

        Collector.collect(self)
コード例 #18
0
ファイル: network_config.py プロジェクト: thomasareed/pict
    def collect(self):
        filename = "network_config.txt"
        filePath = self.collectionPath + filename
        f = open(filePath, "w+")

        f.write("en0\n-----------------------------\n")
        output = os.popen("ifconfig en0").read().rstrip()
        f.write(output + "\n\n")

        f.write("en1\n-----------------------------\n")
        output = os.popen("ifconfig en1").read().rstrip()
        f.write(output + "\n\n")

        # Output will already have a "DNS configuration" heading, no need to add one
        output = os.popen("scutil --dns 2>&1").read().rstrip()
        f.write(output + "\n\n")

        f.write("Proxies\n-----------------------------\n")
        output = os.popen("scutil --proxy 2>&1").read().rstrip()
        f.write(output + "\n\n")

        f.write("pf rules\n-----------------------------\n")
        output = os.popen("sudo pfctl -s rules 2>&1").read().rstrip()
        f.write(output + "\n\n")

        f.close

        self.pathsToCollect.append("/etc/hosts")

        Collector.collect(self)
コード例 #19
0
 def __init__(self, artist_id, cookie, capacity=1024):
     self.url = 'https://www.pixiv.net/ajax/user/' + artist_id + '/profile/all?lang=zh'
     self.ref = 'https://www.pixiv.net/users/' + artist_id + '/illustrations'
     self.headers = {'x-user-id': USER_ID}
     self.headers.update({'Referer': self.ref})
     self.cookie = cookie
     self.collector = Collector(cookie, capacity)
コード例 #20
0
def main():
    '''
    Function which starts the tool.
    '''
    # setup logging
    setupLogging()

    # get start and end dates for collecting if given from config file
    startDate, endDate = _getStartAndEnd()
    # get the list of datanodes we will be collecting measurements from
    iotTicket.getDataNodes()
    # create FIWARE entities to Orion from the buses we collect data from if not already created
    fiware.createEntities()
    if not fiware.sendToQl:
        # we will send measurements to QuantumLeap through Orion subscription(s) so create them if not already created
        fiware.addSubscription()

    # create the collector that takes care of the actual collection process
    myCollector = Collector(startDate, endDate)
    try:
        # and start collecting
        myCollector.startCollecting()

    except KeyboardInterrupt:
        log.info('Got keyboard interrupt. Collecting stopped.')

    except:
        log.exception('Unexpected exception occurred.')
        exit()

    log.info('Data collection done.')
コード例 #21
0
    def test_enhances_caller(self):
        assembly = """
00000098 <pbl_table_addr>:
        8e4:	f000 f824 	bl	930 <app_log>

00000930 <app_log>:
$t():
        """
        c = Collector()
        self.assertEqual(2, c.parse_assembly_text(assembly))
        self.assertTrue(c.symbols.has_key(0x00000098))
        self.assertTrue(c.symbols.has_key(0x00000930))

        pbl_table_addr = c.symbols[0x00000098]
        app_log = c.symbols[0x00000930]

        self.assertFalse(pbl_table_addr.has_key("callers"))
        self.assertFalse(pbl_table_addr.has_key("callees"))
        self.assertFalse(app_log.has_key("callers"))
        self.assertFalse(app_log.has_key("callees"))

        c.enhance_call_tree()

        self.assertEqual(pbl_table_addr["callers"], [])
        self.assertEqual(pbl_table_addr["callees"], [app_log])
        self.assertEqual(app_log["callers"], [pbl_table_addr])
        self.assertEqual(app_log["callees"], [])
コード例 #22
0
def main():
    query_terms=['#NBAFinals2015', '#Warriors']
    # We only need a single mongodb sink; FilteringFacet will just let us
    # avoid storing non-matching tweets
    sink = MongoDBSink('db_restT')
    sink.open('tweets')
    # closing ensures any files written get flushed/closed.
    with closing(
            FilteringFacet(
                RegexMatcher('(' + '|'.join(query_terms) + ')'),
                lambda key: sink)) as facet:
        creds = Credentials(os.path.expanduser('~/.tweepy'))
        auth = tweepy.AppAuthHandler(creds.consumer_key, creds.consumer_secret)
        collector = Collector(auth, facet)
        
        today = datetime.date.today()
        week = datetime.timedelta(7)

        query_ops={}
        query_ops['lang'] = 'en'
        query_ops['result_type'] = 'mixed'
        
        print("Starting search")
        collector.search(query_terms=query_terms,
                         query_ops=query_ops)
        print('All done!  Last ID processed={0}'.format(collector.last_id))
コード例 #23
0
    def __init__(self,
                 classifier,
                 mic_params,
                 is_audio_record=False,
                 root_path='./'):

        # arguments
        self.classifier = classifier
        self.mic_params = mic_params
        self.is_audio_record = is_audio_record
        self.root_path = root_path

        # plot path
        self.plot_path = self.root_path + self.mic_params['plot_path']

        # create folder for plot path
        create_folder([self.plot_path])

        # shortcuts
        self.feature_params = classifier.feature_params

        # feature extractor
        self.feature_extractor = FeatureExtractor(self.feature_params)

        # windowing params
        self.N, self.hop = self.feature_extractor.N, self.feature_extractor.hop

        # queue
        self.q = queue.Queue()

        # collector
        self.collector = Collector(
            N=self.N,
            hop=self.hop,
            frame_size=self.feature_params['frame_size'],
            update_size=self.mic_params['update_size'],
            frames_post=self.mic_params['frames_post'],
            is_audio_record=self.is_audio_record)

        # device
        self.device = sd.default.device[0] if not self.mic_params[
            'select_device'] else self.mic_params['device']

        # determine downsample
        self.downsample = self.mic_params['fs_device'] // self.feature_params[
            'fs']

        # get input devices
        self.input_dev_dict = self.extract_devices()

        # show devices
        print("\ndevice list: \n", sd.query_devices())
        print("\ninput devs: ", self.input_dev_dict.keys())

        # stream
        self.stream = None

        # change device flag
        self.change_device_flag = False
コード例 #24
0
 def test_stack_usage_line2(self):
     line = "puncover.c:8:43:dynamic_stack2	16	dynamic"
     c = Collector()
     c.symbols = {"123": {
         "base_file": "puncover.c",
         "line": 8,
     }}
     self.assertTrue(c.parse_stack_usage_line(line))
コード例 #25
0
    def collect(self):

        for user in self.userList:
            bashHistPath = "/Users/{0}/.bash_history".format(user)
            if os.path.isfile(bashHistPath):
                self.pathsToCollect.append(bashHistPath)

        Collector.collect(self)
コード例 #26
0
 def test_subfix(self):
     col = Collector()
     print(col.suffix("abc.TXT"))
     print(col.suffix("./test_suite.py"))
     print(
         col.suffix(
             r"C:\Users\Qun\PycharmProjects\PhotoCollector\PhotoCollecter\test\test_suite.py"
         ))
コード例 #27
0
    def __init__(self, width, height):

        game_mouse.Game.__init__(self, "Flappy Bird", width, height, 50)

        self.font_height = 12
        self.font = pygame.font.SysFont("arial", self.font_height)
        self.mCollector = Collector(width, height)
        return
コード例 #28
0
ファイル: logs.py プロジェクト: thomasareed/pict
    def applySettings(self, settingsDict):
        if "unifiedLogArguments" in settingsDict:
            self.unifiedLogArguments = settingsDict["unifiedLogArguments"]

        if "collectAuditLogs" in settingsDict:
            self.collectAuditLogs = settingsDict["collectAuditLogs"]

        Collector.applySettings(self, settingsDict)
コード例 #29
0
ファイル: server.py プロジェクト: wang-zifu/phish-collect
def collect_sample():
    c = Collector()
    while True:
        sample = sample_queue.get()
        logging.info('Processing sample {}'.format(sample.url))
        try:
            c.collect(sample)
        except Exception as e:
            logging.info('Error processing sample: ')
コード例 #30
0
 def __init__(self, file_name):
     self.map = []
     self.hero = None  # waiting for the first call of spawn(hero)
     self.hero_x = -1
     self.hero_y = -1
     self.collector = Collector(
     )  # collect spawn cells, enemies and treasures
     self.level = levels.index(file_name) + 1
     self._read_file(file_name)
コード例 #31
0
def main():
    conf = parse_options()

    c = Client(conf)
    collector = Collector(c)
    start = c.get_user(conf.start_user)
    collector.collect(start)

    g = collector.graph
    nx.write_dot(g, conf.output_file)
コード例 #32
0
 def test1(self):
     coll = Collector()
     for x in self.data:
         coll.add(x)
     self.assertAlmostEqual(self.stdev, coll.standard_deviation())
     self.assertAlmostEqual(self.var, coll.variance(), 5)
     self.assertAlmostEqual(self.avg, coll.average(), 5)
     self.assertAlmostEqual(self.sumsq, coll.sum_squares(), 5)
     self.assertAlmostEqual(self.sum, coll.sum(), 5)
     self.assertEqual(self.count, coll.count())
コード例 #33
0
ファイル: collector_tests.py プロジェクト: kevincon/puncover
 def test_stack_usage_line(self):
     line = "puncover.c:14:40:0	16	dynamic,bounded"
     c = Collector()
     c.symbols = {"123": {
         "base_file": "puncover.c",
         "line": 14,
     }}
     self.assertTrue(c.parse_stack_usage_line(line))
     self.assertEqual(16, c.symbols["123"]["stack_size"])
     self.assertEqual("dynamic,bounded", c.symbols["123"]["stack_qualifiers"])
コード例 #34
0
 def __init__(self, config, update=False):
     self.df = pd.DataFrame()
     if update:
         cll = Collector(config)
         cll.download_data()
     self.config = config
     self.symbol = self.config['symbol']
     self.timeframe = self.config['timeframe']
     self.all_candles = []
     self.pull_data()
コード例 #35
0
def collecting(args):
    logger = get_logger()
    db = create_db_connection(args.db_connection)

    collector = Collector(args.node, db)
    if args.rebuild:
        collector.clear()
    collector.stat()
    collector.play(args.limit)
    collector.stat()
コード例 #36
0
ファイル: collector_tests.py プロジェクト: kevincon/puncover
    def test_derive_filename_from_assembly(self):
        c = Collector()
        c.parse_assembly_text("""
000008a8 <uses_doubles2.constprop.0>:
uses_doubles2():
/Users/behrens/Documents/projects/pebble/puncover/pebble/build/../src/puncover.c:19
 8a8:	b508      	push	{r3, lr}
         """)
        s = c.symbol_by_addr("8a8")
        self.assertEqual("/Users/behrens/Documents/projects/pebble/puncover/pebble/build/../src/puncover.c", s[collector.PATH])
        self.assertEqual("puncover.c", s[collector.BASE_FILE])
        self.assertEqual(19, s[collector.LINE])
コード例 #37
0
ファイル: collector_tests.py プロジェクト: kevincon/puncover
    def test_parses_assembly_and_ignores_c(self):
        assembly = """
00000098 <pbl_table_addr>:
/path/to.c:8
pbl_table_addr():
  98:	a8a8a8a8 	.word	0xa8a8a8a8
"""
        c = Collector()
        self.assertEqual(1, c.parse_assembly_text(assembly))
        self.assertTrue(c.symbols.has_key(0x00000098))
        self.assertEqual(c.symbols[0x00000098]["name"], "pbl_table_addr")
        self.assertEqual(len(c.symbols[0x00000098]["asm"]), 2)
        self.assertEqual(c.symbols[0x00000098]["asm"][0], "pbl_table_addr():")
コード例 #38
0
ファイル: collector_tests.py プロジェクト: kevincon/puncover
 def test_derive_file_elements_for_unknown_files(self):
     c = Collector()
     s = c.add_symbol("some_symbol", "00a")
     self.assertEqual("some_symbol", s[collector.NAME])
     self.assertNotIn(collector.PATH, s)
     self.assertNotIn(collector.BASE_FILE, s)
     c.derive_folders()
     self.assertEqual("<unknown>/<unknown>", s[collector.PATH])
     self.assertEqual("<unknown>", s[collector.BASE_FILE])
     self.assertIn(collector.FILE, s)
     file = s[collector.FILE]
     self.assertEqual("<unknown>", file[collector.NAME])
     folder = file[collector.FOLDER]
     self.assertEqual("<unknown>", file[collector.NAME])
コード例 #39
0
ファイル: daemonize.py プロジェクト: neumerance/deploy
 def run(self):
     while True:
          """ collection management module """
          logger.info("CollectorManager has successfuly initiated")
          collector = Collector()
          collector.create_tenants_bills()
          """ subscription management module """
          logger.info("SubscriptionManager has successfuly initiated")
          subs = SubscriptionManager()
          subs.manage_subscription()
          logger.info("SchedulingManager has successfuly initiated")
          sched = SchedulingManager()
          sched.commence()
          time.sleep(60)
コード例 #40
0
ファイル: collector_tests.py プロジェクト: kevincon/puncover
    def test_enhances_assembly(self):
        assembly = """
00000098 <pbl_table_addr>:
pbl_table_addr():
 568:	f7ff ffca 	bl	98
"""
        c = Collector()
        self.assertEqual(1, c.parse_assembly_text(assembly))
        self.assertTrue(c.symbols.has_key(0x00000098))
        self.assertEqual(c.symbols[0x00000098]["name"], "pbl_table_addr")
        self.assertEqual(c.symbols[0x00000098]["asm"][1], " 568:\tf7ff ffca \tbl\t98")

        c.enhance_assembly()
        self.assertEqual(c.symbols[0x00000098]["asm"][1], " 568:\tf7ff ffca \tbl\t98 <pbl_table_addr>")
コード例 #41
0
ファイル: collector_tests.py プロジェクト: kevincon/puncover
    def test_enhance_function_size_from_assembly(self):
        c = Collector()
        c.symbols = { int("0000009c", 16) : {
            collector.ADDRESS: "0000009c",
            collector.ASM: """
$t():
  9c:	f081 4100 	eor.w	r1, r1, #2147483648	; 0x80000000
  a0:	e002      	b.n	a8 <__adddf3>
  a2:	bf00      	nop
            """.split("\n")
        }}

        s = c.symbol_by_addr("9c")
        self.assertFalse(s.has_key(collector.SIZE))
        c.enhance_function_size_from_assembly()
        self.assertEqual(8, s[collector.SIZE])
コード例 #42
0
	def __init__(self, collection_name, query_name, queue_name, table_name):
		self.collection_name = collection_name
		self.redis_conn = get_rc()
		self.table_name = table_name
		self.queue = Queue('collections',connection = self.redis_conn)
		self.query_name = query_name
		self.collector = Collector(query_name, collection_name, queue_name)
コード例 #43
0
ファイル: collector_tests.py プロジェクト: kevincon/puncover
    def test_parses_assembly2(self):
        assembly = """
00000098 <pbl_table_addr.constprop.0>:
pbl_table_addr():
  98:	a8a8a8a8 	.word	0xa8a8a8a8

0000009c <__aeabi_dmul>:
__aeabi_dmul():
  9c:	b570      	push	{r4, r5, r6, lr}
"""
        c = Collector()
        self.assertEqual(2, c.parse_assembly_text(assembly))
        self.assertTrue(c.symbols.has_key(0x0000009c))
        self.assertEqual(c.symbols[0x0000009c]["name"], "__aeabi_dmul")
        self.assertTrue(c.symbols.has_key(0x00000098))
        self.assertEqual(c.symbols[0x00000098]["name"], "pbl_table_addr")
コード例 #44
0
ファイル: db_initializer.py プロジェクト: simpleL/FDM
    def __init__(self):
        conf_path = "%s/FDM/FDM.conf"%(os.getcwd())
        self.__conf = ConfigParser.ConfigParser()
        self.__conf.read(conf_path)

        self.__conn = None
        
        self.collector = Collector()
コード例 #45
0
    def __init__(self, config):
        Collector.__init__(self, config.datapool)
        self.config = config

        # log初始化
        loggingInfo = config.loggerInfo
        logDir, logFile = os.path.split(loggingInfo['filename'])
        if not os.path.exists(logDir):
            os.makedirs(logDir)
        logging.basicConfig(
            level=loggingInfo['level'],
            format=loggingInfo['format'],
            filename=loggingInfo['filename'],
            encoding=loggingInfo['encoding']
        )

        self.collected = 0
        logging.info('collector "%s" starting...' % self.name)
コード例 #46
0
	def run_collector(self):
		'''instigates a collector, adds the specified jobs to it, 
			then returns the job uuids & queue name'''
		details = request.json or {}
		_input = details.pop('Input')
		input_type = details.pop('InputType')
		collector = Collector(**details)
		jobs = collector.schedule_jobs(_input, input_type)
		res = {
			'data':{
				'jobs':jobs,
				'log':collector.log_data,
			},
			'message':'collector initiated successfully',
		}
		return Response(
			response = dumps(res),
			status = 200,
			mimetype = 'application/json'
		)
コード例 #47
0
class Collection:
	def __init__(self, collection_name, query_name, queue_name, table_name):
		self.collection_name = collection_name
		self.redis_conn = get_rc()
		self.table_name = table_name
		self.queue = Queue('collections',connection = self.redis_conn)
		self.query_name = query_name
		self.collector = Collector(query_name, collection_name, queue_name)

	def schedule(self, job_parameters, input_type):
		self.collector.schedule_jobs(job_parameters, input_type)
		finishings = map(self.add_finisher, self.collector.jobs)
		return finishings

	def add_finisher(self, job):
		return self.queue.enqueue(
			consume_data,
			kwargs = {'job_id':job.id,'table_name':self.table_name},
			depends_on = job
		)
コード例 #48
0
ファイル: collector_tests.py プロジェクト: kevincon/puncover
    def test_derive_file_elements(self):
        c = Collector()
        s1 = {collector.PATH: "/Users/behrens/Documents/projects/pebble/puncover/pebble/build/../src/puncover.c"}
        s2 = {collector.PATH: "/Users/thomas/work/arm-eabi-toolchain/build/gcc-final/arm-none-eabi/thumb2/libgcc/../../../../../gcc-4.7-2012.09/libgcc/config/arm/ieee754-df.S"}
        s3 = {collector.PATH: "src/puncover.c"}
        c.symbols = {
            1: s1,
            2: s2,
            3: s3,
        }

        c.derive_folders()
        self.assertEqual("/Users/behrens/Documents/projects/pebble/puncover/pebble/src/puncover.c", s1[collector.PATH])
        self.assertIsNotNone(s1[collector.FILE])

        self.assertEqual("/Users/thomas/work/arm-eabi-toolchain/gcc-4.7-2012.09/libgcc/config/arm/ieee754-df.S", s2[collector.PATH])
        self.assertIsNotNone(s2[collector.FILE])

        self.assertEqual("src/puncover.c", s3[collector.PATH])
        self.assertIsNotNone(s3[collector.FILE])
コード例 #49
0
ファイル: app.py プロジェクト: grid5000/kwapi-g5k
def make_app():
    """Instantiates Flask app, attaches collector database."""
    LOG.info('Starting API')
    app = flask.Flask(__name__)
    app.register_blueprint(v1.blueprint, url_prefix='')

    collector = Collector()
    collector.clean()

    thread.start_new_thread(listen, (collector.add,))

    @app.before_request
    def attach_config():
        flask.request.collector = collector
        collector.lock.acquire()

    @app.after_request
    def unlock(response):
        collector.lock.release()
        return response

    return app
コード例 #50
0
def search(keywords):

    keyword_list = keywords.split(" ")
    collector = Collector()
    if collector.is_meiju_info_file_exist():
        collector.read_all_meiju_info_from_file()
    else:
        collector.save_all_meiju_info()
        collector.write_all_meiju_info_to_file()
    searcher = Searcher()
    meiju_ename_list = searcher.search_meiju_list_by_english_name_keyword(collector, keyword_list)
    click.echo("Total %d Meiju is found. Following are the lists:" % len(meiju_ename_list))
    for meiju_ename in meiju_ename_list:
        click.echo("%s" % meiju_ename)
    return
コード例 #51
0
ファイル: control.py プロジェクト: luipir/qgis-remote-debug
    def __init__(self, data_file=None, data_suffix=False, cover_pylib=False,
                auto_data=False):
        """Create a new coverage measurement context.
        
        `data_file` is the base name of the data file to use, defaulting to
        ".coverage".  `data_suffix` is appended to `data_file` to create the
        final file name.  If `data_suffix` is simply True, then a suffix is
        created with the machine and process identity included.
        
        `cover_pylib` is a boolean determining whether Python code installed
        with the Python interpreter is measured.  This includes the Python
        standard library and any packages installed with the interpreter.
        
        If `auto_data` is true, then any existing data file will be read when
        coverage measurement starts, and data will be saved automatically when
        measurement stops.
        
        """
        self.cover_pylib = cover_pylib
        self.auto_data = auto_data
        
        self.exclude_re = ""
        self.exclude_list = []
        
        self.file_locator = FileLocator()
        
        self.collector = Collector(self._should_trace)

        # Create the data file.
        if data_suffix:
            if not isinstance(data_suffix, basestring):
                # if data_suffix=True, use .machinename.pid
                data_suffix = ".%s.%s" % (socket.gethostname(), os.getpid())
        else:
            data_suffix = None

        self.data = CoverageData(
            basename=data_file, suffix=data_suffix,
            collector="coverage v%s" % __version__
            )

        # The default exclude pattern.
        self.exclude('# *pragma[: ]*[nN][oO] *[cC][oO][vV][eE][rR]')

        # The prefix for files considered "installed with the interpreter".
        if not self.cover_pylib:
            os_file = self.file_locator.canonical_filename(os.__file__)
            self.pylib_prefix = os.path.split(os_file)[0]

        here = self.file_locator.canonical_filename(__file__)
        self.cover_prefix = os.path.split(here)[0]
コード例 #52
0
def main():
    f = open('try_3.txt','w')
    g = open('accs.txt', 'w')
    g.close()
    task = MarioTask("testbed", initMarioMode = 2)
    task.env.initMarioMode = 2
    task.env.levelDifficulty = 1
    task.env.BASE_LEVEL = 500000

    results = [] 
    names = [] 

    iterations = 20
    rounds = 30
    learning_samples = 33
    eval_samples = 10
    
    # iterations = 5
    # rounds = 2
    # learning_samples = 3
    # eval_samples = 2
    
    if args['noisy']:
        agent = NoisySupervise(IT, useKMM = False)
        dire = './training_data_noisy/'
    else:
        agent = Supervise(IT, useKMM = False)
        dire = './training_data/'

    exp = EpisodicExperiment(task, agent) 
    C = Collector(agent,exp)
    C.collect(rounds = rounds, iterations = iterations, 
        learning_samples = learning_samples, eval_samples = eval_samples,
        directory=dire)

    print "finished"
コード例 #53
0
    def start(self):
        if self.debug:
            msg = "Package: mpx.service.logger.periodic_log\n"
            msg = msg + "Class: PeriodicLog\n"
            msg = msg + "Method: start"
            msg = msg + "starting the periodic log collecting..."
            mpx.lib.msglog.log("broadway", mpx.lib.msglog.types.DB, msg)
        if self.collector is not None:
            self.collector.stop()
        self.collector = Collector(self, self.period, self.trigger)
        for node in self.get_child("columns").children_nodes():
            self.collector.add_column(node)

        Log.start(self)
        self.collector.start()
コード例 #54
0
ファイル: probe_runner.py プロジェクト: kdevakum/alto-on-unis
	def setup(self):
		config = self.config
		logger.info('setup', name=config["name"], module=config["probe_module"], config=pprint.pformat(config))
		# logger.warn('NODE: ' + HOSTNAME, name=config["name"], module=config["probe_module"], config=pprint.pformat(config))
		probe_mod = blipp_import(config["probe_module"])
		self.probe = probe_mod.Probe(self.service, self.measurement)

		if "." in config["collection_schedule"]:
			sched_file, sched_name = config["collection_schedule"].split('.')
		else:
			sched_file, sched_name = "builtins", config["collection_schedule"]

		logger.info('setup', sched_file=sched_file, sched_name=sched_name)
		logger.warn('NODE: ' + HOSTNAME, sched_file=sched_file, sched_name=sched_name)
		self.scheduler = blipp_import("schedules." + sched_file, fromlist=[1]).__getattribute__(sched_name)
		self.scheduler = self.scheduler(self.service, self.measurement)
		self.collector = Collector(self.service, self.measurement)
コード例 #55
0
def show(name):

    collector = Collector()
    if collector.is_meiju_info_file_exist():
        collector.read_all_meiju_info_from_file()
    else:
        collector.save_all_meiju_info()
        collector.write_all_meiju_info_to_file()
    if name in collector.meiju_ename_inst_dict:
        meiju_inst = collector.meiju_ename_inst_dict[name]
        click.echo("Detailed information for Meiju - %s" % name)
        for (season_id, season_inst) in meiju_inst.season_id_inst_dict.items():
            output = "Season %d [" % season_id
            for (episode_id, episode_inst) in season_inst.episode_id_inst_dict.items():
                output += "Ep%d, " % episode_id
            output += "]"
            click.echo(output)
    else:
        click.echo("Failed to find any Meiju named %s" % name)
    return
コード例 #56
0
def download(name, season, episode, path):

    collector = Collector()
    if collector.is_meiju_info_file_exist():
        collector.read_all_meiju_info_from_file()
    else:
        collector.save_all_meiju_info()
        collector.write_all_meiju_info_to_file()
    if name in collector.meiju_ename_inst_dict:
        if int(season) == 0 and int(episode) == 0:
            downloader = Downloader()
            downloader.download_meiju(collector, name, path)
        elif int(season) != 0 and int(episode) == 0:
            downloader = Downloader()
            downloader.download_meiju_season(collector, name, int(season), path)
        elif int(season) != 0 and int(episode) != 0:
            downloader = Downloader()
            downloader.download_meiju_episode(collector, name, int(season), int(episode), path)
    else:
        click.echo("Failed to find Meiju named is %s" % name)
    return