Ejemplo n.º 1
0
 def setUpClass(self):
     cfg = Config()
     cfg.popcon_index = "test_data/.sample_pxi"
     cfg.popcon_dir = "test_data/popcon_dir"
     cfg.clusters_dir = "test_data/clusters_dir"
     cfg.popcon = 0
     self.rec = Recommender()
Ejemplo n.º 2
0
class PyToApk(object):
    config = None
    commandArgs = None

    def __init__(self, cmdArgs):
        super(PyToApk, self).__init__()
        self.config = Config(os.path.dirname(os.path.realpath(__file__)))
        self.config.parseCmdArgs(cmdArgs)
        self.commandArgs = cmdArgs.commandArgs

    def executeTask(self, task):
        if not self.config.validateValues():
            return False
        command = getattr(getattr(__import__('src.commands.' + task), 'commands'), task)
        success = False
        try:
            success = command.run(self.config, self.commandArgs)
        except KeyboardInterrupt:
            self.config.logger.error('Cancelling build due to interrupt.')
        except Exception as e:
            import traceback
            self.config.logger.error('Caught exception: ' + str(e))
            output = self.config.logger.getOutput()
            output = sys.stderr if output == sys.stdout else output
            traceback.print_exception(*sys.exc_info(), file=output)
        finally:
            return success
Ejemplo n.º 3
0
 def __init__(self, 
              config_file, 
              thread_count=1, 
              timeout=60, 
              seed=''):
     '''Initializer,
     
         Arguments:
         
             config_file             the file containing configuration data.
             thread_count            the number of worker threads to use.
             timeout                 the time workers should wait on request.
             seed                    a website for Scrappy to crawl.
             
         Throws:
         
             Currently passes all exceptions, but can caught a 
             KeyboardInterrupt.
     '''
              
     self.config_file = config_file
     self.config = Config()
     self.thread_count = thread_count
     self.timeout = timeout
     self.visited_urls = SetWrapper(set())
     self.unvisited_urls = SetWrapper(set())
     self.targets = SetWrapper(set(), format="{},\n")
     self.verbose = False
     self.saverate = 6
     self.seed = seed
Ejemplo n.º 4
0
def main():
    reload(sys)
    sys.setdefaultencoding('utf8')
    parser = argparse.ArgumentParser(description="Processing of messages protobuf")
    parser.add_argument('CONFIG_DAEMON', type=str)
    config_file = ""
    try:
        args = parser.parse_args()
        config_file = args.CONFIG_DAEMON
    except argparse.ArgumentTypeError:
        print("Bad usage, learn how to use me with %s -h" % sys.argv[0])
        sys.exit(1)
    config_data = Config()
    config_data.load(config_file)
    daemon = Daemon(config_data)
    daemon.run()

    sys.exit(0)
Ejemplo n.º 5
0
class Environment:
    """
    Benchmark environment.

    Responsible for bookkeeping and collecting results.
    """

    def __init__(self):
        self.config = Config()
        self.files = []
        self.file_writer = None
        self.results = RunResult()
        self.analyzer = None

    def exec(self, config_file_name):
        self.init_env(config_file_name)
        self.collect_test_files()
        self.print_info()
        self.run_processes()

    def init_env(self, config_file):
        self.config.read_config_file(config_file)

    def analyze(self):
        self.analyzer = ResultProcessor(self.results, self.config)
        self.analyzer.write_result_files()
        self.analyzer.print_summary()

    def print_info(self):
        self.config.print()
        self.print_file_list()

    def collect_test_files(self):
        """
        Searches recursively in the test folder for .sil files.
        These files are filtered against the ignore list and constitute
        all programs to be benchmarked.
        :return: None
        """
        for root, dirs, files in os.walk(self.config.testFolder):
            bench_files = [os.path.normpath(os.path.join(root, f)) for f in files if f.endswith('.sil') and
                           not os.path.join(root, f).endswith(tuple(self.config.ignoreList))]
            self.files.extend(bench_files)

    def print_file_list(self):
        print(str(len(self.files)) + " file(s) included in the benchmark.")
        if self.config.list_files:
            for file in self.files:
                print("    " + file)
        print()

    def run_processes(self):
        """
        Runs all the benchmarks.
        :return: None
        """
        i = 1
        total = len(self.files) * len(self.config.run_configurations) * self.config.repetitions
        for file in self.files:
            for run_config, config_name in zip(self.config.run_configurations, self.config.run_config_names):
                runner = ProcessRunner(run_config, file, config_name, self.config)
                timings = runner.run(i, total)
                self.results.add_results(timings)
                i += self.config.repetitions
Ejemplo n.º 6
0
 def __init__(self):
     self.config = Config()
     self.files = []
     self.file_writer = None
     self.results = RunResult()
     self.analyzer = None
Ejemplo n.º 7
0
def start():
    '''Start the Application'''

    log = configure_log()
    log.info('Starting Cloud Worker Node Agent')
    log.info('--------------------------')

    args = parse_args()

    settings = {'base_url': args.server,
                'secret': args.secret,
                'client_id': C.CLIENT_ID,
                'client_secret': C.CLIENT_SECRET,
                'username': C.USERNAME,
                'password': C.PASSWORD}

    server = Server(settings)

    node = Node(server)

    #Send the hostname, ip etc to the server
    node.send_info()

    #Update the node status to ready
    node.update_node_status(C.STATUS_READY)

    #Get Config
    config = Config(server, node)

    actions = Action(server, node)
    processor = Processor()
    workers = Worker(server, node, processor)
    output = Output(server, node)

    finished = False

    #Loop forever (kind of)
    while not finished:
        log.info('Looping')
        log.info('--------------------------')

        #Update last seen date
        node.update_node_date()

        #Get config
        config.refresh()

        #Get actions
        num_pending = actions.get_pending()

        #Respond to actions
        if actions.has_pending():
            message = 'Responding to %d Actions ...' % num_pending
            output.send(message)

            actions.respond_to_pending()


        #Get workers/commands
        workers.refresh()
        workers.process_workers()

        #TODO
        #Respond to/run commands
        #Send output to server


        log.info('Sleeping for %d seconds ...', 
                 config.get(C.CONFIG_POLL_PERIOD))
        time.sleep(config.get(C.CONFIG_POLL_PERIOD))
Ejemplo n.º 8
0
class Scrappy(object):
    '''Scrappy is an object that represents the spider itself. The main run 
    method will block the current thread. The work is done by UrlMiners and the
    RecordingMiner.
    '''

    def __init__(self, 
                 config_file, 
                 thread_count=1, 
                 timeout=60, 
                 seed=''):
        '''Initializer,
        
            Arguments:
            
                config_file             the file containing configuration data.
                thread_count            the number of worker threads to use.
                timeout                 the time workers should wait on request.
                seed                    a website for Scrappy to crawl.
                
            Throws:
            
                Currently passes all exceptions, but can caught a 
                KeyboardInterrupt.
        '''
                 
        self.config_file = config_file
        self.config = Config()
        self.thread_count = thread_count
        self.timeout = timeout
        self.visited_urls = SetWrapper(set())
        self.unvisited_urls = SetWrapper(set())
        self.targets = SetWrapper(set(), format="{},\n")
        self.verbose = False
        self.saverate = 6
        self.seed = seed
        
    def load(self):
        '''Load all configuration tokens and assign them.
        
            Throws:
            
                Shouldn't throw any exceptions, but may fail to load a file if
                the file doesn't exist. 
        '''
		
        print('\nLoading =>\n')
            
        self.config.load(self.config_file)
        self.target_file = self.config['target_file']
        self.visited_file = self.config['visited_file']
        self.unvisited_file = self.config['unvisited_file']
        
        print('\t\tConfig File: {}'.format(self.config_file))
        print('\t\tTarget File: {}'.format(self.target_file))
        print('\t\tUnvisited File: {}'.format(self.unvisited_file))
        print('\t\tVisited File: {}'.format(self.visited_file))
        print('\t\tTimeout: {}'.format(self.timeout))
        print('\t\tSave rate: {}'.format(self.saverate))
        print('\t\tThread count: {}'.format(self.thread_count))
                    
        if self.seed:
            self.unvisited_urls.add(self.seed)
                    
        if isfile(self.unvisited_file):
            with open(self.unvisited_file, 'r') as infile:
                for url in infile:
                    if url:
                        self.unvisited_urls.add(url)
            print('\n\t\tLoaded {} '
                  'unvisited sites.'.format(len(self.unvisited_urls.data)))
        
        if isfile(self.visited_file):
            with open(self.visited_file, 'r') as infile:
                for url in infile:
                    if url:
                        self.visited_urls.add(url)
            print('\t\tLoaded {} '
                  'visited sites.'.format(len(self.visited_urls.data)))
        
        if isfile(self.target_file):
            with open(self.target_file, 'r') as infile:
                for target in infile:
                    # -2, cut off new line and ,
                    self.targets.add(target[:len(target) - 2])
            print('\t\tLoaded {} '
                  'targets.'.format(len(self.targets.data)))
        
    def run(self):
        '''This is the main method which blocks the current thread's execution.
        The thread will block until the queue is empty. If the queue is never
        empty then the thread will never unblock. You can use a keyboard 
        interrupt to save and end. 
        
            Throws:
            
                Shouldn't throw anything... we hope.
        '''

        start_time = time()
    
        print('\nRunning =>\n')
        print('\t\tStarted at {}\n'.format(TimeStamp().now()))
            
        try:
            url_queue = Queue()
                
            for url in self.unvisited_urls.data:
                url_queue.put(url)
            
            for i in range(0, self.thread_count):
                thread = UrlMiner(url_queue, 
                                  self.unvisited_urls, 
                                  self.visited_urls, 
                                  self.targets, 
                                  self.timeout)
                thread.setDaemon(True)
                thread.verbose = self.verbose
                thread.start()
                    
            recorder = RecordingMiner([[self.unvisited_urls,self.unvisited_file],
                                       [self.visited_urls, self.visited_file], 
                                       [self.targets, self.target_file]], 
                                       interval=self.saverate)        
                                                  
            recorder.setDaemon(True)
            recorder.start()

            while not url_queue.empty():
                sleep(10)
                
            recorder.running = False
            
        except KeyboardInterrupt:
            recorder.running = False
            recorder.save()
            
        print('\nEnding =>\n'
              '\t\tElapsed time {0:.2f} '
              'minutes.'.format((time() - start_time) / 60))
              
        print('\t\tScraped {} sites.'.format(len(self.visited_urls.data)))
                                       
        print('\t\tUnvisited: '
              '{}\n\t\tVisited: {}'.format(len(self.unvisited_urls.data),
                                            len(self.visited_urls.data)))
                                                           
        print('\t\tTargets: {}'.format(len(self.targets.data)))
Ejemplo n.º 9
0
 def __init__(self, cmdArgs):
     super(PyToApk, self).__init__()
     self.config = Config(os.path.dirname(os.path.realpath(__file__)))
     self.config.parseCmdArgs(cmdArgs)
     self.commandArgs = cmdArgs.commandArgs