Esempio n. 1
0
    def _cfn_param_updater(self, param, comparison_parameters,
                           stage_parameters, file_name):  # pylint: disable=R0912
        """
        Generic CFN Updater method
        """
        resolver = Resolver(self.parameter_store, stage_parameters,
                            comparison_parameters)

        for key, value in comparison_parameters[param].items():
            if str(value).startswith('resolve:'):
                if resolver.fetch_parameter_store_value(value, key, param):
                    continue
            if str(value).startswith('import:'):
                if resolver.fetch_stack_output(value, key, param):
                    continue
            if str(value).startswith('upload:'):
                if resolver.upload(value, key, file_name, param):
                    continue
            resolver.update_cfn(key, param)

        for key, value in stage_parameters[param].items():
            if str(value).startswith('resolve:'):
                if resolver.fetch_parameter_store_value(value, key, param):
                    continue
            if str(value).startswith('import:'):
                if resolver.fetch_stack_output(value, key, param):
                    continue
            if str(value).startswith('upload:'):
                if resolver.upload(value, key, file_name, param):
                    continue

        return resolver.__dict__.get('stage_parameters')
Esempio n. 2
0
class TestResolver(unittest.TestCase):
    """
    Test class for Resolver
    """
    def setUp(self):
        self.resolver = Resolver()

    def testARecords(self):
        reference_a_record = [(1, '31.170.165.34')]
        test_a_record = self.resolver.resolve('eur.al')
        self.assertEqual(reference_a_record, test_a_record)

    def testNoResponse(self):
        reference_no_response = Resolver.NO_RESPONSE
        self.resolver.port = 65000
        test = self.resolver.resolve('eur.al')
        self.assertEqual(reference_no_response, test)
        self.resolver.port = 53

    def testNotFound(self):
        reference = Resolver.NAME_NOT_FOUND
        test = self.resolver.resolve('opsidjgsdkjf.paris')
        self.assertEqual(reference, test)

    def testCNameRecord(self):
        reference = 5, 'dijkstra.urgu.org'
        test = self.resolver.resolve('anytask.urgu.org')
        self.assertEqual(reference, test[0])

    def testRecursion(self):
        self.resolver = Resolver('198.41.0.4')
        reference_a_record = [(1, '31.170.165.34')]
        test_a_record = self.resolver.resolve('eur.al')
        self.assertEqual(reference_a_record, test_a_record)
    def __init__(self, url, uid, storage):
        """
        @param url: monitored URL
        @type url:  basestring (str or unicode)
        @param uid: user identifier. This ID has to be unique for each one,
                    who is using changemonitor.
        @type uid: str
        @param storage: storage of monitored-resource data
        @type storage: model.Storage
        """
        # resource data
        self.url = url
        self.uid = uid

        # models
        self.storage = storage
        self.headers = storage._headermeta

        # resolver
        self.resolver = Resolver(storage)

        # file
        try:
            self.file = self.storage.get(url)
        except DocumentNotAvailable:
            # if the file is not in the storage, resolver has to check
            # the url and load actual data into the storage
            self.resolver.resolve(url)
            try:
                self.file = self.storage.get(url)
            except DocumentNotAvailable:
                raise DocumentNotAvailable("Resource '%s' is not available." % url)
Esempio n. 4
0
    def __init__(self,
                 conf=Config.default,
                 user=None,
                 host=None,
                 reactor=reactor):
        """Construct a new client context
        
        Prepare a new context with the given config.
        If user and host are None then they are collected
        from the environment.
        """
        self.conf, self.reactor = conf, reactor

        self.circuits = CACircuitFactory(self)

        self.resolv = Resolver(self.circuits, conf)
        self._onClose = DeferredManager()

        if host is None:
            from socket import gethostname
            host = gethostname()
        self.host = host

        if user is None:
            from getpass import getuser
            user = getuser()
        self.user = user

        reactor.addSystemEventTrigger("before", "shutdown", self.close)
Esempio n. 5
0
    def _sc_param_updater(self, comparison_parameters, stage_parameters,
                          file_name):  # pylint: disable=R0912
        """
        Compares parameter files used for the Service Catalog deployment type
        """
        resolver = Resolver(self.parameter_store, stage_parameters,
                            comparison_parameters)

        for key, value in comparison_parameters.items():
            if str(value).startswith('resolve:'):
                if resolver.fetch_parameter_store_value(value, key):
                    continue
            if str(value).startswith('import:'):
                if resolver.fetch_stack_output(value, key):
                    continue
            if str(value).startswith('upload:'):
                if resolver.upload(value, key, file_name):
                    continue
            resolver.update_sc(key)

        for key, value in stage_parameters.items():
            if str(value).startswith('resolve:'):
                if resolver.fetch_parameter_store_value(value, key):
                    continue
            if str(value).startswith('import:'):
                if resolver.fetch_stack_output(value, key):
                    continue
            if str(value).startswith('upload:'):
                if resolver.upload(value, key, file_name):
                    continue

        return resolver.__dict__.get('stage_parameters')
    def run(self):
        """ Run the handler thread """

        recv_header = self.received_message.header

        try:
            questions = self.received_message.questions[0]
            qd_count = len(questions)
        except IndexError:
            return

        # if recv_header.rd:
        try:
            resolver = Resolver(True)
            answer_sname, addresses, aliasses = resolver.gethostbyname(questions[0].rdata.data)
        except ResolverException as e:
            print e
            return

        answers = list()
        for address in addresses:
            answer = message.Section(answer_sname, Type.A, Class.IN, 86400, len(address), address)
            answers.append(answer)

        send_header = message.Header(recv_header.ident, 0, qd_count, len(addresses), 0, 0)
        send_header.qr = 1
        send_header.opcode = 0
        send_header.rd = recv_header.rd
        response_message = message.Message(send_header, questions, answers)
        response_bytes = response_message.to_bytes()
        sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
        sock.settimeout(3)
        sock.send(response_bytes, (self.ip_address, 53))
Esempio n. 7
0
    def find_serializer_fields(self, serializer_name):
        nodes = self.serializer_registry.nodes

        if serializer_name in self.memo_dict:
            return self.memo_dict[serializer_name]

        class_node = nodes[serializer_name]
        fields = Fields()
        init_node = None

        # Look at own class variables first, this trumps everything else
        for node in class_node.body:
            if self.is_class_var(node):
                # explicit class var trumps Meta
                fields.add(Resolver.drf_field_assignment(node), overwrite=True)
            elif self.is_meta(node):
                fields.extend(Resolver.drf_meta_fields(node))
            elif self.is_init_method(node):
                init_node = node

        # add fields from bases, in left to right order. The bases of the base
        # trumps the neighbour of the base if there's overlap.
        for base in class_node.bases:
            base = Resolver.resolve(base)

            if base == 'object':
                continue

            if base not in nodes:
                # TODO: ???
                continue

            base_class_vars = self.find_serializer_fields(base)
            fields.extend(base_class_vars)

        # Check for dynamic fields that were inherited from direct ancestors.
        # TODO: Find a better way to support inheritance
        parent_in_dynamic_fields = any(
            getattr(parent_class, 'attr', None) in self.dynamic_fields
            or getattr(parent_class, 'id', None) in self.dynamic_fields
            for parent_class in class_node.bases)

        # dynamic fields trump or augment existing fields
        if serializer_name in self.dynamic_fields or parent_in_dynamic_fields:
            if init_node:
                dynamic_fields = Resolver.init_method(init_node)
                for field_name, field in dynamic_fields.iteritems():
                    if field_name not in fields:
                        fields.add(field)
                        continue

                    previous_field = fields[field_name]
                    augmented_field = self.augment_field(previous_field, field)
                    fields.add(augmented_field, overwrite=True)

        self.memo_dict[serializer_name] = fields

        return fields
Esempio n. 8
0
def main():
    parser = create_parser()
    parsed_args = parser.parse_args(sys.argv[1:])
    dns_addr = parsed_args.start_server
    resolver = Resolver(dns_addr)
    try:
        resolver.start_listening()
    except socket.error:
        sys.exit()
Esempio n. 9
0
class Recognizer:
    def __init__(self):
        self.resolver = Resolver()
        self.thread = Thread(target=self.run, args=()).start()

    def run(self):
        faces = []

        while True:
            try:
                frame = Store.movement_frames.get(True, 2)

                face_locations = face_recognition.face_locations(frame)
                face_encodings = face_recognition.face_encodings(
                    frame, face_locations)

                for (top, right, bottom,
                     left), encoding in zip(face_locations, face_encodings):
                    cropped_img = frame[top - 10:bottom + 10,
                                        left - 10:right + 10]

                    faces.append({
                        "img": frame[top:bottom, left:right],
                        "encoding": encoding
                    })

            except queue.Empty:
                if len(faces) > 0:
                    self.resolver.resolve(self.preprocess_faces(faces).copy())
                    faces = []

    def preprocess_faces(self, faces):
        # Cluster the faces with chinese whispers
        encodings = [dlib.vector(face['encoding']) for face in faces]
        labels = dlib.chinese_whispers_clustering(encodings, 0.5)

        selected_faces = []

        # Select face most close to average group
        groups = list(set(labels))
        for group in groups:
            # Get indices for each group
            indices = [i for i in range(len(labels)) if labels[i] == group]
            group_encodings = [faces[i]['encoding'] for i in indices]

            # Get centroid for group encodings
            avg_group_encoding = np.average(group_encodings, axis=0)

            # Get the closest face to the centroid
            avg_distance = face_recognition.face_distance(
                group_encodings, avg_group_encoding)
            min_index = np.argmin(avg_distance)

            face_index = indices[min_index]
            selected_faces.append(faces[face_index])

        return selected_faces
Esempio n. 10
0
    def find_serializer_fields(self, serializer_name):
        nodes = self.serializer_registry.nodes

        if serializer_name in self.memo_dict:
            return self.memo_dict[serializer_name]

        class_node = nodes[serializer_name]
        fields = Fields()
        init_node = None

        # Look at own class variables first, this trumps everything else
        for node in class_node.body:
            if self.is_class_var(node):
                # explicit class var trumps Meta
                fields.add(Resolver.class_var_drf_field(node), overwrite=True)
            elif self.is_meta(node):
                fields.extend(Resolver.drf_meta_fields(node))
            elif self.is_init_method(node):
                init_node = node

        # add fields from bases, in left to right order. The bases of the base
        # trumps the neighbour of the base if there's overlap.
        for base in class_node.bases:
            base = Resolver.resolve(base)

            if base == 'object':
                continue

            if base not in nodes:
                # TODO: ???
                continue

            base_class_vars = self.find_serializer_fields(base)
            fields.extend(base_class_vars)

        # dynamic fields trump or augment existing fields
        if serializer_name in self.dynamic_fields:
            if not init_node:
                msg = ('Did not find __init__ in {} but view specifies dynamic'
                       ' fields.').format(serializer_name)
                raise Exception(msg)

            dynamic_fields = Resolver.init_method(node)
            for field in dynamic_fields:
                if field not in fields:
                    fields.add(field)
                    continue

                previous_field = fields[field['field_name']]
                augmented_field = self.augment_field(previous_field, field)
                fields.add(augmented_field, overwrite=True)

        self.memo_dict[serializer_name] = fields

        return fields
    def test_do_simple(self):
        """ Simple config resolution """
        resolver = Resolver({
            'foo': {
                'module': 'example_classes',
                'class': 'Foo'
            }
        })

        services = resolver.do()

        self.assertTrue('foo' in services)
        assert isinstance(services['foo'], Foo)
Esempio n. 12
0
 def test_finds_source_from_js_component_folder(self):
     file = "/test/js/components/FileRenamePrompt-test.js"
     r = Resolver().get_source(file, spec_folder="test")
     self.assertEqual(len(r), 2)
     self.assertEqual(r[0],
                      "/app/javascript/components/FileRenamePrompt.vue")
     self.assertEqual(r[1], "/javascript/components/FileRenamePrompt.vue")
Esempio n. 13
0
    def dynamic_fields(self):
        if not self._dynamic_field_map:
            nodes = self.view_registry.nodes
            for class_name, class_node in nodes.iteritems():
                props = {
                    'include_filters': None,
                    'expand_filters': None,
                    'exclude_filters': None,
                    'serializer_class': None
                }

                for node in class_node.body:
                    if not self.is_class_var(node):
                        continue

                    lhs, rhs = Resolver.resolve(node)
                    if lhs in props:
                        props[lhs] = self.resolve_view_var(rhs)

                serializer_name = props.pop('serializer_class')
                truncated_props = {
                    key: val
                    for key, val in props.iteritems() if val
                }
                if truncated_props:
                    self._dynamic_field_map[serializer_name] = truncated_props

        return self._dynamic_field_map
Esempio n. 14
0
 def test_finds_source_from_erb(self):
     file = "/spec/views/namespace/users/_something.html.erb_spec.rb"
     r = Resolver().get_source(file)
     self.assertEqual(len(r), 2)
     self.assertEqual(r[0],
                      "/app/views/namespace/users/_something.html.erb")
     self.assertEqual(r[1], "/views/namespace/users/_something.html.erb")
Esempio n. 15
0
def fmt_serializer(node, fields):
    output = ('{}({})\n' '{}\n')
    table_data = tabulate(fields, headers="keys", tablefmt='grid')

    return output.format(
        node.name, ', '.join(Resolver.resolve(base) for base in node.bases),
        table_data)
Esempio n. 16
0
def run(line: str):
    scanner: Scanner = Scanner(line, error_scan)
    tokens: List[Token] = scanner.scanTokens()
    parser: Parser = Parser(tokens, error_parse)
    statements: List[stmt.Stmt] = parser.parse()

    # // Stop if there was a syntax error.
    if hadError:
        return

    interpreter: Interpreter = Interpreter(runtimeError)
    resolver: Resolver = Resolver(interpreter, error_scan)
    resolver.resolve(statements)

    # // Stop if there was a resolution error.
    if hadError:
        return

    if os.environ.get("DEBUG"):
        print("**TOKENS**")
        for t in tokens:
            print(t)
        print("**STATEMENTS**")
        for s in statements:
            print(s)
        ipdb.set_trace()

    interpreter.interpret(statements)
Esempio n. 17
0
 def _param_updater(self, comparison_parameters, stage_parameters):
     """
     Generic Parameter Updater method
     """
     resolver = Resolver(self.parameter_store, stage_parameters, comparison_parameters)
     self._determine_parameter_structure(comparison_parameters, resolver)
     self._determine_parameter_structure(stage_parameters, resolver)
     return resolver.__dict__.get('stage_parameters')
    def test_advanced(self):
        # datalogger:
        #     module: datalogger
        #     class: DataLogger
        #     args:
        #         -
        #             name: rob_logger
        #             path: /var/log/monorail
        #             file_base_name: rob
        #             max_file_bytes: 1073741824
        #             backup_count: 10
        #             port: $_port
        config = {
            'logger': {
                'module': 'example_classes',
                'class': 'TestLogger',
                'args': [
                    {
                        'name': 'test_logger',
                        'path': '/var/foo',
                        'file_base_name': 'testlog',
                        'max_file_bytes': 500,
                        'backup_count': 10,
                        'port': '$_port'
                    }
                ]
            }
        }
        scalars = {'_port': 555}
        resolver = Resolver(config, scalars)

        services = resolver.do()
        logger = services['logger']

        assert isinstance(logger, TestLogger)
        self.assertEquals(
            {
                'name': 'test_logger',
                'path': '/var/foo',
                'file_base_name': 'testlog',
                'max_file_bytes': 500,
                'backup_count': 10,
                'port': 555
            },
            logger.config
        )
Esempio n. 19
0
    def run(self, source):
        scanner = Scanner(source, self)
        tokens = scanner.scan_tokens()

        parser = Parser(self, tokens)
        statements = parser.parse()

        if self.hadError:
            return
        if self.had_runtime_error:
            return

        resolver = Resolver(self.interpreter, self)
        resolver.resolve(statements)

        if self.hadError:
            return

        self.interpreter.interpret(statements)
Esempio n. 20
0
def resolve_file(filename):
    parser = Parser()
    try:
        with open(filename, 'r') as file:
            for line in file:
                line = re.sub('\s', '', line)
                line_is_ignored = ignored_line.match(line)
                if line_is_ignored is not None:
                    continue
                try:
                    parser.handle_line(line)
                except ParseError as exception:
                    print(
                        colored(
                            'Parse error of type : {} detected on line\n\t --> {}'
                            .format(exception.exception_type, line), 'red'))
                    raise FileError()
        if not parser.queries:
            print(colored('No queries furnished', 'red'))
            raise FileError
        if args.verbose:
            print(parser)
        resolver = Resolver(parser.facts, parser.rules, args.verbose)
        for query in parser.queries:
            try:
                if args.verbose:
                    print(
                        colored('Query to solve : {}'.format(query), 'yellow'))
                response = resolver.resolve_query(query)
                print(
                    colored(
                        'query {}  has been resolved to : {}'.format(
                            query, response), 'green'))
            except ContradictionError as exception:
                print(
                    colored(
                        'Contradiction detected on {0} that is {1} while his opposite is also {1}'
                        .format(exception.name, exception.value), 'red'))
    except (FileNotFoundError, NameError, PermissionError, IsADirectoryError,
            UnicodeDecodeError) as error:
        print(colored(error, 'red'))
        raise FileError()
Esempio n. 21
0
    def run(self, source: str):
        scanner = Scanner(self, source)
        tokens = scanner.scan_tokens()

        if DEBUG:
            print(tokens)

        parser = Parser(self, tokens)
        statements = parser.parse()

        if self.had_error:
            return

        resolver = Resolver(self, self.interpreter)
        resolver.resolve_statements(statements)

        if self.had_error:
            return

        self.interpreter.interpret(statements)
Esempio n. 22
0
    def cli():
        path = input("Введите путь до изображения:\n")
        image_analyzer = ImageAnalyzer()

        image_analyzer.read_image(path)
        print("{} прочитано".format(path))
        image_analyzer.process_xo()

        main_map = image_analyzer.get_map()
        print("Расчет анализатора:\n {}".format(main_map))
        print("0 - пустота, 1 - Х, 2 - О")

        resolver = Resolver(main_map)
        res_points = resolver.resolve()
        print("Ответ алгоритма: {}".format(res_points))

        if image_analyzer.process_output(res_points):
            print("Результат в файле output.pnt")

        print("Done")
Esempio n. 23
0
File: server.py Progetto: ly95/dns
def run_server(ip='0.0.0.0'):
    try:
        s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
        s.bind((ip, 53))
    except socket.error as e:
        logging.error(e)
        return

    while True:
        data, addr = s.recvfrom(1024)
        ret = Resolver(data).do()
        s.sendto(ret, addr)
Esempio n. 24
0
def main():
    parser = argparse.ArgumentParser(description='YOBAdns-resolver')
    parser.add_argument(
        "address", metavar="Address", nargs="+", type=str,
        help='Address that you need to resolve')
    parser.add_argument(
        "--server", "-s", metavar="Server", nargs="?", type=str,
        default=DEFAULT_SERVER,
        help='Set the DNS server to first request, default: 8.8.8.8'
    )
    parser.add_argument(
        "--port", "-p", metavar="Port", nargs="?", type=int,
        default=DEFAULT_PORT,
        help='Set the port of dedicated DNS server'
    )
    parser.add_argument("-d", "--debug", action="store_true",
                        help="switch on debug mode")
    parser.add_argument("-n", "--num", nargs="?", type=int,
                        default=DEFAULT_NUM_OF_RETRIES,
                        help="number of retries")
    parser.add_argument("-w", "--waiting", nargs="?", type=int,
                        default=DEFAULT_TIMEOUT,
                        help="waiting time of request")

    args = parser.parse_args()
    ##############################################
    resolver = Resolver(args.server, args.debug,
                        args.port, args.num, args.waiting)
    received = resolver.resolve(args.address[0])
    if received == resolver.NO_RESPONSE:
        print('\tNo response')
    else:
        if received == resolver.NAME_NOT_FOUND:
            print('\t\tName {} does not exist'.format(args.address[0]))
        else:
            print('\tDomain: ' + args.address[0])
            print('\tResponses:')
            for response in received:
                print('\t\t{}\t{}'.format(resolver.TYPES[response[0]],
                                          response[1]))
    def test_do_complicated(self):
        """ Instantiate services from advanced YAML file """
        config = yaml.load(open('test/test_config.yml', 'r')) or {}
        resolver = Resolver(config)

        services = resolver.do()

        assert isinstance(services['foo'], Foo)
        assert isinstance(services['bar'], Bar)
        assert isinstance(services['baz'], Baz)
        assert isinstance(services['qux'], Qux)
        assert isinstance(services['wobble'], Wobble)
        assert isinstance(services['spam'], Spam)

        spam = services['spam']
        self.assertEquals(spam.ham, 'ham!')
        self.assertEquals(spam.eggs, 'eggz')

        wobble = services['wobble']
        assert isinstance(wobble.foo, Foo)
        assert isinstance(wobble.bar, Bar)
        assert isinstance(wobble.baz, Baz)
        assert isinstance(wobble.spam, Spam)
Esempio n. 26
0
    def __init__(self, handlers=None):
        if not handlers:
            handlers = []

        AsyncHTTPClient.configure(
            "tornado.curl_httpclient.CurlAsyncHTTPClient",
            max_clients=CURL['MAX_CLIENTS'],
        )

        self.cache = RedisCache(**REDIS['CLIENT'])
        self.cache.initialize(**REDIS['TORNADO_CLIENT'])

        tornado.ioloop.IOLoop.instance().add_future(
            self.cache.run(), lambda future: future.result())

        Api = get_db_api(is_cached=True)
        self.db_api = Api(
            DB_API['HOST'],
            DB_API['PORT'],
            http_client=AsyncHTTPClient(),
            resolver=Resolver(ttl=DNS_RECORD_TTL),
            cache=self.cache,
            cache_key_ttl=DB_API['CACHE_KEY_TTL'],
            connect_timeout=CURL['CONNECT_TIMEOUT'],
            request_timeout=CURL['REQUEST_TIMEOUT'],
        )

        self.jinja = Environment(loader=FileSystemLoader(
            JINJA['TEMPLATE_ROOT']),
                                 **JINJA['SETTINGS'])

        handlers += [
            (r'/db/(?P<api_path>.*)', DBApiRequestHandler),
        ]

        if DEBUG:
            handlers += [
                (r'/test/', TestView),
            ]

        config = dict(debug=DEBUG, )

        tornado.web.Application.__init__(self, handlers, **config)
Esempio n. 27
0
 def test_resolver(self):
     from resolver import Resolver
     import pytis.presentation
     r = Resolver(search=('pytis', 'pytis.defs.profiles'))
     view = r.get('cms.Languages', 'view_spec')
     self.assertIsInstance(view, pytis.presentation.ViewSpec)
     spec = r.specification('cms.Languages')
     self.assertIsInstance(spec, pytis.presentation.Specification)
     # Test top level specification name (from pytis.defs.profiles).
     spec2 = r.specification('FormProfiles')
     self.assertIsInstance(spec2, pytis.presentation.Specification)
     specifications = [spec for name, spec in r.walk()]
     from pytis.cms import Languages
     self.assertIn(Languages, specifications)
class MonitoredResource(object):
    """
    Monitored resource (URL). The ressource is generally any document in any
    format, but most often it will be HTML code.

    This class wraps the URL content and metadata.

    The contents can be manipulated within the time so it can provide information
    about how the content changed in different versions of the document.

    Warning:
    Application developers should generally not need to instantiate this class
    directly. The only correct way how to get this object is through
    Monitor.get() method.

    Design pattern: Active Record

    Example of checking new version of document:
        >>> from rrslib.web.changemonitor import Monitor
        >>> monitor = Monitor(user_id="myuid")
        >>> monitor
        Monitor(conn=Connection('localhost', 27017), dbname='webarchive', uid='myuid')
        >>> resource = monitor.get("http://www.myusefulpage.com/index.html")
        >>> resource
        <MonitoredResource(url='http://www.myusefulpage.com/index.html', uid='myuid') at 0xb7398accL>
        >>> resource.check()
        True
        >>> # the resource has changed

    Checking availability of the document on the URL
        >>> from rrslib.web.changemonitor import HTTPDateTime
        >>> resource = monitor.get("http://www.nonexistentpage.com")
        >>> resource.available()
        False
        >>> resource = monitor.get("http://www.myusefulpage.com/index.html")
        >>> resource.available(HTTPDateTime(2012, 6, 30, 15, 34))
        True

    Example of getting last available version of the document on the URL
        >>> resource = monitor.get("http://www.myusefulpage.com/index.html")
        >>> content = resource.get_last_version()
        >>> print content.data
        <html><head>
        ...
        >>> resource = monitor.get("http://www.crazynonexistentpage.com")
        >>> content = resource.get_last_version()
        Traceback (most recent call last):
          File "<stdin>", line 1, in <module>
        DocumentNotAvailable: The content of this URL is not available and it
        is not in the storage.

    Example of getting version of the document in exact time
        >>> resource = monitor.get("http://www.myusefulpage.com/index.html")
        >>> content = resource.get_version(HTTPDateTime(2012, 6, 30, 15, 34))

    Getting the last time when the document was checked:
        >>> resource = monitor.get("http://www.crazynotexistentpage.com")
        >>> resource.last_checked()
        HTTPDateTime(Thu, 01 Jan 1970 00:00:00 GMT)
    """
    def __init__(self, url, uid, storage):
        """
        @param url: monitored URL
        @type url:  basestring (str or unicode)
        @param uid: user identifier. This ID has to be unique for each one,
                    who is using changemonitor.
        @type uid: str
        @param storage: storage of monitored-resource data
        @type storage: model.Storage
        """
        # resource data
        self.url = url
        self.uid = uid

        # models
        self.storage = storage
        self.headers = storage._headermeta

        # resolver
        self.resolver = Resolver(storage)

        # file
        try:
            self.file = self.storage.get(url)
        except DocumentNotAvailable:
            # if the file is not in the storage, resolver has to check
            # the url and load actual data into the storage
            self.resolver.resolve(url)
            try:
                self.file = self.storage.get(url)
            except DocumentNotAvailable:
                raise DocumentNotAvailable("Resource '%s' is not available." % url)


    def check(self):
        """
        Check the resource URL and load the most recent version into database.

        TODO: consider using @lazy decorator. Most of use cases use this method
        so we have to insure that it will be called only once.

        @raises: DocumentTooLargeException
        @returns: True if the document has changed since last check.
        """
        # bude vyuzivat resolveru pro checknuti URL a ziskani informace o tom,
        # jestli byl dokument zmenen. Mozna bude take dobre nahrat rovnou do
        # self.file nejnovejsi verzi, ale o tom je potreba jeste pouvazovat.

        # urcite je potreba pred kazdym checkem refreshout file cache
        # self.file.refresh_cache()

        #self.resolver.resolve(self.url)
        raise NotImplementedError()


    def get_last_version(self):
        """
        Get last available content of the document. If the document is available
        at this time, returns most recent version which is on the web server.

        @returns: Last available content of this resource.
        @rtype: Content
        @raises: DocumentNotAvailable if no content available (resource does not
        exist on the URL and never existed within the known history)
        """
        self.resolver.resolve(self.url)
        try:
            return self.file.get_last_version()
        except NoFile: # FIXME tady to prece nemuze byt??!
            raise DocumentNotAvailable("Resource is not available.")


    def get_version(self, time_or_version):
        """
        Get content of this document in specified time or version. If the
        document was not available in given time, returns last available content.
        If there is no available content until given time, raises exception.

        @param time_or_version: Time or version of the content we want to retrieve.
            Version numbering is a convenience atop the GridFS API provided
            by MongoDB. version ``-1`` will be the most recently uploaded
            matching file, ``-2`` the second most recently uploaded, etc.
            Version ``0`` will be the first version
            uploaded, ``1`` the second version, etc. So if three versions
            have been uploaded, then version ``0`` is the same as version
            ``-3``, version ``1`` is the same as version ``-2``, and
            version ``2`` is the same as version ``-1``.
        @type time_or_version: HTTPDateTime or int
        @raises: DocumentHistoryNotAvailable if there is no available content until
        given time or version
        """
        if isinstance(time_or_version, HTTPDateTime):
            return self.file.get_version(time_or_version.to_timestamp())
        elif isinstance(time_or_version, int):
            return self.file.get_version(time_or_version)
        else:
            raise TypeError("Version time has to be type HTTPDateTime or GridFS version (int).")


    def get_diff(self, start, end):
        """
        @param start: start time or version to be diffed
        @type start: HTTPDateTime or int
        @param end: end time or version to be diffed
        @type end: HTTPDateTime or int
        @returns: either textual or binary diff of the file (if available).
                  If contents are equal (document did not change within this
                  time range) returns None.
        @rtype: unicode
        @raises: DocumentHistoryNotAvaliable if the storage doesn't provide
                 enough data for computing the diff.
        """
        content_start = self.get_version(start)
        content_end = self.get_version(end)
        if content_start == content_end:
            return None
        return content_start.diff_to(content_end)


    def available(self, httptime=None):
        if not isinstance(httptime, HTTPDateTime):
            raise TypeError("Time of availability has to be type HTTPDateTime.")
        # Pokud je httptime=None, pak se jedna o dostupnost v tomto okamziku
        raise NotImplementedError()


    def last_checked(self):
        """
        Get information about the time of last check of this resource.

        @returns: time of last check or None if the resource was never checked
                  (or the HTTP requests timed out)
        @rtype: HTTPDateTime or None
        """
        return self.headers.last_checked(self.url)


    def __repr__(self):
        return "<MonitoredResource(url='%s', uid='%s') at %s>" % \
            (self.url, self.uid, hex(id(self)))

    __str__ = __repr__
# 03_02-Callable Instances

from resolver import Resolver
resolve = Resolver()
resolve('localhost')
resolve.__call__('localhost')
resolve._cache
resolve('sixty-north.com')
resolve.__call__('sixty-north.com')
resolve._cache
resolve('pluralsight.com')
resolve._cache


from timeit import timeit
timeit(setup='from __main__ import resolve', stmt="resolve('localhost')", number=1)
timeit(setup='from __main__ import resolve', stmt="resolve('localhost')", number=1)

timeit(setup='from __main__ import resolve', stmt="resolve('python.org')", number=1)
timeit(setup='from __main__ import resolve', stmt="resolve('python.org')", number=1)
print("{:f}".format(_))
exit()

from resolver import Resolver
resolve = Resolver()
resolve.has_host('localhost')
resolve('localhost')
resolve.has_host('localhost')

resolve.clear()
resolve.has_host('localhost')
Esempio n. 30
0
 def step_three(self, ast):
     resolver = Resolver(ast)
     resolver.traverse()
Esempio n. 31
0
class Downloader(object):
    def __init__(self, base="http://repo1.maven.org/maven2", username=None, password=None):
        self.requestor = Requestor(username, password)
        self.resolver = Resolver(base, self.requestor)

    
    def download(self, artifact, filename=None, suppress_log=False):
        filename = artifact.get_filename(filename)
        url = self.resolver.uri_for_artifact(artifact)
        if not self.verify_md5(filename, url + ".md5"):
            if not suppress_log:
                print("Downloading artifact " + str(artifact))
                hook=self._chunk_report
            else:
                hook=self._chunk_report_suppress

            onError = lambda uri, err: _throwDownloadFailed("Failed to download artifact " + str(artifact) + "from " + uri)
            response = self.requestor.request(url, onError, lambda r: r)
            
            if response:
                with open(filename, 'w') as f:
                    self._write_chunks(response, f, report_hook=hook)
                if not suppress_log:
                    print("Downloaded artifact %s to %s" % (artifact, filename))
                return (artifact, True)
            else:
                return (artifact, False)
        else:
            if not suppress_log:
                print("%s is already up to date" % artifact)
            return (artifact, True)

    def _throwDownloadFailed(msg):
        raise RequestException(msg)

    def _chunk_report_suppress(self, bytes_so_far, chunk_size, total_size):
        pass

    def _chunk_report(self, bytes_so_far, chunk_size, total_size):
        percent = float(bytes_so_far) / total_size
        percent = round(percent*100, 2)
        sys.stdout.write("Downloaded %d of %d bytes (%0.2f%%)\r" %
                 (bytes_so_far, total_size, percent))

        if bytes_so_far >= total_size:
            sys.stdout.write('\n')

    def _write_chunks(self, response, file, chunk_size=8192, report_hook=None):
        total_size = response.info().getheader('Content-Length').strip()
        total_size = int(total_size)
        bytes_so_far = 0

        while 1:
            chunk = response.read(chunk_size)
            bytes_so_far += len(chunk)

            if not chunk:
                break

            file.write(chunk)
            if report_hook:
                report_hook(bytes_so_far, chunk_size, total_size)

        return bytes_so_far

    def verify_md5(self, file, remote_md5):
        if not os.path.exists(file):
            return False
        else:
            local_md5 = self._local_md5(file)
            onError = lambda uri, err: _throwDownloadFailed("Failed to download MD5 from " + uri)
            remote = self.requestor.request(remote_md5, onError, lambda r: r.read())
            return local_md5 == remote

    def _local_md5(self, file):
        md5 = hashlib.md5()
        with open(file, 'rb') as f:
            for chunk in iter(lambda: f.read(8192), ''):
                md5.update(chunk)
        return md5.hexdigest()
Esempio n. 32
0
username = None
password = None
domain = None
base = None

def onLoginInfo(base, domain, username, password):
    print base, domain, username, password
    
    client_jid = jid.JID("%s@%s" % (username, domain))
    secret = password
    c = Client(client_jid, secret, True, base)
    username = username
    password = password
    domain = domain
    base = base
    print 'Begin to connect'
    resolv.getFriendsList()
    
def onListDownloaded(friendlist):
    print friendlist


if(__name__ == "__main__"):
    resolv = Resolver('kkszysiu', 'xxxxx')
    resolv.getLoginInfo()
    resolv.onLoginInfo = onLoginInfo
    resolv.onListDownloaded = onListDownloaded

    #log.startLogging(sys.stdout)
    reactor.run()
Esempio n. 33
0
from resolver import Resolver
from timeit import timeit

resolve = Resolver()

time_elapsed = timeit(setup='from __main__ import resolve ',
                      stmt='resolve("google.com")',
                      number=1)
print("Time elapsed for google.com:{0:f}".format(time_elapsed))

time_elapsed = timeit(setup='from __main__ import resolve ',
                      stmt='resolve("google.com")',
                      number=1)
print("Time elapsed for google.com:{0:f}".format(time_elapsed))

print("Is microsoft.com present in resolve cache:{0}".format(
    resolve.has_host("microsoft.com")))

resolve("microsoft.com")

print("Is microsoft.com present in resolve cache:{0}".format(
    resolve.has_host("microsoft.com")))

resolve.clear()
print("Is microsoft.com present in resolve cache:{0}".format(
    resolve.has_host("microsoft.com")))
Esempio n. 34
0
 def testRecursion(self):
     self.resolver = Resolver('198.41.0.4')
     reference_a_record = [(1, '31.170.165.34')]
     test_a_record = self.resolver.resolve('eur.al')
     self.assertEqual(reference_a_record, test_a_record)
Esempio n. 35
0
 def setUp(self):
     self.resolver = Resolver()
Esempio n. 36
0
class MonitoredResource(object):
    """
    Monitored resource (URL). The ressource is generally any document in any
    format, but most often it will be HTML code.

    This class wraps the URL content and metadata.

    The contents can be manipulated within the time so it can provide information
    about how the content changed in different versions of the document.

    Warning:
    Application developers should generally not need to instantiate this class
    directly. The only correct way how to get this object is through
    Monitor.get() method.

    Design pattern: Active Record

    Example of checking new version of document:
        >>> from rrslib.web.changemonitor import Monitor
        >>> monitor = Monitor(user_id="myuid")
        >>> monitor
        Monitor(conn=Connection('localhost', 27017), dbname='webarchive', uid='myuid')
        >>> resource = monitor.get("http://www.myusefulpage.com/index.html")
        >>> resource
        <MonitoredResource(url='http://www.myusefulpage.com/index.html', uid='myuid') at 0xb7398accL>
        >>> resource.check()
        True
        >>> # the resource has changed

    Checking availability of the document on the URL
        >>> from rrslib.web.changemonitor import HTTPDateTime
        >>> resource = monitor.get("http://www.nonexistentpage.com")
        >>> resource.available()
        False
        >>> resource = monitor.get("http://www.myusefulpage.com/index.html")
        >>> resource.available(HTTPDateTime(2012, 6, 30, 15, 34))
        True

    Example of getting last available version of the document on the URL
        >>> resource = monitor.get("http://www.myusefulpage.com/index.html")
        >>> content = resource.get_last_version()
        >>> print content.data
        <html><head>
        ...
        >>> resource = monitor.get("http://www.crazynonexistentpage.com")
        >>> content = resource.get_last_version()
        Traceback (most recent call last):
          File "<stdin>", line 1, in <module>
        DocumentNotAvailable: The content of this URL is not available and it
        is not in the storage.

    Example of getting version of the document in exact time
        >>> resource = monitor.get("http://www.myusefulpage.com/index.html")
        >>> content = resource.get_version(HTTPDateTime(2012, 6, 30, 15, 34))

    Getting the last time when the document was checked:
        >>> resource = monitor.get("http://www.crazynotexistentpage.com")
        >>> resource.last_checked()
        HTTPDateTime(Thu, 01 Jan 1970 00:00:00 GMT)
    """
    def __init__(self, url, uid, storage):
        """
        @param url: monitored URL
        @type url:  basestring (str or unicode)
        @param uid: user identifier. This ID has to be unique for each one,
                    who is using changemonitor.
        @type uid: str
        @param storage: storage of monitored-resource data
        @type storage: model.Storage
        """
        # resource data
        self.url = url
        self.uid = uid

        # models
        self.storage = storage
        self.headers = storage._headermeta

        # resolver
        self.resolver = Resolver(storage)
        self._checked = False
        # file
        try:
            self.file = self.storage.get(url)
        except DocumentNotAvailable:
            # if the file is not in the storage, resolver has to check
            # the url and load actual data into the storage
            self.resolver.resolve(url)
            self._checked = True
            try:
                self.file = self.storage.get(url)
            except DocumentNotAvailable:
                raise DocumentNotAvailable("Resource '%s' is not available." % url)


    def check(self, force=False):
        """
        Check the resource URL and load the most recent version into database.

        TODO: consider using @lazy decorator. Most of use cases use this method
        so we have to insure that it will be called only once.
        
        @param force: force use of resolver (no effect on first call), 
            if force=False, doesn't try to download new content if called more than once
        @type force: Bool

        @raises: DocumentTooLargeException
        @raises: DocumentNotAvailable
        @returns: True if the document has changed since last check.
        """
        # bude vyuzivat resolveru pro checknuti URL a ziskani informace o tom,
        # jestli byl dokument zmenen. Mozna bude take dobre nahrat rovnou do
        # self.file nejnovejsi verzi, ale o tom je potreba jeste pouvazovat.

        # urcite je potreba pred kazdym checkem refreshout file cache
        self.file.refresh_cache()

        if force: 
            self._checked = False

        if not self._checked: # use resolver to get most recent version if not yet checked
            self.resolver.resolve(self.url)
            self._checked = True
            try:
                self.file = self.storage.get(self.url)
            except DocumentNotAvailable:
                raise DocumentNotAvailable("Resource '%s' is not available." % self.url)
        
        # and determine return value
        try:
        # time of last check
            _now = self.headers.get_by_version(self.url,-1,True)['timestamp'] 
            _now = HTTPDateTime().from_timestamp(_now+1) 
            # header and content are saved at the same time
            # we need to find content inserted before _now, that's why (_now+1)
        except Exception:
            raise DocumentNotAvailable("Check failed. Cannot get header information of '%s'." % self.url)
        try:
        # time of previous check
            _prev = self.headers.get_by_version(self.url,-2,True)
            _prev = HTTPDateTime().from_timestamp(_prev['timestamp']+1)
        except TypeError: # this is first time document is checked
            if self._checked: return False  # if already checked, and have no v=-2 header, there was no change
            else:             return True   # this is the first-time check

        #DEBUG
#?        print "header time: _now: ",_now,"\nheader time: _prev: ",_prev,"\n"

        d = self.get_diff(_prev,_now)
        
        if d is None: return False
        if isinstance(d, basestring): # PlainTextDiff, maybe htmldiff in some cases
            if len(d)==0: return False
            else: return True
        if isinstance(d, dict): # BinaryDiff     
            if string.find(d['metainfo'],"(patch data)")==-1 : 
            # TODO: find what indicates that BinaryDiff-ed file hasn't changed
            # current version seems to work, but needs more testing               
                return False
            else: return True
        try: # d is htmldiff output (generator object)
            chunk = d.next()
            if len(chunk.added)==0 and len(chunk.removed)==0:
                return False
        except (StopIteration, TypeError): # if can't get d.next(), then d is probably empty -> no change
            return False
        
        return True
        
    def get_last_version(self):
        """
        Get last available content of the document. If the document is available
        at this time, returns most recent version which is on the web server.

        @returns: Last available content of this resource.
        @rtype: Content
        @raises: DocumentNotAvailable if no content available (resource does not
        exist on the URL and never existed within the known history)
        """
        self.resolver.resolve(self.url)
        self._checked = True
        try:
            self.file = self.storage.get(self.url)
            return self.file.get_last_version()
        except NoFile: # FIXME tady to prece nemuze byt??!
            raise DocumentNotAvailable("Resource '%s' is not available." % self.url)


    def get_version(self, time_or_version):
        """
        Get content of this document in specified time or version. If the
        document was not available in given time, returns last available content.
        If there is no available content until given time, raises exception.

        @param time_or_version: Time or version of the content we want to retrieve.
            Version numbering is a convenience atop the GridFS API provided
            by MongoDB. version ``-1`` will be the most recently uploaded
            matching file, ``-2`` the second most recently uploaded, etc.
            Version ``0`` will be the first version
            uploaded, ``1`` the second version, etc. So if three versions
            have been uploaded, then version ``0`` is the same as version
            ``-3``, version ``1`` is the same as version ``-2``, and
            version ``2`` is the same as version ``-1``.
        @type time_or_version: HTTPDateTime or int
        @raises: DocumentHistoryNotAvailable if there is no available content until
        given time or version
        """
        if isinstance(time_or_version, HTTPDateTime):
            return self.file.get_version(time_or_version.to_timestamp())
        elif isinstance(time_or_version, int):
            return self.file.get_version(time_or_version)
        else:
            raise TypeError("Version time has to be type HTTPDateTime or GridFS version (int).")


    def get_diff(self, start, end):
        """
        @param start: start time or version to be diffed
        @type start: HTTPDateTime or int
        @param end: end time or version to be diffed
        @type end: HTTPDateTime or int
        @returns: either textual or binary diff of the file (if available).
                  If contents are equal (document did not change within this
                  time range) returns None.
        @rtype: unicode
        @raises: DocumentHistoryNotAvaliable if the storage doesn't provide
                 enough data for computing the diff.
        """
        content_start = self.get_version(start)
        content_end = self.get_version(end)
        if content_start == content_end:
            return None
        return content_start.diff_to(content_end)


    def available(self, httptime=None):
        if (not isinstance(httptime, HTTPDateTime)) and (httptime is not None):
            raise TypeError("Time of availability has to be type HTTPDateTime.")
        # Pokud je httptime=None, pak se jedna o dostupnost v tomto okamziku
        if (httptime is None):
            try:
                self.check(force=True)
            except DocumentNotAvailable:
                return False
            return True
        else:
            # when was last checked before 'httptime'
            h = self.headers.get_by_time(self.url,httptime.to_timestamp())
            if h is None: return False # not checked before time 'httptime'
            t1 = h['timestamp']
            try:
                t2 = self.get_version(HTTPDateTime().from_timestamp(t1)).upload_date
            except DocumentHistoryNotAvaliable:
                return False
            return True


    def last_checked(self):
        """
        Get information about the time of last check of this resource.

        @returns: time of last check or None if the resource was never checked
                  (or the HTTP requests timed out)
        @rtype: HTTPDateTime or None
        """
        return self.headers.last_checked(self.url)


    def __repr__(self):
        return "<MonitoredResource(url='%s', uid='%s') at %s>" % \
            (self.url, self.uid, hex(id(self)))

    __str__ = __repr__
Esempio n. 37
0
def play(url):
    from resolver import Resolver
    r = Resolver(url)
    items.play_item(r.resolved_url, r.startpercent)
Esempio n. 38
0
class Downloader(object):
    def __init__(self,
                 base="http://repo1.maven.org/maven2",
                 username=None,
                 password=None):
        self.requestor = Requestor(username, password)
        self.resolver = Resolver(base, self.requestor)

    def download(self, artifact, filename=None, suppress_log=False):
        filename = artifact.get_filename(filename)
        url = self.resolver.uri_for_artifact(artifact)
        if not self.verify_md5(filename, url + ".md5"):
            if not suppress_log:
                print("Downloading artifact " + str(artifact))
                hook = self._chunk_report
            else:
                hook = self._chunk_report_suppress

            onError = lambda uri, err: self._throwDownloadFailed(
                "Failed to download artifact " + str(artifact) + "from " + uri)
            response = self.requestor.request(url, onError, lambda r: r)

            if response:
                with open(filename, 'w') as f:
                    self._write_chunks(response, f, report_hook=hook)
                if not suppress_log:
                    print("Downloaded artifact %s to %s" %
                          (artifact, filename))
                return (artifact, True)
            else:
                return (artifact, False)
        else:
            if not suppress_log:
                print("%s is already up to date" % artifact)
            return (artifact, True)

    def _throwDownloadFailed(self, msg):
        raise RequestException(msg)

    def _chunk_report_suppress(self, bytes_so_far, chunk_size, total_size):
        pass

    def _chunk_report(self, bytes_so_far, chunk_size, total_size):
        percent = float(bytes_so_far) / total_size
        percent = round(percent * 100, 2)
        sys.stdout.write("Downloaded %d of %d bytes (%0.2f%%)\r" %
                         (bytes_so_far, total_size, percent))

        if bytes_so_far >= total_size:
            sys.stdout.write('\n')

    def _write_chunks(self, response, file, chunk_size=8192, report_hook=None):
        total_size = response.info().getheader('Content-Length').strip()
        total_size = int(total_size)
        bytes_so_far = 0

        while 1:
            chunk = response.read(chunk_size)
            bytes_so_far += len(chunk)

            if not chunk:
                break

            file.write(chunk)
            if report_hook:
                report_hook(bytes_so_far, chunk_size, total_size)

        return bytes_so_far

    def verify_md5(self, file, remote_md5):
        if not os.path.exists(file):
            return False
        else:
            local_md5 = self._local_md5(file)
            onError = lambda uri, err: self._throwDownloadFailed(
                "Failed to download MD5 from " + uri)
            remote = self.requestor.request(remote_md5, onError,
                                            lambda r: r.read())
            return local_md5 == remote

    def _local_md5(self, file):
        md5 = hashlib.md5()
        with open(file, 'rb') as f:
            for chunk in iter(lambda: f.read(8192), ''):
                md5.update(chunk)
        return md5.hexdigest()
Esempio n. 39
0
 def __init__(self,
              base="http://repo1.maven.org/maven2",
              username=None,
              password=None):
     self.requestor = Requestor(username, password)
     self.resolver = Resolver(base, self.requestor)
Esempio n. 40
0
from resolver import Resolver

r = Resolver()
r.import_file()
print(r.df.info())
print(r.build())
Esempio n. 41
0
 def __init__(self, base="http://repo1.maven.org/maven2", username=None, password=None):
     self.requestor = Requestor(username, password)
     self.resolver = Resolver(base, self.requestor)
Esempio n. 42
0
 def resolve_view_var(self, node):
     try:
         return Resolver.resolve(node)
     except AttributeError:
         raise