def _cfn_param_updater(self, param, comparison_parameters,
stage_parameters, file_name): # pylint: disable=R0912
"""
Generic CFN Updater method
"""
resolver = Resolver(self.parameter_store, stage_parameters,
comparison_parameters)
for key, value in comparison_parameters[param].items():
if str(value).startswith('resolve:'):
if resolver.fetch_parameter_store_value(value, key, param):
continue
if str(value).startswith('import:'):
if resolver.fetch_stack_output(value, key, param):
continue
if str(value).startswith('upload:'):
if resolver.upload(value, key, file_name, param):
continue
resolver.update_cfn(key, param)
for key, value in stage_parameters[param].items():
if str(value).startswith('resolve:'):
if resolver.fetch_parameter_store_value(value, key, param):
continue
if str(value).startswith('import:'):
if resolver.fetch_stack_output(value, key, param):
continue
if str(value).startswith('upload:'):
if resolver.upload(value, key, file_name, param):
continue
return resolver.__dict__.get('stage_parameters')
class TestResolver(unittest.TestCase):
"""
Test class for Resolver
"""
def setUp(self):
self.resolver = Resolver()
def testARecords(self):
reference_a_record = [(1, '31.170.165.34')]
test_a_record = self.resolver.resolve('eur.al')
self.assertEqual(reference_a_record, test_a_record)
def testNoResponse(self):
reference_no_response = Resolver.NO_RESPONSE
self.resolver.port = 65000
test = self.resolver.resolve('eur.al')
self.assertEqual(reference_no_response, test)
self.resolver.port = 53
def testNotFound(self):
reference = Resolver.NAME_NOT_FOUND
test = self.resolver.resolve('opsidjgsdkjf.paris')
self.assertEqual(reference, test)
def testCNameRecord(self):
reference = 5, 'dijkstra.urgu.org'
test = self.resolver.resolve('anytask.urgu.org')
self.assertEqual(reference, test[0])
def testRecursion(self):
self.resolver = Resolver('198.41.0.4')
reference_a_record = [(1, '31.170.165.34')]
test_a_record = self.resolver.resolve('eur.al')
self.assertEqual(reference_a_record, test_a_record)
def __init__(self, url, uid, storage):
"""
@param url: monitored URL
@type url: basestring (str or unicode)
@param uid: user identifier. This ID has to be unique for each one,
who is using changemonitor.
@type uid: str
@param storage: storage of monitored-resource data
@type storage: model.Storage
"""
# resource data
self.url = url
self.uid = uid
# models
self.storage = storage
self.headers = storage._headermeta
# resolver
self.resolver = Resolver(storage)
# file
try:
self.file = self.storage.get(url)
except DocumentNotAvailable:
# if the file is not in the storage, resolver has to check
# the url and load actual data into the storage
self.resolver.resolve(url)
try:
self.file = self.storage.get(url)
except DocumentNotAvailable:
raise DocumentNotAvailable("Resource '%s' is not available." % url)
def __init__(self,
conf=Config.default,
user=None,
host=None,
reactor=reactor):
"""Construct a new client context
Prepare a new context with the given config.
If user and host are None then they are collected
from the environment.
"""
self.conf, self.reactor = conf, reactor
self.circuits = CACircuitFactory(self)
self.resolv = Resolver(self.circuits, conf)
self._onClose = DeferredManager()
if host is None:
from socket import gethostname
host = gethostname()
self.host = host
if user is None:
from getpass import getuser
user = getuser()
self.user = user
reactor.addSystemEventTrigger("before", "shutdown", self.close)
def _sc_param_updater(self, comparison_parameters, stage_parameters,
file_name): # pylint: disable=R0912
"""
Compares parameter files used for the Service Catalog deployment type
"""
resolver = Resolver(self.parameter_store, stage_parameters,
comparison_parameters)
for key, value in comparison_parameters.items():
if str(value).startswith('resolve:'):
if resolver.fetch_parameter_store_value(value, key):
continue
if str(value).startswith('import:'):
if resolver.fetch_stack_output(value, key):
continue
if str(value).startswith('upload:'):
if resolver.upload(value, key, file_name):
continue
resolver.update_sc(key)
for key, value in stage_parameters.items():
if str(value).startswith('resolve:'):
if resolver.fetch_parameter_store_value(value, key):
continue
if str(value).startswith('import:'):
if resolver.fetch_stack_output(value, key):
continue
if str(value).startswith('upload:'):
if resolver.upload(value, key, file_name):
continue
return resolver.__dict__.get('stage_parameters')
def run(self):
""" Run the handler thread """
recv_header = self.received_message.header
try:
questions = self.received_message.questions[0]
qd_count = len(questions)
except IndexError:
return
# if recv_header.rd:
try:
resolver = Resolver(True)
answer_sname, addresses, aliasses = resolver.gethostbyname(questions[0].rdata.data)
except ResolverException as e:
print e
return
answers = list()
for address in addresses:
answer = message.Section(answer_sname, Type.A, Class.IN, 86400, len(address), address)
answers.append(answer)
send_header = message.Header(recv_header.ident, 0, qd_count, len(addresses), 0, 0)
send_header.qr = 1
send_header.opcode = 0
send_header.rd = recv_header.rd
response_message = message.Message(send_header, questions, answers)
response_bytes = response_message.to_bytes()
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
sock.settimeout(3)
sock.send(response_bytes, (self.ip_address, 53))
def find_serializer_fields(self, serializer_name):
nodes = self.serializer_registry.nodes
if serializer_name in self.memo_dict:
return self.memo_dict[serializer_name]
class_node = nodes[serializer_name]
fields = Fields()
init_node = None
# Look at own class variables first, this trumps everything else
for node in class_node.body:
if self.is_class_var(node):
# explicit class var trumps Meta
fields.add(Resolver.drf_field_assignment(node), overwrite=True)
elif self.is_meta(node):
fields.extend(Resolver.drf_meta_fields(node))
elif self.is_init_method(node):
init_node = node
# add fields from bases, in left to right order. The bases of the base
# trumps the neighbour of the base if there's overlap.
for base in class_node.bases:
base = Resolver.resolve(base)
if base == 'object':
continue
if base not in nodes:
# TODO: ???
continue
base_class_vars = self.find_serializer_fields(base)
fields.extend(base_class_vars)
# Check for dynamic fields that were inherited from direct ancestors.
# TODO: Find a better way to support inheritance
parent_in_dynamic_fields = any(
getattr(parent_class, 'attr', None) in self.dynamic_fields
or getattr(parent_class, 'id', None) in self.dynamic_fields
for parent_class in class_node.bases)
# dynamic fields trump or augment existing fields
if serializer_name in self.dynamic_fields or parent_in_dynamic_fields:
if init_node:
dynamic_fields = Resolver.init_method(init_node)
for field_name, field in dynamic_fields.iteritems():
if field_name not in fields:
fields.add(field)
continue
previous_field = fields[field_name]
augmented_field = self.augment_field(previous_field, field)
fields.add(augmented_field, overwrite=True)
self.memo_dict[serializer_name] = fields
return fields
def main():
parser = create_parser()
parsed_args = parser.parse_args(sys.argv[1:])
dns_addr = parsed_args.start_server
resolver = Resolver(dns_addr)
try:
resolver.start_listening()
except socket.error:
sys.exit()
class Recognizer:
def __init__(self):
self.resolver = Resolver()
self.thread = Thread(target=self.run, args=()).start()
def run(self):
faces = []
while True:
try:
frame = Store.movement_frames.get(True, 2)
face_locations = face_recognition.face_locations(frame)
face_encodings = face_recognition.face_encodings(
frame, face_locations)
for (top, right, bottom,
left), encoding in zip(face_locations, face_encodings):
cropped_img = frame[top - 10:bottom + 10,
left - 10:right + 10]
faces.append({
"img": frame[top:bottom, left:right],
"encoding": encoding
})
except queue.Empty:
if len(faces) > 0:
self.resolver.resolve(self.preprocess_faces(faces).copy())
faces = []
def preprocess_faces(self, faces):
# Cluster the faces with chinese whispers
encodings = [dlib.vector(face['encoding']) for face in faces]
labels = dlib.chinese_whispers_clustering(encodings, 0.5)
selected_faces = []
# Select face most close to average group
groups = list(set(labels))
for group in groups:
# Get indices for each group
indices = [i for i in range(len(labels)) if labels[i] == group]
group_encodings = [faces[i]['encoding'] for i in indices]
# Get centroid for group encodings
avg_group_encoding = np.average(group_encodings, axis=0)
# Get the closest face to the centroid
avg_distance = face_recognition.face_distance(
group_encodings, avg_group_encoding)
min_index = np.argmin(avg_distance)
face_index = indices[min_index]
selected_faces.append(faces[face_index])
return selected_faces
def find_serializer_fields(self, serializer_name):
nodes = self.serializer_registry.nodes
if serializer_name in self.memo_dict:
return self.memo_dict[serializer_name]
class_node = nodes[serializer_name]
fields = Fields()
init_node = None
# Look at own class variables first, this trumps everything else
for node in class_node.body:
if self.is_class_var(node):
# explicit class var trumps Meta
fields.add(Resolver.class_var_drf_field(node), overwrite=True)
elif self.is_meta(node):
fields.extend(Resolver.drf_meta_fields(node))
elif self.is_init_method(node):
init_node = node
# add fields from bases, in left to right order. The bases of the base
# trumps the neighbour of the base if there's overlap.
for base in class_node.bases:
base = Resolver.resolve(base)
if base == 'object':
continue
if base not in nodes:
# TODO: ???
continue
base_class_vars = self.find_serializer_fields(base)
fields.extend(base_class_vars)
# dynamic fields trump or augment existing fields
if serializer_name in self.dynamic_fields:
if not init_node:
msg = ('Did not find __init__ in {} but view specifies dynamic'
' fields.').format(serializer_name)
raise Exception(msg)
dynamic_fields = Resolver.init_method(node)
for field in dynamic_fields:
if field not in fields:
fields.add(field)
continue
previous_field = fields[field['field_name']]
augmented_field = self.augment_field(previous_field, field)
fields.add(augmented_field, overwrite=True)
self.memo_dict[serializer_name] = fields
return fields
def test_do_simple(self):
""" Simple config resolution """
resolver = Resolver({
'foo': {
'module': 'example_classes',
'class': 'Foo'
}
})
services = resolver.do()
self.assertTrue('foo' in services)
assert isinstance(services['foo'], Foo)
def test_finds_source_from_js_component_folder(self):
file = "/test/js/components/FileRenamePrompt-test.js"
r = Resolver().get_source(file, spec_folder="test")
self.assertEqual(len(r), 2)
self.assertEqual(r[0],
"/app/javascript/components/FileRenamePrompt.vue")
self.assertEqual(r[1], "/javascript/components/FileRenamePrompt.vue")
def dynamic_fields(self):
if not self._dynamic_field_map:
nodes = self.view_registry.nodes
for class_name, class_node in nodes.iteritems():
props = {
'include_filters': None,
'expand_filters': None,
'exclude_filters': None,
'serializer_class': None
}
for node in class_node.body:
if not self.is_class_var(node):
continue
lhs, rhs = Resolver.resolve(node)
if lhs in props:
props[lhs] = self.resolve_view_var(rhs)
serializer_name = props.pop('serializer_class')
truncated_props = {
key: val
for key, val in props.iteritems() if val
}
if truncated_props:
self._dynamic_field_map[serializer_name] = truncated_props
return self._dynamic_field_map
def test_finds_source_from_erb(self):
file = "/spec/views/namespace/users/_something.html.erb_spec.rb"
r = Resolver().get_source(file)
self.assertEqual(len(r), 2)
self.assertEqual(r[0],
"/app/views/namespace/users/_something.html.erb")
self.assertEqual(r[1], "/views/namespace/users/_something.html.erb")
def fmt_serializer(node, fields):
output = ('{}({})\n' '{}\n')
table_data = tabulate(fields, headers="keys", tablefmt='grid')
return output.format(
node.name, ', '.join(Resolver.resolve(base) for base in node.bases),
table_data)
def run(line: str):
scanner: Scanner = Scanner(line, error_scan)
tokens: List[Token] = scanner.scanTokens()
parser: Parser = Parser(tokens, error_parse)
statements: List[stmt.Stmt] = parser.parse()
# // Stop if there was a syntax error.
if hadError:
return
interpreter: Interpreter = Interpreter(runtimeError)
resolver: Resolver = Resolver(interpreter, error_scan)
resolver.resolve(statements)
# // Stop if there was a resolution error.
if hadError:
return
if os.environ.get("DEBUG"):
print("**TOKENS**")
for t in tokens:
print(t)
print("**STATEMENTS**")
for s in statements:
print(s)
ipdb.set_trace()
interpreter.interpret(statements)
def _param_updater(self, comparison_parameters, stage_parameters):
"""
Generic Parameter Updater method
"""
resolver = Resolver(self.parameter_store, stage_parameters, comparison_parameters)
self._determine_parameter_structure(comparison_parameters, resolver)
self._determine_parameter_structure(stage_parameters, resolver)
return resolver.__dict__.get('stage_parameters')
def test_advanced(self):
# datalogger:
# module: datalogger
# class: DataLogger
# args:
# -
# name: rob_logger
# path: /var/log/monorail
# file_base_name: rob
# max_file_bytes: 1073741824
# backup_count: 10
# port: $_port
config = {
'logger': {
'module': 'example_classes',
'class': 'TestLogger',
'args': [
{
'name': 'test_logger',
'path': '/var/foo',
'file_base_name': 'testlog',
'max_file_bytes': 500,
'backup_count': 10,
'port': '$_port'
}
]
}
}
scalars = {'_port': 555}
resolver = Resolver(config, scalars)
services = resolver.do()
logger = services['logger']
assert isinstance(logger, TestLogger)
self.assertEquals(
{
'name': 'test_logger',
'path': '/var/foo',
'file_base_name': 'testlog',
'max_file_bytes': 500,
'backup_count': 10,
'port': 555
},
logger.config
)
def run(self, source):
scanner = Scanner(source, self)
tokens = scanner.scan_tokens()
parser = Parser(self, tokens)
statements = parser.parse()
if self.hadError:
return
if self.had_runtime_error:
return
resolver = Resolver(self.interpreter, self)
resolver.resolve(statements)
if self.hadError:
return
self.interpreter.interpret(statements)
def resolve_file(filename):
parser = Parser()
try:
with open(filename, 'r') as file:
for line in file:
line = re.sub('\s', '', line)
line_is_ignored = ignored_line.match(line)
if line_is_ignored is not None:
continue
try:
parser.handle_line(line)
except ParseError as exception:
print(
colored(
'Parse error of type : {} detected on line\n\t --> {}'
.format(exception.exception_type, line), 'red'))
raise FileError()
if not parser.queries:
print(colored('No queries furnished', 'red'))
raise FileError
if args.verbose:
print(parser)
resolver = Resolver(parser.facts, parser.rules, args.verbose)
for query in parser.queries:
try:
if args.verbose:
print(
colored('Query to solve : {}'.format(query), 'yellow'))
response = resolver.resolve_query(query)
print(
colored(
'query {} has been resolved to : {}'.format(
query, response), 'green'))
except ContradictionError as exception:
print(
colored(
'Contradiction detected on {0} that is {1} while his opposite is also {1}'
.format(exception.name, exception.value), 'red'))
except (FileNotFoundError, NameError, PermissionError, IsADirectoryError,
UnicodeDecodeError) as error:
print(colored(error, 'red'))
raise FileError()
def run(self, source: str):
scanner = Scanner(self, source)
tokens = scanner.scan_tokens()
if DEBUG:
print(tokens)
parser = Parser(self, tokens)
statements = parser.parse()
if self.had_error:
return
resolver = Resolver(self, self.interpreter)
resolver.resolve_statements(statements)
if self.had_error:
return
self.interpreter.interpret(statements)
def cli():
path = input("Введите путь до изображения:\n")
image_analyzer = ImageAnalyzer()
image_analyzer.read_image(path)
print("{} прочитано".format(path))
image_analyzer.process_xo()
main_map = image_analyzer.get_map()
print("Расчет анализатора:\n {}".format(main_map))
print("0 - пустота, 1 - Х, 2 - О")
resolver = Resolver(main_map)
res_points = resolver.resolve()
print("Ответ алгоритма: {}".format(res_points))
if image_analyzer.process_output(res_points):
print("Результат в файле output.pnt")
print("Done")
def run_server(ip='0.0.0.0'):
try:
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
s.bind((ip, 53))
except socket.error as e:
logging.error(e)
return
while True:
data, addr = s.recvfrom(1024)
ret = Resolver(data).do()
s.sendto(ret, addr)
def main():
parser = argparse.ArgumentParser(description='YOBAdns-resolver')
parser.add_argument(
"address", metavar="Address", nargs="+", type=str,
help='Address that you need to resolve')
parser.add_argument(
"--server", "-s", metavar="Server", nargs="?", type=str,
default=DEFAULT_SERVER,
help='Set the DNS server to first request, default: 8.8.8.8'
)
parser.add_argument(
"--port", "-p", metavar="Port", nargs="?", type=int,
default=DEFAULT_PORT,
help='Set the port of dedicated DNS server'
)
parser.add_argument("-d", "--debug", action="store_true",
help="switch on debug mode")
parser.add_argument("-n", "--num", nargs="?", type=int,
default=DEFAULT_NUM_OF_RETRIES,
help="number of retries")
parser.add_argument("-w", "--waiting", nargs="?", type=int,
default=DEFAULT_TIMEOUT,
help="waiting time of request")
args = parser.parse_args()
##############################################
resolver = Resolver(args.server, args.debug,
args.port, args.num, args.waiting)
received = resolver.resolve(args.address[0])
if received == resolver.NO_RESPONSE:
print('\tNo response')
else:
if received == resolver.NAME_NOT_FOUND:
print('\t\tName {} does not exist'.format(args.address[0]))
else:
print('\tDomain: ' + args.address[0])
print('\tResponses:')
for response in received:
print('\t\t{}\t{}'.format(resolver.TYPES[response[0]],
response[1]))
def test_do_complicated(self):
""" Instantiate services from advanced YAML file """
config = yaml.load(open('test/test_config.yml', 'r')) or {}
resolver = Resolver(config)
services = resolver.do()
assert isinstance(services['foo'], Foo)
assert isinstance(services['bar'], Bar)
assert isinstance(services['baz'], Baz)
assert isinstance(services['qux'], Qux)
assert isinstance(services['wobble'], Wobble)
assert isinstance(services['spam'], Spam)
spam = services['spam']
self.assertEquals(spam.ham, 'ham!')
self.assertEquals(spam.eggs, 'eggz')
wobble = services['wobble']
assert isinstance(wobble.foo, Foo)
assert isinstance(wobble.bar, Bar)
assert isinstance(wobble.baz, Baz)
assert isinstance(wobble.spam, Spam)
def __init__(self, handlers=None):
if not handlers:
handlers = []
AsyncHTTPClient.configure(
"tornado.curl_httpclient.CurlAsyncHTTPClient",
max_clients=CURL['MAX_CLIENTS'],
)
self.cache = RedisCache(**REDIS['CLIENT'])
self.cache.initialize(**REDIS['TORNADO_CLIENT'])
tornado.ioloop.IOLoop.instance().add_future(
self.cache.run(), lambda future: future.result())
Api = get_db_api(is_cached=True)
self.db_api = Api(
DB_API['HOST'],
DB_API['PORT'],
http_client=AsyncHTTPClient(),
resolver=Resolver(ttl=DNS_RECORD_TTL),
cache=self.cache,
cache_key_ttl=DB_API['CACHE_KEY_TTL'],
connect_timeout=CURL['CONNECT_TIMEOUT'],
request_timeout=CURL['REQUEST_TIMEOUT'],
)
self.jinja = Environment(loader=FileSystemLoader(
JINJA['TEMPLATE_ROOT']),
**JINJA['SETTINGS'])
handlers += [
(r'/db/(?P<api_path>.*)', DBApiRequestHandler),
]
if DEBUG:
handlers += [
(r'/test/', TestView),
]
config = dict(debug=DEBUG, )
tornado.web.Application.__init__(self, handlers, **config)
def test_resolver(self):
from resolver import Resolver
import pytis.presentation
r = Resolver(search=('pytis', 'pytis.defs.profiles'))
view = r.get('cms.Languages', 'view_spec')
self.assertIsInstance(view, pytis.presentation.ViewSpec)
spec = r.specification('cms.Languages')
self.assertIsInstance(spec, pytis.presentation.Specification)
# Test top level specification name (from pytis.defs.profiles).
spec2 = r.specification('FormProfiles')
self.assertIsInstance(spec2, pytis.presentation.Specification)
specifications = [spec for name, spec in r.walk()]
from pytis.cms import Languages
self.assertIn(Languages, specifications)
class MonitoredResource(object):
"""
Monitored resource (URL). The ressource is generally any document in any
format, but most often it will be HTML code.
This class wraps the URL content and metadata.
The contents can be manipulated within the time so it can provide information
about how the content changed in different versions of the document.
Warning:
Application developers should generally not need to instantiate this class
directly. The only correct way how to get this object is through
Monitor.get() method.
Design pattern: Active Record
Example of checking new version of document:
>>> from rrslib.web.changemonitor import Monitor
>>> monitor = Monitor(user_id="myuid")
>>> monitor
Monitor(conn=Connection('localhost', 27017), dbname='webarchive', uid='myuid')
>>> resource = monitor.get("http://www.myusefulpage.com/index.html")
>>> resource
<MonitoredResource(url='http://www.myusefulpage.com/index.html', uid='myuid') at 0xb7398accL>
>>> resource.check()
True
>>> # the resource has changed
Checking availability of the document on the URL
>>> from rrslib.web.changemonitor import HTTPDateTime
>>> resource = monitor.get("http://www.nonexistentpage.com")
>>> resource.available()
False
>>> resource = monitor.get("http://www.myusefulpage.com/index.html")
>>> resource.available(HTTPDateTime(2012, 6, 30, 15, 34))
True
Example of getting last available version of the document on the URL
>>> resource = monitor.get("http://www.myusefulpage.com/index.html")
>>> content = resource.get_last_version()
>>> print content.data
<html><head>
...
>>> resource = monitor.get("http://www.crazynonexistentpage.com")
>>> content = resource.get_last_version()
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
DocumentNotAvailable: The content of this URL is not available and it
is not in the storage.
Example of getting version of the document in exact time
>>> resource = monitor.get("http://www.myusefulpage.com/index.html")
>>> content = resource.get_version(HTTPDateTime(2012, 6, 30, 15, 34))
Getting the last time when the document was checked:
>>> resource = monitor.get("http://www.crazynotexistentpage.com")
>>> resource.last_checked()
HTTPDateTime(Thu, 01 Jan 1970 00:00:00 GMT)
"""
def __init__(self, url, uid, storage):
"""
@param url: monitored URL
@type url: basestring (str or unicode)
@param uid: user identifier. This ID has to be unique for each one,
who is using changemonitor.
@type uid: str
@param storage: storage of monitored-resource data
@type storage: model.Storage
"""
# resource data
self.url = url
self.uid = uid
# models
self.storage = storage
self.headers = storage._headermeta
# resolver
self.resolver = Resolver(storage)
# file
try:
self.file = self.storage.get(url)
except DocumentNotAvailable:
# if the file is not in the storage, resolver has to check
# the url and load actual data into the storage
self.resolver.resolve(url)
try:
self.file = self.storage.get(url)
except DocumentNotAvailable:
raise DocumentNotAvailable("Resource '%s' is not available." % url)
def check(self):
"""
Check the resource URL and load the most recent version into database.
TODO: consider using @lazy decorator. Most of use cases use this method
so we have to insure that it will be called only once.
@raises: DocumentTooLargeException
@returns: True if the document has changed since last check.
"""
# bude vyuzivat resolveru pro checknuti URL a ziskani informace o tom,
# jestli byl dokument zmenen. Mozna bude take dobre nahrat rovnou do
# self.file nejnovejsi verzi, ale o tom je potreba jeste pouvazovat.
# urcite je potreba pred kazdym checkem refreshout file cache
# self.file.refresh_cache()
#self.resolver.resolve(self.url)
raise NotImplementedError()
def get_last_version(self):
"""
Get last available content of the document. If the document is available
at this time, returns most recent version which is on the web server.
@returns: Last available content of this resource.
@rtype: Content
@raises: DocumentNotAvailable if no content available (resource does not
exist on the URL and never existed within the known history)
"""
self.resolver.resolve(self.url)
try:
return self.file.get_last_version()
except NoFile: # FIXME tady to prece nemuze byt??!
raise DocumentNotAvailable("Resource is not available.")
def get_version(self, time_or_version):
"""
Get content of this document in specified time or version. If the
document was not available in given time, returns last available content.
If there is no available content until given time, raises exception.
@param time_or_version: Time or version of the content we want to retrieve.
Version numbering is a convenience atop the GridFS API provided
by MongoDB. version ``-1`` will be the most recently uploaded
matching file, ``-2`` the second most recently uploaded, etc.
Version ``0`` will be the first version
uploaded, ``1`` the second version, etc. So if three versions
have been uploaded, then version ``0`` is the same as version
``-3``, version ``1`` is the same as version ``-2``, and
version ``2`` is the same as version ``-1``.
@type time_or_version: HTTPDateTime or int
@raises: DocumentHistoryNotAvailable if there is no available content until
given time or version
"""
if isinstance(time_or_version, HTTPDateTime):
return self.file.get_version(time_or_version.to_timestamp())
elif isinstance(time_or_version, int):
return self.file.get_version(time_or_version)
else:
raise TypeError("Version time has to be type HTTPDateTime or GridFS version (int).")
def get_diff(self, start, end):
"""
@param start: start time or version to be diffed
@type start: HTTPDateTime or int
@param end: end time or version to be diffed
@type end: HTTPDateTime or int
@returns: either textual or binary diff of the file (if available).
If contents are equal (document did not change within this
time range) returns None.
@rtype: unicode
@raises: DocumentHistoryNotAvaliable if the storage doesn't provide
enough data for computing the diff.
"""
content_start = self.get_version(start)
content_end = self.get_version(end)
if content_start == content_end:
return None
return content_start.diff_to(content_end)
def available(self, httptime=None):
if not isinstance(httptime, HTTPDateTime):
raise TypeError("Time of availability has to be type HTTPDateTime.")
# Pokud je httptime=None, pak se jedna o dostupnost v tomto okamziku
raise NotImplementedError()
def last_checked(self):
"""
Get information about the time of last check of this resource.
@returns: time of last check or None if the resource was never checked
(or the HTTP requests timed out)
@rtype: HTTPDateTime or None
"""
return self.headers.last_checked(self.url)
def __repr__(self):
return "<MonitoredResource(url='%s', uid='%s') at %s>" % \
(self.url, self.uid, hex(id(self)))
__str__ = __repr__
# 03_02-Callable Instances
from resolver import Resolver
resolve = Resolver()
resolve('localhost')
resolve.__call__('localhost')
resolve._cache
resolve('sixty-north.com')
resolve.__call__('sixty-north.com')
resolve._cache
resolve('pluralsight.com')
resolve._cache
from timeit import timeit
timeit(setup='from __main__ import resolve', stmt="resolve('localhost')", number=1)
timeit(setup='from __main__ import resolve', stmt="resolve('localhost')", number=1)
timeit(setup='from __main__ import resolve', stmt="resolve('python.org')", number=1)
timeit(setup='from __main__ import resolve', stmt="resolve('python.org')", number=1)
print("{:f}".format(_))
exit()
from resolver import Resolver
resolve = Resolver()
resolve.has_host('localhost')
resolve('localhost')
resolve.has_host('localhost')
resolve.clear()
resolve.has_host('localhost')
def step_three(self, ast):
resolver = Resolver(ast)
resolver.traverse()
class Downloader(object):
def __init__(self, base="http://repo1.maven.org/maven2", username=None, password=None):
self.requestor = Requestor(username, password)
self.resolver = Resolver(base, self.requestor)
def download(self, artifact, filename=None, suppress_log=False):
filename = artifact.get_filename(filename)
url = self.resolver.uri_for_artifact(artifact)
if not self.verify_md5(filename, url + ".md5"):
if not suppress_log:
print("Downloading artifact " + str(artifact))
hook=self._chunk_report
else:
hook=self._chunk_report_suppress
onError = lambda uri, err: _throwDownloadFailed("Failed to download artifact " + str(artifact) + "from " + uri)
response = self.requestor.request(url, onError, lambda r: r)
if response:
with open(filename, 'w') as f:
self._write_chunks(response, f, report_hook=hook)
if not suppress_log:
print("Downloaded artifact %s to %s" % (artifact, filename))
return (artifact, True)
else:
return (artifact, False)
else:
if not suppress_log:
print("%s is already up to date" % artifact)
return (artifact, True)
def _throwDownloadFailed(msg):
raise RequestException(msg)
def _chunk_report_suppress(self, bytes_so_far, chunk_size, total_size):
pass
def _chunk_report(self, bytes_so_far, chunk_size, total_size):
percent = float(bytes_so_far) / total_size
percent = round(percent*100, 2)
sys.stdout.write("Downloaded %d of %d bytes (%0.2f%%)\r" %
(bytes_so_far, total_size, percent))
if bytes_so_far >= total_size:
sys.stdout.write('\n')
def _write_chunks(self, response, file, chunk_size=8192, report_hook=None):
total_size = response.info().getheader('Content-Length').strip()
total_size = int(total_size)
bytes_so_far = 0
while 1:
chunk = response.read(chunk_size)
bytes_so_far += len(chunk)
if not chunk:
break
file.write(chunk)
if report_hook:
report_hook(bytes_so_far, chunk_size, total_size)
return bytes_so_far
def verify_md5(self, file, remote_md5):
if not os.path.exists(file):
return False
else:
local_md5 = self._local_md5(file)
onError = lambda uri, err: _throwDownloadFailed("Failed to download MD5 from " + uri)
remote = self.requestor.request(remote_md5, onError, lambda r: r.read())
return local_md5 == remote
def _local_md5(self, file):
md5 = hashlib.md5()
with open(file, 'rb') as f:
for chunk in iter(lambda: f.read(8192), ''):
md5.update(chunk)
return md5.hexdigest()
username = None
password = None
domain = None
base = None
def onLoginInfo(base, domain, username, password):
print base, domain, username, password
client_jid = jid.JID("%s@%s" % (username, domain))
secret = password
c = Client(client_jid, secret, True, base)
username = username
password = password
domain = domain
base = base
print 'Begin to connect'
resolv.getFriendsList()
def onListDownloaded(friendlist):
print friendlist
if(__name__ == "__main__"):
resolv = Resolver('kkszysiu', 'xxxxx')
resolv.getLoginInfo()
resolv.onLoginInfo = onLoginInfo
resolv.onListDownloaded = onListDownloaded
#log.startLogging(sys.stdout)
reactor.run()
from resolver import Resolver
from timeit import timeit
resolve = Resolver()
time_elapsed = timeit(setup='from __main__ import resolve ',
stmt='resolve("google.com")',
number=1)
print("Time elapsed for google.com:{0:f}".format(time_elapsed))
time_elapsed = timeit(setup='from __main__ import resolve ',
stmt='resolve("google.com")',
number=1)
print("Time elapsed for google.com:{0:f}".format(time_elapsed))
print("Is microsoft.com present in resolve cache:{0}".format(
resolve.has_host("microsoft.com")))
resolve("microsoft.com")
print("Is microsoft.com present in resolve cache:{0}".format(
resolve.has_host("microsoft.com")))
resolve.clear()
print("Is microsoft.com present in resolve cache:{0}".format(
resolve.has_host("microsoft.com")))
def testRecursion(self):
self.resolver = Resolver('198.41.0.4')
reference_a_record = [(1, '31.170.165.34')]
test_a_record = self.resolver.resolve('eur.al')
self.assertEqual(reference_a_record, test_a_record)
def setUp(self):
self.resolver = Resolver()
class MonitoredResource(object):
"""
Monitored resource (URL). The ressource is generally any document in any
format, but most often it will be HTML code.
This class wraps the URL content and metadata.
The contents can be manipulated within the time so it can provide information
about how the content changed in different versions of the document.
Warning:
Application developers should generally not need to instantiate this class
directly. The only correct way how to get this object is through
Monitor.get() method.
Design pattern: Active Record
Example of checking new version of document:
>>> from rrslib.web.changemonitor import Monitor
>>> monitor = Monitor(user_id="myuid")
>>> monitor
Monitor(conn=Connection('localhost', 27017), dbname='webarchive', uid='myuid')
>>> resource = monitor.get("http://www.myusefulpage.com/index.html")
>>> resource
<MonitoredResource(url='http://www.myusefulpage.com/index.html', uid='myuid') at 0xb7398accL>
>>> resource.check()
True
>>> # the resource has changed
Checking availability of the document on the URL
>>> from rrslib.web.changemonitor import HTTPDateTime
>>> resource = monitor.get("http://www.nonexistentpage.com")
>>> resource.available()
False
>>> resource = monitor.get("http://www.myusefulpage.com/index.html")
>>> resource.available(HTTPDateTime(2012, 6, 30, 15, 34))
True
Example of getting last available version of the document on the URL
>>> resource = monitor.get("http://www.myusefulpage.com/index.html")
>>> content = resource.get_last_version()
>>> print content.data
<html><head>
...
>>> resource = monitor.get("http://www.crazynonexistentpage.com")
>>> content = resource.get_last_version()
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
DocumentNotAvailable: The content of this URL is not available and it
is not in the storage.
Example of getting version of the document in exact time
>>> resource = monitor.get("http://www.myusefulpage.com/index.html")
>>> content = resource.get_version(HTTPDateTime(2012, 6, 30, 15, 34))
Getting the last time when the document was checked:
>>> resource = monitor.get("http://www.crazynotexistentpage.com")
>>> resource.last_checked()
HTTPDateTime(Thu, 01 Jan 1970 00:00:00 GMT)
"""
def __init__(self, url, uid, storage):
"""
@param url: monitored URL
@type url: basestring (str or unicode)
@param uid: user identifier. This ID has to be unique for each one,
who is using changemonitor.
@type uid: str
@param storage: storage of monitored-resource data
@type storage: model.Storage
"""
# resource data
self.url = url
self.uid = uid
# models
self.storage = storage
self.headers = storage._headermeta
# resolver
self.resolver = Resolver(storage)
self._checked = False
# file
try:
self.file = self.storage.get(url)
except DocumentNotAvailable:
# if the file is not in the storage, resolver has to check
# the url and load actual data into the storage
self.resolver.resolve(url)
self._checked = True
try:
self.file = self.storage.get(url)
except DocumentNotAvailable:
raise DocumentNotAvailable("Resource '%s' is not available." % url)
def check(self, force=False):
"""
Check the resource URL and load the most recent version into database.
TODO: consider using @lazy decorator. Most of use cases use this method
so we have to insure that it will be called only once.
@param force: force use of resolver (no effect on first call),
if force=False, doesn't try to download new content if called more than once
@type force: Bool
@raises: DocumentTooLargeException
@raises: DocumentNotAvailable
@returns: True if the document has changed since last check.
"""
# bude vyuzivat resolveru pro checknuti URL a ziskani informace o tom,
# jestli byl dokument zmenen. Mozna bude take dobre nahrat rovnou do
# self.file nejnovejsi verzi, ale o tom je potreba jeste pouvazovat.
# urcite je potreba pred kazdym checkem refreshout file cache
self.file.refresh_cache()
if force:
self._checked = False
if not self._checked: # use resolver to get most recent version if not yet checked
self.resolver.resolve(self.url)
self._checked = True
try:
self.file = self.storage.get(self.url)
except DocumentNotAvailable:
raise DocumentNotAvailable("Resource '%s' is not available." % self.url)
# and determine return value
try:
# time of last check
_now = self.headers.get_by_version(self.url,-1,True)['timestamp']
_now = HTTPDateTime().from_timestamp(_now+1)
# header and content are saved at the same time
# we need to find content inserted before _now, that's why (_now+1)
except Exception:
raise DocumentNotAvailable("Check failed. Cannot get header information of '%s'." % self.url)
try:
# time of previous check
_prev = self.headers.get_by_version(self.url,-2,True)
_prev = HTTPDateTime().from_timestamp(_prev['timestamp']+1)
except TypeError: # this is first time document is checked
if self._checked: return False # if already checked, and have no v=-2 header, there was no change
else: return True # this is the first-time check
#DEBUG
#? print "header time: _now: ",_now,"\nheader time: _prev: ",_prev,"\n"
d = self.get_diff(_prev,_now)
if d is None: return False
if isinstance(d, basestring): # PlainTextDiff, maybe htmldiff in some cases
if len(d)==0: return False
else: return True
if isinstance(d, dict): # BinaryDiff
if string.find(d['metainfo'],"(patch data)")==-1 :
# TODO: find what indicates that BinaryDiff-ed file hasn't changed
# current version seems to work, but needs more testing
return False
else: return True
try: # d is htmldiff output (generator object)
chunk = d.next()
if len(chunk.added)==0 and len(chunk.removed)==0:
return False
except (StopIteration, TypeError): # if can't get d.next(), then d is probably empty -> no change
return False
return True
def get_last_version(self):
"""
Get last available content of the document. If the document is available
at this time, returns most recent version which is on the web server.
@returns: Last available content of this resource.
@rtype: Content
@raises: DocumentNotAvailable if no content available (resource does not
exist on the URL and never existed within the known history)
"""
self.resolver.resolve(self.url)
self._checked = True
try:
self.file = self.storage.get(self.url)
return self.file.get_last_version()
except NoFile: # FIXME tady to prece nemuze byt??!
raise DocumentNotAvailable("Resource '%s' is not available." % self.url)
def get_version(self, time_or_version):
"""
Get content of this document in specified time or version. If the
document was not available in given time, returns last available content.
If there is no available content until given time, raises exception.
@param time_or_version: Time or version of the content we want to retrieve.
Version numbering is a convenience atop the GridFS API provided
by MongoDB. version ``-1`` will be the most recently uploaded
matching file, ``-2`` the second most recently uploaded, etc.
Version ``0`` will be the first version
uploaded, ``1`` the second version, etc. So if three versions
have been uploaded, then version ``0`` is the same as version
``-3``, version ``1`` is the same as version ``-2``, and
version ``2`` is the same as version ``-1``.
@type time_or_version: HTTPDateTime or int
@raises: DocumentHistoryNotAvailable if there is no available content until
given time or version
"""
if isinstance(time_or_version, HTTPDateTime):
return self.file.get_version(time_or_version.to_timestamp())
elif isinstance(time_or_version, int):
return self.file.get_version(time_or_version)
else:
raise TypeError("Version time has to be type HTTPDateTime or GridFS version (int).")
def get_diff(self, start, end):
"""
@param start: start time or version to be diffed
@type start: HTTPDateTime or int
@param end: end time or version to be diffed
@type end: HTTPDateTime or int
@returns: either textual or binary diff of the file (if available).
If contents are equal (document did not change within this
time range) returns None.
@rtype: unicode
@raises: DocumentHistoryNotAvaliable if the storage doesn't provide
enough data for computing the diff.
"""
content_start = self.get_version(start)
content_end = self.get_version(end)
if content_start == content_end:
return None
return content_start.diff_to(content_end)
def available(self, httptime=None):
if (not isinstance(httptime, HTTPDateTime)) and (httptime is not None):
raise TypeError("Time of availability has to be type HTTPDateTime.")
# Pokud je httptime=None, pak se jedna o dostupnost v tomto okamziku
if (httptime is None):
try:
self.check(force=True)
except DocumentNotAvailable:
return False
return True
else:
# when was last checked before 'httptime'
h = self.headers.get_by_time(self.url,httptime.to_timestamp())
if h is None: return False # not checked before time 'httptime'
t1 = h['timestamp']
try:
t2 = self.get_version(HTTPDateTime().from_timestamp(t1)).upload_date
except DocumentHistoryNotAvaliable:
return False
return True
def last_checked(self):
"""
Get information about the time of last check of this resource.
@returns: time of last check or None if the resource was never checked
(or the HTTP requests timed out)
@rtype: HTTPDateTime or None
"""
return self.headers.last_checked(self.url)
def __repr__(self):
return "<MonitoredResource(url='%s', uid='%s') at %s>" % \
(self.url, self.uid, hex(id(self)))
__str__ = __repr__
def play(url):
from resolver import Resolver
r = Resolver(url)
items.play_item(r.resolved_url, r.startpercent)
class Downloader(object):
def __init__(self,
base="http://repo1.maven.org/maven2",
username=None,
password=None):
self.requestor = Requestor(username, password)
self.resolver = Resolver(base, self.requestor)
def download(self, artifact, filename=None, suppress_log=False):
filename = artifact.get_filename(filename)
url = self.resolver.uri_for_artifact(artifact)
if not self.verify_md5(filename, url + ".md5"):
if not suppress_log:
print("Downloading artifact " + str(artifact))
hook = self._chunk_report
else:
hook = self._chunk_report_suppress
onError = lambda uri, err: self._throwDownloadFailed(
"Failed to download artifact " + str(artifact) + "from " + uri)
response = self.requestor.request(url, onError, lambda r: r)
if response:
with open(filename, 'w') as f:
self._write_chunks(response, f, report_hook=hook)
if not suppress_log:
print("Downloaded artifact %s to %s" %
(artifact, filename))
return (artifact, True)
else:
return (artifact, False)
else:
if not suppress_log:
print("%s is already up to date" % artifact)
return (artifact, True)
def _throwDownloadFailed(self, msg):
raise RequestException(msg)
def _chunk_report_suppress(self, bytes_so_far, chunk_size, total_size):
pass
def _chunk_report(self, bytes_so_far, chunk_size, total_size):
percent = float(bytes_so_far) / total_size
percent = round(percent * 100, 2)
sys.stdout.write("Downloaded %d of %d bytes (%0.2f%%)\r" %
(bytes_so_far, total_size, percent))
if bytes_so_far >= total_size:
sys.stdout.write('\n')
def _write_chunks(self, response, file, chunk_size=8192, report_hook=None):
total_size = response.info().getheader('Content-Length').strip()
total_size = int(total_size)
bytes_so_far = 0
while 1:
chunk = response.read(chunk_size)
bytes_so_far += len(chunk)
if not chunk:
break
file.write(chunk)
if report_hook:
report_hook(bytes_so_far, chunk_size, total_size)
return bytes_so_far
def verify_md5(self, file, remote_md5):
if not os.path.exists(file):
return False
else:
local_md5 = self._local_md5(file)
onError = lambda uri, err: self._throwDownloadFailed(
"Failed to download MD5 from " + uri)
remote = self.requestor.request(remote_md5, onError,
lambda r: r.read())
return local_md5 == remote
def _local_md5(self, file):
md5 = hashlib.md5()
with open(file, 'rb') as f:
for chunk in iter(lambda: f.read(8192), ''):
md5.update(chunk)
return md5.hexdigest()
def __init__(self,
base="http://repo1.maven.org/maven2",
username=None,
password=None):
self.requestor = Requestor(username, password)
self.resolver = Resolver(base, self.requestor)
from resolver import Resolver r = Resolver() r.import_file() print(r.df.info()) print(r.build())
def __init__(self, base="http://repo1.maven.org/maven2", username=None, password=None):
self.requestor = Requestor(username, password)
self.resolver = Resolver(base, self.requestor)
def resolve_view_var(self, node):
try:
return Resolver.resolve(node)
except AttributeError:
raise
