def enqueue_initial_messages(self,
sub_key,
topic_name,
endpoint_name,
_group_size=20):
""" Looks up any messages for input task in the database and pushes them all and enqueues in batches any found.
"""
with self.sub_key_locks[sub_key]:
pub_time_max = utcnow_as_ms()
session = None
try:
# One SQL session for all queries
session = self.pubsub.server.odb.session()
# Get IDs of any messages already queued up so as to break them out into batches of messages to fetch
msg_ids = self.pubsub.get_initial_sql_msg_ids_by_sub_key(
session, sub_key, pub_time_max)
msg_ids = [elem.pub_msg_id for elem in msg_ids]
if msg_ids:
len_msg_ids = len(msg_ids)
suffix = ' ' if len_msg_ids == 1 else 's '
groups = list(grouper(_group_size, msg_ids))
len_groups = len(groups)
# This we log using both loggers because we run during server startup so we should
# let users know that their server have to do something extra
for _logger in logger, logger_zato:
_logger.info(
'Found %d initial message%sto enqueue for sub_key:`%s` (%s -> %s), `%s`, g:%d, gs:%d',
len_msg_ids, suffix, sub_key, topic_name,
endpoint_name, msg_ids, len(groups), _group_size)
for idx, group in enumerate(groups, 1):
group_msg_ids = [elem for elem in group if elem]
logger.info(
'Enqueuing group %d/%d (gs:%d) (%s, %s -> %s) `%s`',
idx, len_groups, _group_size, sub_key, topic_name,
endpoint_name, group_msg_ids)
msg_list = self.pubsub.get_sql_messages_by_msg_id_list(
session, sub_key, pub_time_max, group_msg_ids)
self._enqueue_gd_messages_by_sub_key(sub_key, msg_list)
except Exception:
for _logger in logger, logger_zato:
_logger.warn(
'Could not enqueue initial messages for `%s` (%s -> %s), e:`%s`',
sub_key, topic_name, endpoint_name, format_exc())
finally:
if session:
session.close()
def handle(self):
result = self.translate(self.request.input.system1, self.request.input.key1, self.request.input.value1,
self.request.input.system2, self.request.input.key2)
if result:
self.response.payload.value2 = result.decode('utf-8')
self.response.payload.repr = repr(result)
self.response.payload.hex = ' '.join([elem1+elem2 for (elem1, elem2) in grouper(2, result.encode('hex'))])
self.response.payload.sha1 = sha1(result).hexdigest()
self.response.payload.sha256 = sha256(result).hexdigest()
def handle(self):
# Grab month and year from user-provided input or use defaults, i.e. current date,
# note that day is not needed so it's discarded
year, month, _ = get_date(self.request.input)
# Fetch connection by its name
out = self.outgoing.plain_http.get('treasury.gov')
# Build a query string the backend data source expects
query_string = {
'$filter': 'month(QUOTE_DATE) eq {} and year(QUOTE_DATE) eq {}'.format(month, year)
}
# Invoke the backend with query string, fetch the response as a UTF-8 string
# and turn it into an XML object
response = out.conn.get(self.cid, query_string)
response = response.text.encode('utf-8')
xml = etree.fromstring(response)
# Look up all XML elements needed (date and rate) using XPath
elements = xml.xpath('//m:properties/d:*/text()', namespaces=NAMESPACES)
# elements is a flat list that needs to be turned into pairs using the 'grouper'
# function before iterating over
elements = grouper(2, elements)
for date, rate in elements:
# Create a date object out of string
date = parse(date)
# Build a key for Redis and store the data under it
key = REDIS_KEY_PATTERN.format(date.year, str(date.month).zfill(2), str(date.day).zfill(2))
self.logger.info("redis key : %s"%key)
self.kvdb.conn.set(key, rate)
# Leave a trace of our activity
self.logger.info('Key %s set to %s', key, rate)
def handle(self):
result = self.translate(
self.request.input.system1,
self.request.input.key1,
self.request.input.value1,
self.request.input.system2,
self.request.input.key2,
)
if result:
self.response.payload.value2 = result.decode("utf-8")
self.response.payload.repr = repr(result)
self.response.payload.hex = " ".join([elem1 + elem2 for (elem1, elem2) in grouper(2, result.encode("hex"))])
self.response.payload.sha1 = sha1(result).hexdigest()
self.response.payload.sha256 = sha256(result).hexdigest()
def generate_password(length=16):
chain = MarkovChain(
c for c in japanese.lower() + occitan.lower() + breton.lower() if c in string.ascii_lowercase
)
return '-'.join(''.join(elems) for elems in (grouper(4, ''.join(itertools.islice(chain, length)))))
