async def ff_stream(self, length: str):
time = parse_timestamp(length)
if time:
current, _ = self._media_player.get_video_time()
await self._seek_stream(current + time)
else:
await self._bot.say('Invalid parameter.')
def download_logs():
""" Download mailgun logs and store them in the database """
# use ordered dict to protect against new logs arriving while downloading logs
logs = OrderedDict()
skip = 0
# Fetch all unsaved logs and add them to a LIFO queue
fetch_more = True
while fetch_more:
print("fecthing logs skip={}".format(skip))
logs_tmp = api.get_logs(limit=1000, skip=skip)['items']
if len(logs_tmp) == 0:
break
for log in logs_tmp:
log_data = json.dumps(log)
log_hash = hashlib.sha256(log_data).hexdigest()
if db.MailgunLog.objects.filter(log_hash=log_hash).exists():
fetch_more = False
break
else:
logs[log_hash] = (log_hash, log_data, parse_timestamp(log['created_at']))
skip += 1000
# take items from LIFO queue and save to db
print("Saving {0} logs to database".format(len(logs)))
for i, (log_hash, data, timestamp) in enumerate(logs.values()):
db.MailgunLog(
log_hash=log_hash,
data=data,
timestamp=timestamp
).save()
if (i+1) % 100 == 0:
transaction.commit()
transaction.commit()
def download_logs():
""" Download mailgun logs and store them in the database """
# use ordered dict to protect against new logs arriving while downloading logs
logs = OrderedDict()
skip = 0
# Fetch all unsaved logs and add them to a LIFO queue
fetch_more = True
while fetch_more:
print("fecthing logs skip={}".format(skip))
logs_tmp = api.get_logs(limit=1000, skip=skip)['items']
if len(logs_tmp) == 0:
break
for log in logs_tmp:
log_data = json.dumps(log)
log_hash = hashlib.sha256(log_data).hexdigest()
if db.MailgunLog.objects.filter(log_hash=log_hash).exists():
fetch_more = False
break
else:
logs[log_hash] = (log_hash, log_data,
parse_timestamp(log['created_at']))
skip += 1000
# take items from LIFO queue and save to db
print("Saving {0} logs to database".format(len(logs)))
for i, (log_hash, data, timestamp) in enumerate(logs.values()):
db.MailgunLog(log_hash=log_hash, data=data, timestamp=timestamp).save()
if (i + 1) % 100 == 0:
transaction.commit()
transaction.commit()
def query_by_timestamp(timestamp_lt, timestamp_gt):
"""
Returns all the Trades within the "less than" and "greater than"
timestamps
"""
query = Trade.query()
if timestamp_lt is not None:
timestamp_lt, nanoseconds_lt = parse_timestamp(timestamp_lt)
query = query.filter(
ndb.OR(
Trade.timestamp < timestamp_lt,
ndb.AND(
Trade.timestamp == timestamp_lt,
Trade.timestamp_nanoseconds < nanoseconds_lt
)
)
)
second_query = None
if timestamp_gt is not None:
timestamp_gt, nanoseconds_gt = parse_timestamp(timestamp_gt)
args = [
ndb.OR(
Trade.timestamp > timestamp_gt,
ndb.AND(
Trade.timestamp == timestamp_gt,
Trade.timestamp_nanoseconds > nanoseconds_gt
)
)
]
if timestamp_lt is None:
query = query.filter(*args)
else:
second_query = Trade.query(*args)
if second_query is None:
return query
else:
# The Datastore rejects queries using inequality filtering
# on more than one property. Therefore we need to evaluate
# the diff between the two queries manually
query_keys = [x for x in query.iter(keys_only=True)]
second_query_keys = [x for x in second_query.iter(keys_only=True)]
shared_keys = set(query_keys) & set(second_query_keys)
return Trade.query(Trade.key.IN(shared_keys))
def __init__(self, title=None, timestamp=None, author=None, base_url=None, msg=None, link=None):
self.title = title
self.author = author
self.time = utils.parse_timestamp(timestamp)
self.timestamp = str(self.time)
self.msg = msg
self.base_url = base_url
self.filename = self.create_filename(self.title)
self.link = self.create_link()
def collect_match_info(response, profile_dict, matched_ideas, run_time):
"""Prepare information about matches made to be added to the
matches database table. Return a list of dicts, where each
dict is a row to add to the database.
"""
match_info = []
for idea in matched_ideas:
match_info.append({
'participant_userid': profile_dict['userid'],
'p_profile_pageid': profile_dict['profile_id'],
'p_interest': None,
'p_skill': None,
'request_time': utils.parse_timestamp(profile_dict['cat_time']),
'match_time': utils.parse_timestamp(response['newtimestamp']),
'match_revid': response['newrevid'],
'idea_pageid': idea['profileid'],
'run_time': run_time
})
return match_info
def setup_information(self):
model = self.model.screen_controller.model
self._body.part_text_set("author", "By: %s" % self.model.prop["author"])
date = parse_timestamp(self.model.prop["date"])
self._body.part_text_set("date", "Date: %s" % \
date.strftime("%b %d, %Y %I:%M%p"))
comment_panel = self.model.prop["content"]
comment_panel = comment_panel.replace("\n", "<br>")
self.description.text_set(comment_panel)
def ping_recv(self, msg):
"""Parse ping 1 (without flag & seq) and send ping 2."""
#logging.debug("recv ping1")
self.cancel_job.cancel()
time0 = parse_timestamp(msg[:11])
self.latency = int(time() * 1000) - time0
logging.debug("latency: %dms" % self.latency)
raw_packet = "1" + "2" + msg[11:]
to_write = self.cipher.encrypt(raw_packet) + self.split_char
if self.transport:
#logging.debug("send ping2")
self.transport.write(to_write)
def ping_recv(self, msg):
"""Parse ping 1 (without flag & seq) and send ping 2."""
#logging.debug("recv ping1")
self.cancel_job.cancel()
time0 = parse_timestamp(msg[:11])
self.latency = int(time() * 1000) - time0
logging.debug("latency: %dms" % self.latency)
raw_packet = "1" + "2" + msg[11:]
to_write = self.cipher.encrypt(raw_packet) + self.split_char
if self.transport:
#logging.debug("send ping2")
self.transport.write(to_write)
def isclearstates(self, timestamp):
"""
Define rules here to clear the server states
"""
hour_to_clear = 3 # clear states after this hour in the day
hour, minute, second = utils.parse_timestamp(timestamp)
try:
dH, dM, dS = utils.parse_timestamp(timestamp - self.hour_to_clear)
except:
# if this is the first start the server, hour_to_clear won't exist
# then return True to tell the class to clear the states
# set hour_to_clear to last hour_to_clear
offset_to_htc = (hour - hour_to_clear) * 60 * 60
self.hour_to_clear = timestamp - offset_to_htc
return True
# more than one day from last purge, return True
if dH > 24:
self.hour_to_clear = timestamp
return True
else:
return False
def parse(row: OrderedDict) -> OrderedDict:
"""Parse data types"""
data_types = dict(
validity=int,
verification=int,
value=float,
# Output timestamp in ISO 8601
timestamp=lambda s: utils.parse_timestamp(s).isoformat(),
)
# Cast to new data type (or default to string)
_row = OrderedDict()
for key, value in row.items():
DataType = data_types.get(key, str)
_row[key] = DataType(value)
return _row
def readings(self, session, **params) -> iter:
"""
Stream CSV data as rows (one dictionary per row)
"""
endpoint = "{endpoint}/readings.csv".format(endpoint=self.endpoint)
stream = session.call_iter(endpoint, params=params)
# First row
headers = next(csv.reader(stream))
for row in csv.DictReader(stream, fieldnames=headers):
# Parse
try:
row = OrderedDict(station=self.object_id,
measure=row['measure'],
timestamp=utils.parse_timestamp(
row['dateTime']),
value=utils.parse_value(row['value']))
yield row
except ValueError:
LOGGER.error(row)
raise
def get_archive(cls, session, date):
"""
Historic Readings. The measurement readings are archived daily as dump files in CSV format.
https://environment.data.gov.uk/flood-monitoring/doc/reference#historic-readings
"""
for row in cls._get_archive(session=session, date=date):
# Rename columns
yield OrderedDict(
timestamp=utils.parse_timestamp(row['dateTime']),
station=row['station'],
station_reference=row['stationReference'],
measure=row['measure'],
unit_name=row['unitName'],
value=utils.parse_value(row['value']),
datumType=row['datumType'],
label=row['label'],
parameter=row['parameter'],
qualifier=row['qualifier'],
period=row['period'],
value_type=row['valueType'],
observed_property=settings.PARAMETER_MAP[row['parameter']],
)
def log(json_data):
logging.info('Received JSON: %s' % json_data)
payload = json.loads(json_data)
nickname = payload.get('nickname')
server = payload.get('server', 'unknown')
channel = payload.get('channel')
message_type = payload.get('message_type')
message_content = payload.get('message_content')
timestamp = parse_timestamp(payload.get('timestamp'))
user = get_or_create(User, server=server, nickname=nickname)
user.last_seen_at = datetime.utcnow()
user.put()
channel = get_or_create(Channel, server=server, channel=channel)
# store the message
msg = Message(user=user, channel=channel, message_type=message_type,
message_content=message_content, json=json_data,
timestamp=timestamp)
msg.put()
return msg
def process_request(self, request_message):
request_type, userId, userInfo, timestamp = self.parse_request(
request_message)
self.request_type = request_type
self.request_userId = userId
# reset the server states (renew user list and everything) at some point in a day
if self.isclearstates(timestamp):
hour, minute, second = utils.parse_timestamp(timestamp)
print("Clear server states at {}:{}:{}".format(
hour, minute, second))
self.reset()
# add to user list if the request came from new user
broadcast_decision = False
if userId not in self.userList:
userInfo['reserving'] = False
userInfo['acquiring'] = False
userInfo['returns'] = 0
self.userList.update({userId: userInfo})
broadcast_decision = True
# if new connection/user call for a status update, broadcast the system status
if request_type is self._requestTypes['update']: return True
#
# NOTE: the request could be sent from other people, so the userId and the userInfo
# may not be the user of interest, we need to change them for further editing
if None: pass
elif request_type == self._requestTypes['force-return']:
userId = self.find_user(lambda x: x['acquiring'])
request_type = self._requestTypes[
'return'] # pretend this is a normal request
elif request_type == self._requestTypes['force-cancel']:
userId = self.find_user(lambda x: x['reserving'])
request_type = self._requestTypes[
'cancel'] # pretend this is a normal request
if not userId:
print("can't find user of interest. request:({})".format(
request_type))
return broadcast_decision
userInfo = self.userList[userId]
#
# stop log ill logics
#
if self.token_userId and request_type == self._requestTypes['acquire']:
print("can't acquire when somebody is keeping it.")
return broadcast_decision
if self.reserve_userId and userId != self.reserve_userId and request_type == self._requestTypes[
'acquire']:
print("can't acquire when other has reserved it.")
return broadcast_decision
if self.reserve_userId and request_type == self._requestTypes[
'reserve']:
print("can't reserve when somebody has already reserved.")
return broadcast_decision
if userId == self.token_userId and request_type == self._requestTypes[
'reserve']:
print("can't reserve when keeping the token.")
return broadcast_decision
if not self.token_userId and request_type == self._requestTypes[
'reserve']:
print(
"can't reserve when the token is free (why not acquiring it).")
return broadcast_decision
if userId != self.token_userId and request_type is self._requestTypes[
'return']:
print("can't return the token without keeping it.")
return broadcast_decision
if userId == self.reserve_userId and request_type == self._requestTypes[
'reserve']:
print("can't reserve twice.")
return broadcast_decision
if userId != self.reserve_userId and request_type == self._requestTypes[
'cancel']:
print("can't cancel without reserving.")
return broadcast_decision
#
# process legitimate logics
#
if False: pass
elif request_type is self._requestTypes['return']:
userInfo['returns'] += 1 # record a completed acquiring
userInfo['acquiring'] = False
self.token_userId = False
self.token_time = -1
broadcast_decision = True
# if someone reserved the token, start the reservation timer (initialize the timestamp)
if self.find_user(lambda x: x['reserving']):
self.reserve_time = timestamp
elif request_type is self._requestTypes['acquire']:
self.token_userId = userId
self.token_time = timestamp
userInfo['acquiring'] = True # record keeping the token
# if acquire request from reserve user, then reset reserve time and reserve_user
if userId == self.reserve_userId:
self.reserve_userId = False # release reservation
self.reserve_time = -1
userInfo['reserving'] = False
broadcast_decision = True
elif request_type is self._requestTypes['reserve']:
userInfo['reserving'] = True # record reserving the token
self.reserve_userId = userId
broadcast_decision = True
elif request_type is self._requestTypes['cancel']:
# cancel reservation
userInfo['reserving'] = False
self.reserve_userId = False
self.reserve_time = -1
broadcast_decision = True
else:
print("Unkown request_type {}".format(request_type))
return broadcast_decision
async def seek_stream(self, timestamp: str):
time = parse_timestamp(timestamp)
if time:
await self._seek_stream(time)
else:
await self._bot.say('Invalid parameter.')
