def count_users(self, timestamp=None):
if timestamp is None:
timestamp = get_timestamp()
res = self._safe_execute(_COUNT_USER_RECORDS, timestamp=timestamp)
row = res.fetchone()
res.close()
return row[0]
def allocate_user(self, service, email, generation=0, client_state='',
keys_changed_at=0, node=None, timestamp=None):
if timestamp is None:
timestamp = get_timestamp()
if node is None:
nodeid, node = self.get_best_node(service, email)
else:
nodeid = self.get_node_id(service, node)
params = {
'service': service,
'email': email,
'nodeid': nodeid,
'generation': generation,
'keys_changed_at': keys_changed_at,
'client_state': client_state,
'timestamp': timestamp
}
res = self._safe_execute(_CREATE_USER_RECORD, **params)
return {
'email': email,
'uid': res.lastrowid,
'node': node,
'generation': generation,
'keys_changed_at': keys_changed_at,
'client_state': client_state,
'old_client_states': {},
'first_seen_at': timestamp,
}
def replace_user_record(self, service, uid, timestamp=None):
"""Mark an existing service record as replaced."""
if timestamp is None:
timestamp = get_timestamp()
params = {'service': service, 'uid': uid, 'timestamp': timestamp}
res = self._safe_execute(_REPLACE_USER_RECORD, **params)
res.close()
def allocate_user(self, service, email, generation=0, client_state="", node=None, timestamp=None):
if timestamp is None:
timestamp = get_timestamp()
if node is None:
nodeid, node = self.get_best_node(service)
else:
nodeid = self.get_node_id(service, node)
params = {
"service": service,
"email": email,
"nodeid": nodeid,
"generation": generation,
"client_state": client_state,
"timestamp": timestamp,
}
res = self._safe_execute(_CREATE_USER_RECORD, **params)
res.close()
return {
"email": email,
"uid": res.lastrowid,
"node": node,
"generation": generation,
"client_state": client_state,
"old_client_states": {},
"first_seen_at": timestamp,
}
def replace_user_record(self, service, uid, timestamp=None):
"""Mark an existing service record as replaced."""
if timestamp is None:
timestamp = get_timestamp()
params = {"service": service, "uid": uid, "timestamp": timestamp}
res = self._safe_execute(_REPLACE_USER_RECORD, **params)
res.close()
def replace_user_records(self, service, email, timestamp=None):
"""Mark all existing service records for a user as replaced."""
if timestamp is None:
timestamp = get_timestamp()
params = {'service': service, 'email': email, 'timestamp': timestamp}
res = self._safe_execute(_REPLACE_USER_RECORDS, **params)
res.close()
def count_users(self, timestamp=None):
if timestamp is None:
timestamp = get_timestamp()
res = self._safe_execute(_COUNT_USER_RECORDS, timestamp=timestamp)
row = res.fetchone()
res.close()
return row[0]
def replace_user_records(self, service, email, timestamp=None):
"""Mark all existing service records for a user as replaced."""
if timestamp is None:
timestamp = get_timestamp()
params = {"service": service, "email": email, "timestamp": timestamp}
res = self._safe_execute(_REPLACE_USER_RECORDS, **params)
res.close()
def retire_user(self, email, engine=None):
now = get_timestamp()
params = {"email": email, "timestamp": now, "generation": MAX_GENERATION}
# Pass through explicit engine to help with sharded implementation,
# since we can't shard by service name here.
res = self._safe_execute(_RETIRE_USER_RECORDS, engine=engine, **params)
res.close()
def retire_user(self, email, engine=None):
now = get_timestamp()
params = {
'email': email, 'timestamp': now, 'generation': MAX_GENERATION
}
# Pass through explicit engine to help with sharded implementation,
# since we can't shard by service name here.
res = self._safe_execute(_RETIRE_USER_RECORDS, engine=engine, **params)
res.close()
def update_user(self, service, user, generation=None, client_state=None,
node=None):
if client_state is None and node is None:
# We're just updating the generation, re-use the existing record.
if generation is not None:
params = {
'service': service,
'email': user['email'],
'generation': generation
}
res = self._safe_execute(_UPDATE_GENERATION_NUMBER, **params)
res.close()
user['generation'] = max(generation, user['generation'])
else:
# Reject previously-seen client-state strings.
if client_state is None:
client_state = user['client_state']
else:
if client_state == user['client_state']:
raise BackendError('previously seen client-state string')
if client_state in user['old_client_states']:
raise BackendError('previously seen client-state string')
# Need to create a new record for new user state.
# If the node is not explicitly changing, try to keep them on the
# same node, but if e.g. it no longer exists them allocate them to
# a new one.
if node is not None:
nodeid = self.get_node_id(service, node)
user['node'] = node
else:
try:
nodeid = self.get_node_id(service, user['node'])
except ValueError:
nodeid, node = self.get_best_node(service)
user['node'] = node
if generation is not None:
generation = max(user['generation'], generation)
else:
generation = user['generation']
now = get_timestamp()
params = {
'service': service, 'email': user['email'],
'nodeid': nodeid, 'generation': generation,
'client_state': client_state, 'timestamp': now,
}
res = self._safe_execute(_CREATE_USER_RECORD, **params)
res.close()
user['uid'] = res.lastrowid
user['generation'] = generation
user['old_client_states'][user['client_state']] = True
user['client_state'] = client_state
# mark old records as having been replaced.
# if we crash here, they are unmarked and we may fail to
# garbage collect them for a while, but the active state
# will be undamaged.
self.replace_user_records(service, user['email'], now)
def get_old_user_records(self, service, grace_period=-1, limit=100):
"""Get user records that were replaced outside the grace period."""
if grace_period < 0:
grace_period = 60 * 60 * 24 * 7 # one week, in seconds
grace_period = int(grace_period * 1000) # convert seconds -> millis
params = {"service": service, "timestamp": get_timestamp() - grace_period, "limit": limit}
res = self._safe_execute(_GET_OLD_USER_RECORDS_FOR_SERVICE, **params)
try:
for row in res:
yield row
finally:
res.close()
def unassign_node(self, service, node, timestamp=None, nodeid=None):
"""Clear any assignments to a node."""
if timestamp is None:
timestamp = get_timestamp()
if nodeid is None:
nodeid = self.get_node_id(service, node)
res = self._safe_execute(sqltext("""
update users
set replaced_at=:timestamp
where nodeid=:nodeid
"""),
nodeid=nodeid,
timestamp=timestamp)
res.close()
def unassign_node(self, service, node, timestamp=None, nodeid=None):
"""Clear any assignments to a node."""
if timestamp is None:
timestamp = get_timestamp()
if nodeid is None:
nodeid = self.get_node_id(service, node)
res = self._safe_execute(sqltext(
"""
update users
set replaced_at=:timestamp
where nodeid=:nodeid
"""),
nodeid=nodeid, timestamp=timestamp
)
res.close()
def get_old_user_records(self, service, grace_period=-1, limit=100):
"""Get user records that were replaced outside the grace period."""
if grace_period < 0:
grace_period = 60 * 60 * 24 * 7 # one week, in seconds
grace_period = int(grace_period * 1000) # convert seconds -> millis
params = {
"service": service,
"timestamp": get_timestamp() - grace_period,
"limit": limit,
}
res = self._safe_execute(_GET_OLD_USER_RECORDS_FOR_SERVICE, **params)
try:
for row in res:
yield row
finally:
res.close()
def allocate_user(self, service, email, generation=0, client_state='',
node=None):
if (service, email) in self._users:
raise BackendError('user already exists: ' + email)
if node is not None and node != self.service_entry:
raise ValueError("unknown node: %s" % (node,))
user = {
'email': email,
'uid': self._next_uid,
'node': self.service_entry,
'generation': generation,
'client_state': client_state,
'old_client_states': {},
'first_seen_at': get_timestamp()
}
self._users[(service, email)] = user
self._next_uid += 1
return user
def allocate_user(self, service, email, generation=0, client_state='',
keys_changed_at=0, node=None):
if (service, email) in self._users:
raise BackendError('user already exists: ' + email)
if node is not None and node != self.service_entry:
raise ValueError("unknown node: %s" % (node,))
user = {
'email': email,
'uid': self._next_uid,
'node': self.service_entry,
'generation': generation,
'keys_changed_at': keys_changed_at,
'client_state': client_state,
'old_client_states': {},
'first_seen_at': get_timestamp(),
}
self._users[(service, email)] = user
self._next_uid += 1
return user.copy()
def allocate_user(self, service, email, generation=0, client_state='',
node=None, timestamp=None):
if timestamp is None:
timestamp = get_timestamp()
if node is None:
nodeid, node = self.get_best_node(service)
else:
nodeid = self.get_node_id(service, node)
params = {
'service': service, 'email': email, 'nodeid': nodeid,
'generation': generation, 'client_state': client_state,
'timestamp': timestamp
}
res = self._safe_execute(_CREATE_USER_RECORD, **params)
res.close()
return {
'email': email,
'uid': res.lastrowid,
'node': node,
'generation': generation,
'client_state': client_state,
'old_client_states': {},
'first_seen_at': timestamp
}
def update_user(self,
service,
user,
generation=None,
client_state=None,
node=None):
if client_state is None and node is None:
# We're just updating the generation, re-use the existing record.
if generation is not None:
params = {
'service': service,
'email': user['email'],
'generation': generation
}
res = self._safe_execute(_UPDATE_GENERATION_NUMBER, **params)
res.close()
user['generation'] = max(generation, user['generation'])
else:
# Reject previously-seen client-state strings.
if client_state is None:
client_state = user['client_state']
else:
if client_state == user['client_state']:
raise BackendError('previously seen client-state string')
if client_state in user['old_client_states']:
raise BackendError('previously seen client-state string')
# Need to create a new record for new user state.
# If the node is not explicitly changing, try to keep them on the
# same node, but if e.g. it no longer exists them allocate them to
# a new one.
if node is not None:
nodeid = self.get_node_id(service, node)
user['node'] = node
else:
try:
nodeid = self.get_node_id(service, user['node'])
except ValueError:
nodeid, node = self.get_best_node(service)
user['node'] = node
if generation is not None:
generation = max(user['generation'], generation)
else:
generation = user['generation']
now = get_timestamp()
params = {
'service': service,
'email': user['email'],
'nodeid': nodeid,
'generation': generation,
'client_state': client_state,
'timestamp': now,
}
res = self._safe_execute(_CREATE_USER_RECORD, **params)
res.close()
user['uid'] = res.lastrowid
user['generation'] = generation
user['old_client_states'][user['client_state']] = True
user['client_state'] = client_state
# mark old records as having been replaced.
# if we crash here, they are unmarked and we may fail to
# garbage collect them for a while, but the active state
# will be undamaged.
self.replace_user_records(service, user['email'], now)
def update_user(self, service, user, generation=None, client_state=None,
keys_changed_at=None, node=None):
if client_state is None and node is None:
# No need for a node-reassignment, just update the row in place.
# Note that if we're changing keys_changed_at without changing
# client_state, it's because we're seeing an existing value of
# keys_changed_at for the first time.
params = {
'service': service,
'email': user['email'],
'generation': generation,
'keys_changed_at': keys_changed_at
}
res = self._safe_execute(_UPDATE_USER_RECORD_IN_PLACE, **params)
res.close()
user['generation'] = max(generation, user['generation'])
user['keys_changed_at'] = max(keys_changed_at,
user['keys_changed_at'])
else:
# Reject previously-seen client-state strings.
if client_state is None:
client_state = user['client_state']
else:
if client_state == user['client_state']:
raise BackendError('previously seen client-state string')
if client_state in user['old_client_states']:
raise BackendError('previously seen client-state string')
# Need to create a new record for new user state.
# If the node is not explicitly changing, try to keep them on the
# same node, but if e.g. it no longer exists them allocate them to
# a new one.
if node is not None:
nodeid = self.get_node_id(service, node)
user['node'] = node
else:
try:
nodeid = self.get_node_id(service, user['node'])
except ValueError:
nodeid, node = self.get_best_node(service)
user['node'] = node
if generation is not None:
generation = max(user['generation'], generation)
else:
generation = user['generation']
if keys_changed_at is not None:
keys_changed_at = max(user['keys_changed_at'], keys_changed_at)
else:
keys_changed_at = user['keys_changed_at']
now = get_timestamp()
params = {
'service': service, 'email': user['email'],
'nodeid': nodeid, 'generation': generation,
'keys_changed_at': keys_changed_at,
'client_state': client_state, 'timestamp': now,
}
res = self._safe_execute(_CREATE_USER_RECORD, **params)
res.close()
user['uid'] = res.lastrowid
user['generation'] = generation
user['keys_changed_at'] = keys_changed_at
user['old_client_states'][user['client_state']] = True
user['client_state'] = client_state
# mark old records as having been replaced.
# if we crash here, they are unmarked and we may fail to
# garbage collect them for a while, but the active state
# will be undamaged.
self.replace_user_records(service, user['email'], now)