def _run_impl(self):
"""
Polls for newly registered accounts and checks for start/stop commands.
"""
while self.keep_running:
# Determine which accounts need to be started
with session_scope() as db_session:
sync_on_this_node = or_(Account.sync_state.is_(None),
Account.sync_host == platform.node())
sync_explicitly_stopped = and_(Account.sync_state == 'stopped',
Account.sync_state.isnot(None))
start_on_this_cpu = \
(func.mod(Account.id, self.total_cpus) == self.cpu_id)
start_accounts = \
[id_ for id_, in db_session.query(Account.id).filter(
sync_on_this_node,
~sync_explicitly_stopped,
start_on_this_cpu)]
# perform the appropriate action on each account
for account_id in start_accounts:
if account_id not in self.monitors:
self.start_sync(account_id)
stop_accounts = set(self.monitors.keys()) - \
set(start_accounts)
for account_id in stop_accounts:
self.log.info('sync service stopping sync',
account_id=account_id)
self.stop_sync(account_id)
gevent.sleep(self.poll_interval)
def sync_rides_distributed(
self,
total_segments: int,
segment: int,
start_date: datetime = None,
end_date: datetime = None,
):
"""
:param total_segments: The number of segments to divide athletes into (e.g. 24 if this is being run hourly)
:param segment: Which segment (0-based) to select.
:param start_date: Will default to competition start.
:param end_date: Will default to competition end.
"""
with meta.transaction_context() as sess:
q = sess.query(Athlete)
q = q.filter(Athlete.access_token != None)
q = q.filter(func.mod(Athlete.id, total_segments) == segment)
athletes: List[Athlete] = q.all()
self.logger.info(
"Selecting segment {} / {}, found {} athletes".format(
segment, total_segments, len(athletes)))
athlete_ids = [a.id for a in athletes]
if athlete_ids:
return self.sync_rides(start_date=start_date,
end_date=end_date,
athlete_ids=athlete_ids)
def _run_impl(self):
"""
Polls for newly registered accounts and checks for start/stop commands.
"""
while self.keep_running:
# Determine which accounts need to be started
with session_scope() as db_session:
sync_on_this_node = or_(Account.sync_state.is_(None),
Account.sync_host == platform.node())
sync_explicitly_stopped = and_(Account.sync_state == 'stopped',
Account.sync_state.isnot(None))
start_on_this_cpu = \
(func.mod(Account.id, self.total_cpus) == self.cpu_id)
start_accounts = \
[id_ for id_, in db_session.query(Account.id).filter(
sync_on_this_node,
~sync_explicitly_stopped,
start_on_this_cpu)]
# perform the appropriate action on each account
for account_id in start_accounts:
if account_id not in self.monitors:
self.start_sync(account_id)
stop_accounts = set(self.monitors.keys()) - \
set(start_accounts)
for account_id in stop_accounts:
self.log.info('sync service stopping sync',
account_id=account_id)
self.stop_sync(account_id)
gevent.sleep(self.poll_interval)
def gcd_table(table) :
modulo = select([table.c.id,
table.c.id.label('raw_id'),
table.c.num,
table.c.den]).\
cte(name=table.name+"_gcd_first", recursive=True)
modulo_a = modulo.alias(name = table.name+"_gcd_prev")
modulo = modulo.union_all(
select([
(modulo_a.c.id + 1).label('id'),
modulo_a.c.raw_id.label('raw_id'),
func.mod(modulo_a.c.den, modulo_a.c.num).label('num'),
modulo_a.c.num.label('den')
]).
where(modulo_a.c.num > 0)
)
'''
modulo = modulo.union_all(
select([
(modulo_a.c.id + 1).label('id'),
modulo_a.c.raw_id.label('raw_id'),
case([(modulo_a.c.den >= modulo_a.c.num, modulo_a.c.num)], else_= modulo_a.c.den).label('num'),
case([(modulo_a.c.den >= modulo_a.c.num, modulo_a.c.den - modulo_a.c.num)], else_=modulo_a.c.num).label('den')
]).
where(modulo_a.c.num > 0)
)
'''
'''
modulo = modulo.union_all(
select([
(modulo_a.c.id + 1).label('id'),
modulo_a.c.raw_id.label('raw_id'),
func.kludge_0(modulo_a.c.num, modulo_a.c.den).label('num'),
func.kludge_1(modulo_a.c.num, modulo_a.c.den).label('den')
]).
where(modulo_a.c.num > 0)
)
'''
stmt = select([table.c.id.label('id'),
(table.c.num / modulo.c.den).label('num'),
(table.c.den / modulo.c.den).label('den')]).\
select_from(
table.\
join(modulo, onclause=table.c.id == modulo.c.raw_id)).\
where(modulo.c.num == 0)
return stmt
def import_fulltexts(overwrite=False, mod_n=None, mod_i=None):
nodes = q(Data).filter(Data.files.any(File.filetype == "fulltext")).order_by(Data.id)
if mod_n:
if mod_i is None:
raise Exception("mod_i must be specified when mod_n is given!")
nodes = nodes.filter(sqlfunc.mod(Data.id, mod_n) == mod_i)
# don't overwrite = ignore nodes that have a fulltext
if not overwrite:
nodes = nodes.filter_by(fulltext=None)
import_count = 0
for node in nodes:
imported = import_node_fulltext(node, overwrite=overwrite)
if imported:
import_count += 1
return import_count
def import_fulltexts(overwrite=False, mod_n=None, mod_i=None):
nodes = q(Data).filter(
Data.files.any(File.filetype == "fulltext")).order_by(Data.id)
if mod_n:
if mod_i is None:
raise Exception("mod_i must be specified when mod_n is given!")
nodes = nodes.filter(sqlfunc.mod(Data.id, mod_n) == mod_i)
# don't overwrite = ignore nodes that have a fulltext
if not overwrite:
nodes = nodes.filter_by(fulltext=None)
import_count = 0
for node in nodes:
imported = import_node_fulltext(node, overwrite=overwrite)
if imported:
import_count += 1
return import_count
def _get_local_accounts(self):
with session_scope() as db_session:
# Whether this node should use a work-stealing style approach
# to claiming accounts that don't have a specified sync_host
steal = and_(Account.sync_host.is_(None),
config.get('SYNC_STEAL_ACCOUNTS', True))
# Whether accounts should be claimed via explicis scheduling
explicit = Account.sync_host == platform.node()
# Start new syncs on this node if the sync_host is set
# explicitly to this node, or if the sync_host is not set and
# this node is configured to use a work-stealing style approach
# to scheduling accounts.
start = and_(Account.sync_state.is_(None),
or_(steal, explicit))
# Don't restart a previous sync if it's sync_host is not
# this node (i.e. it's running elsewhere),
# was explicitly stopped or
# killed due to invalid credentials
dont_start = or_(Account.sync_host != platform.node(),
Account.sync_state.in_(['stopped',
'invalid']))
# Start IFF an account IS in the set of startable syncs OR
# NOT in the set of dont_start syncs
sync_on_this_node = or_(start, ~dont_start)
start_on_this_cpu = \
(func.mod(Account.id, self.total_cpus) == self.cpu_id)
start_accounts = \
[id_ for id_, in db_session.query(Account.id).filter(
sync_on_this_node,
start_on_this_cpu)]
return start_accounts
def _get_local_accounts(self):
with session_scope() as db_session:
# Whether this node should use a work-stealing style approach
# to claiming accounts that don't have a specified sync_host
steal = and_(Account.sync_host.is_(None),
config.get('SYNC_STEAL_ACCOUNTS', True))
# Whether accounts should be claimed via explicis scheduling
explicit = Account.sync_host == platform.node()
# Start new syncs on this node if the sync_host is set
# explicitly to this node, or if the sync_host is not set and
# this node is configured to use a work-stealing style approach
# to scheduling accounts.
start = and_(Account.sync_state.is_(None),
or_(steal, explicit))
# Don't restart a previous sync if it's sync_host is not
# this node (i.e. it's running elsewhere),
# was explicitly stopped or
# killed due to invalid credentials
dont_start = or_(Account.sync_host != platform.node(),
Account.sync_state.in_(['stopped',
'invalid']))
# Start IFF an account IS in the set of startable syncs OR
# NOT in the set of dont_start syncs
sync_on_this_node = or_(start, ~dont_start)
start_on_this_cpu = \
(func.mod(Account.id, self.total_cpus) == self.cpu_id)
start_accounts = \
[id_ for id_, in db_session.query(Account.id).filter(
sync_on_this_node,
start_on_this_cpu)]
return start_accounts
def is_user(cls):
return func.mod(func.nlevel(cls._path), 2) == 1
