def dump_source(self, master_gid, gid):
print('Copying source [{0}:{1}]...'.format(master_gid, gid))
# add child gid to pollers first
self.data_d.register_gid(gid)
# add the gid from the list of child accounts
print('Linking GID: [{0}]m <-- [{1}]s'.format(master_gid, gid))
self.data_d.add_linked_account(master_gid, gid)
destinations = self.data.get_destinations(gid)
self.log.debug('{"dest": [')
c = 0
for destination in destinations:
users = self.data.get_destination_users(gid, destination)
for user in users:
if c != 0:
self.log.debug(',')
# dump destination
self.dump_destination(master_gid, gid, destination, user)
c += 1
self.log.debug('],')
# dump gid data keys
self.log.debug('"keys": [')
self.log.debug('"{0},"'.format(S1.gid_key(gid)))
self.log.debug('"{0},"'.format(S1.gid_log_key(gid)))
self.log.debug('"{0},"'.format(S1.links_key(gid)))
self.log.debug('"{0}"'.format(S1.cache_key(gid)))
self.log.debug(']}')
# copy keys
self.copy_hash(S1.gid_key(gid))
self.copy_zset(S1.gid_log_key(gid))
self.copy_hash(S1.cache_key(gid))
self.copy_set(S1.links_key(gid))
# copy tokens for all linked destinations (will overwrite some data)
links = self.data.get_linked_accounts(master_gid) or dict()
for k in links:
# copy token
p = k.split(':')
if not p[0] in self.data.provider:
continue
token = self.data.get_user_token(gid, p[0], p[1])
self.data_d.set_user_token(gid, p[0], p[1], token)
# copy user params
for p_name in S1.PROVIDER_PARAMS:
p_val = self.data.provider[p[0]].get_user_param(p[1], p_name)
if p_val:
self.data_d.provider[p[0]].set_user_param(p[1], p_name, p_val)
def cache_activities_doc(self, gid, activities_doc):
""" stores activities document (google's) into local database (redis)
@param gid: source user id
@param activities_doc: source data dict
"""
self.rc.hset(S1.cache_key(gid), S1.cache_items_key(),
json.dumps(activities_doc, encoding='utf-8'))
def is_poll_after(self, gid, stamp):
polled_str = self.rc.hget(S1.cache_key(gid), S1.polled_key())
if polled_str:
polled = float(polled_str)
if polled >= stamp:
return True
return False
def is_cache(self, gid, option):
#check last requested timestamp
#get last request timestamp and filter
requested_str = self.rc.hget(S1.cache_key(gid),
S1.cache_requested_key())
last_filter = self.rc.hget(S1.cache_key(gid), S1.cache_filter_key())
#save last request time and filter
self.rc.hset(S1.cache_key(gid), S1.cache_requested_key(), time.time())
self.rc.hset(S1.cache_key(gid), S1.cache_filter_key(), option)
if requested_str:
if (not last_filter or option == last_filter) and time.time(
) - float(requested_str) < config.DEFAULT_FORCE_MISS_TIMEOUT:
self.logger.warning('Force-request detected [{0}]'.format(gid))
else:
# cache hit
return True
else:
self.logger.warning('New user detected [{0}]'.format(gid))
# cache miss
return False
def check_orphan(self, gid, at_time):
""" returns true if gid is orphaned """
#get last requested timestamp
requested_str = self.rc.hget(S1.cache_key(gid),
S1.cache_requested_key())
if not requested_str:
return False
# compare to orphaned timeout
if at_time - float(requested_str) < config.DEFAULT_ORPHANED_TIMEOUT:
return False
return True
def dump_gid(self, gid):
print('Dumping user, GID: {0}'.format(gid))
# get child bindings for this account
children = set(self.data.get_destination_users(gid, 'children'))
if not children or (len(children) == 1 and gid in children):
if not self.data.rc.exists(S1.cache_key(gid)):
print('****** SELF CHILD + NO CACHE, SKIPPED, GID: {0}'.format(gid))
return
# just to be safe
children.add(gid)
for child in children:
self.dump_source(gid, child)
def get_poll_stamp(self, gid):
polled_str = self.rc.hget(S1.cache_key(gid), S1.polled_key())
if polled_str:
return float(polled_str)
return 0
def get_activities(self, gid):
str_value = self.rc.hget(S1.cache_key(gid), S1.cache_items_key())
return json.loads(str_value) if str_value else None
def incr_num_minute_updates(self, gid, stamp):
# get the minute of the day
minute = (stamp / 60) % 1440
self.rc.hincrby(S1.cache_key(gid), S1.updated_minute_fmt(minute))
def get_num_minute_updates(self, gid, stamp, spread_minutes):
keys = self.rc.hkeys(S1.cache_key(gid))
return sum([
self.hget_int(S1.cache_key(gid), k) for k in keys
if self._is_key_in_stamp(k, stamp, spread_minutes)
])
def set_gid_max_results(self, gid, max_results):
return self.rc.hset(S1.cache_key(gid), S1.cache_max_results_key(),
max_results)
def get_gid_max_results(self, gid):
return self.get_hfield(S1.cache_key(gid), S1.cache_max_results_key(),
config.DEFAULT_MAX_RESULTS)
def reset_cache(self, gid):
self.set_poll_stamp(gid, 0)
self.rc.hset(S1.cache_key(gid), S1.cache_requested_key(), time.time())
def set_poll_stamp(self, gid, stamp):
self.rc.hset(S1.cache_key(gid), S1.polled_key(), stamp)
