def dump_source(self, master_gid, gid): print('Copying source [{0}:{1}]...'.format(master_gid, gid)) # add child gid to pollers first self.data_d.register_gid(gid) # add the gid from the list of child accounts print('Linking GID: [{0}]m <-- [{1}]s'.format(master_gid, gid)) self.data_d.add_linked_account(master_gid, gid) destinations = self.data.get_destinations(gid) self.log.debug('{"dest": [') c = 0 for destination in destinations: users = self.data.get_destination_users(gid, destination) for user in users: if c != 0: self.log.debug(',') # dump destination self.dump_destination(master_gid, gid, destination, user) c += 1 self.log.debug('],') # dump gid data keys self.log.debug('"keys": [') self.log.debug('"{0},"'.format(S1.gid_key(gid))) self.log.debug('"{0},"'.format(S1.gid_log_key(gid))) self.log.debug('"{0},"'.format(S1.links_key(gid))) self.log.debug('"{0}"'.format(S1.cache_key(gid))) self.log.debug(']}') # copy keys self.copy_hash(S1.gid_key(gid)) self.copy_zset(S1.gid_log_key(gid)) self.copy_hash(S1.cache_key(gid)) self.copy_set(S1.links_key(gid)) # copy tokens for all linked destinations (will overwrite some data) links = self.data.get_linked_accounts(master_gid) or dict() for k in links: # copy token p = k.split(':') if not p[0] in self.data.provider: continue token = self.data.get_user_token(gid, p[0], p[1]) self.data_d.set_user_token(gid, p[0], p[1], token) # copy user params for p_name in S1.PROVIDER_PARAMS: p_val = self.data.provider[p[0]].get_user_param(p[1], p_name) if p_val: self.data_d.provider[p[0]].set_user_param(p[1], p_name, p_val)
def cache_activities_doc(self, gid, activities_doc): """ stores activities document (google's) into local database (redis) @param gid: source user id @param activities_doc: source data dict """ self.rc.hset(S1.cache_key(gid), S1.cache_items_key(), json.dumps(activities_doc, encoding='utf-8'))
def is_poll_after(self, gid, stamp): polled_str = self.rc.hget(S1.cache_key(gid), S1.polled_key()) if polled_str: polled = float(polled_str) if polled >= stamp: return True return False
def is_cache(self, gid, option): #check last requested timestamp #get last request timestamp and filter requested_str = self.rc.hget(S1.cache_key(gid), S1.cache_requested_key()) last_filter = self.rc.hget(S1.cache_key(gid), S1.cache_filter_key()) #save last request time and filter self.rc.hset(S1.cache_key(gid), S1.cache_requested_key(), time.time()) self.rc.hset(S1.cache_key(gid), S1.cache_filter_key(), option) if requested_str: if (not last_filter or option == last_filter) and time.time( ) - float(requested_str) < config.DEFAULT_FORCE_MISS_TIMEOUT: self.logger.warning('Force-request detected [{0}]'.format(gid)) else: # cache hit return True else: self.logger.warning('New user detected [{0}]'.format(gid)) # cache miss return False
def check_orphan(self, gid, at_time): """ returns true if gid is orphaned """ #get last requested timestamp requested_str = self.rc.hget(S1.cache_key(gid), S1.cache_requested_key()) if not requested_str: return False # compare to orphaned timeout if at_time - float(requested_str) < config.DEFAULT_ORPHANED_TIMEOUT: return False return True
def dump_gid(self, gid): print('Dumping user, GID: {0}'.format(gid)) # get child bindings for this account children = set(self.data.get_destination_users(gid, 'children')) if not children or (len(children) == 1 and gid in children): if not self.data.rc.exists(S1.cache_key(gid)): print('****** SELF CHILD + NO CACHE, SKIPPED, GID: {0}'.format(gid)) return # just to be safe children.add(gid) for child in children: self.dump_source(gid, child)
def get_poll_stamp(self, gid): polled_str = self.rc.hget(S1.cache_key(gid), S1.polled_key()) if polled_str: return float(polled_str) return 0
def get_activities(self, gid): str_value = self.rc.hget(S1.cache_key(gid), S1.cache_items_key()) return json.loads(str_value) if str_value else None
def incr_num_minute_updates(self, gid, stamp): # get the minute of the day minute = (stamp / 60) % 1440 self.rc.hincrby(S1.cache_key(gid), S1.updated_minute_fmt(minute))
def get_num_minute_updates(self, gid, stamp, spread_minutes): keys = self.rc.hkeys(S1.cache_key(gid)) return sum([ self.hget_int(S1.cache_key(gid), k) for k in keys if self._is_key_in_stamp(k, stamp, spread_minutes) ])
def set_gid_max_results(self, gid, max_results): return self.rc.hset(S1.cache_key(gid), S1.cache_max_results_key(), max_results)
def get_gid_max_results(self, gid): return self.get_hfield(S1.cache_key(gid), S1.cache_max_results_key(), config.DEFAULT_MAX_RESULTS)
def reset_cache(self, gid): self.set_poll_stamp(gid, 0) self.rc.hset(S1.cache_key(gid), S1.cache_requested_key(), time.time())
def set_poll_stamp(self, gid, stamp): self.rc.hset(S1.cache_key(gid), S1.polled_key(), stamp)