def access_token(self): access_token = memcache.get(GOOGLE_ACCESS_TOKEN_MEMCACHE_KEY % self.id) if access_token: INFO('Found access token %s for user %s' % ( access_token, self.name )) if not access_token: args = dict( client_id=config['GOOGLE_CLIENT_ID'], client_secret=config['GOOGLE_CLIENT_SECRET'], refresh_token=self.refresh_token, grant_type='refresh_token', ) response = requests.post('https://accounts.google.com/o/oauth2/token', data=args, verify=False) if 'access_token' not in response.json: ERROR('There is no token in google response %s, status_code: %s, refresh_token: %s, user.email: %s' % (response.json, response.status_code, self.refresh_token, self.email)) return None data = response.json INFO('Received response with access_token for user %s: %s' % ( self.name, data )) access_token = data['access_token'] expires = data['expires_in'] expires = int(expires / 2) INFO('Saving access_token %s for user %s in memcached for %s s' % ( access_token, self.name, expires, )) memcache.set( GOOGLE_ACCESS_TOKEN_MEMCACHE_KEY % self.id, access_token, expires ) return access_token
def parse_data(self, callback, data): jdata = json.loads(data) users = self._get_users(jdata['people']) projects = self._get_projects(jdata['projects']) whiteboard_field_numbers = self._get_whiteboard_number(jdata['projects']) components = self._get_components(jdata.get('components', [])) milestones = self._get_milestones(jdata.get('milestones', [])) custom_fields = self._get_custom_fields(jdata.get('custom_field_values', [])) data = { 'users': users, 'projects': projects, 'components': components, 'milestones': milestones, 'custom_fields': custom_fields, 'whiteboard_field_numbers': whiteboard_field_numbers, } self.unfuddle_data = data memcache.set( self.UNFUDDLE_DATA_KEY, data, timeout=self.UNFUDDLE_DATA_TIMEOUT ) callback()
def parse_token(self, callback, data): key = self.TOKEN_MEMCACHE_KEY.format(tracker_type=self.tracker.type, tracker_id=self.tracker.id, user_id=self.user.id) data = ET.fromstring(data) self._token = data.find('guid').text memcache.set(key, self._token, timeout=self.TOKEN_MEMCACHE_TIMEOUT) callback()
def __init__(self, tracker): """ Creates a selector mapping for given tracker None -> project_id project_name -> project_id (project_name, component_name) -> project_id """ self.tracker = tracker self.by_ticket_id = {} self.default = None self.by_project = {} # key: project_name self.by_component = {} # key: project_name, component_name self.by_version = {} # key: project_name, version self.by_component_version = {} # key: project_name, component_name, version cache_key = SELECTOR_CACHE_KEY % tracker.id mapping = memcache.get(cache_key) if mapping: self.clone(mapping) return projects = Project.query.filter(Project.tracker_id == tracker.id) \ .filter(Project.turn_off_selectors == False) \ .filter(Project.active == True) self.projects = dict([(project.id, project.name) for project in projects]) for project in projects: self._create_for_project(project) memcache.set(cache_key, self) DEBUG('Created selector mapping for tracker %s: %s, %s' % ( tracker.id, pformat(self.by_ticket_id), pformat(self.by_component)) )
def get_story_points_field_id(self): story_points_key = STORY_POINTS_KEY.format(self.tracker.id) story_points_field_id = memcache.get(story_points_key) if story_points_field_id is None: story_points_field_id = self.query_story_points_field_id() memcache.set(story_points_key, story_points_field_id) return story_points_field_id
def __iter__(self): """ iterate over fetched tickets """ if self.cache_key and self.error is None: # cache bugs if key was designeated and no error occured memcache.set(self.cache_key, self.bugs, timeout=self.CACHE_TIMEOUT) DEBUG(u"Cached %s bugs for key %s" % (len(self.bugs), self.cache_key)) for bug in self.bugs.itervalues(): self.resolve(bug) yield bug
def parse_users(self, callback, data): jdata = json.loads(data) login_mapping = dict([ ( user['id'], user['username'] ) for user in jdata]) login_mapping[None] = 'nobody' self.unfuddle_login_mapping = login_mapping memcache.set(self.UNFUDDLE_LOGIN_MAPPING_KEY, login_mapping, timeout=self.UNFUDDLE_LOGIN_MAPPING_TIMEOUT) callback()
def all(cls, cache=True): holidays = None if cache: holidays = memcache.get(HOLIDAYS_MEMCACHE_KEY) if holidays is None: holidays = dict([ (date[0], True) for date in DBSession.query(Holiday.date) ]) memcache.set(HOLIDAYS_MEMCACHE_KEY, holidays, HOLIDAYS_MEMCACHE_TIME) DEBUG(u"Holidays cached") return holidays
def get_office_ip(cls): office_ip = memcache.get(OFFICE_IP_MEMCACHE_KEY) if not office_ip: config = cls.get_current_config(allow_empty=True) if config is None: return office_ip = [c.strip() for c in config.office_ip.split(',')] memcache.set(OFFICE_IP_MEMCACHE_KEY, office_ip) return office_ip
def get_sprint(self, sprint): bugs = memcache.get(SCRUM_BUG_CACHE_KEY % sprint.id) if bugs: return bugs query = self.request.db_session.query project_ids = sprint.bugs_project_ids entries = query(Project, Tracker, TrackerCredentials) \ .filter(Project.id.in_(project_ids)) \ .filter(Project.tracker_id==Tracker.id) \ .filter(TrackerCredentials.tracker_id==Project.tracker_id) \ .filter(TrackerCredentials.user_id==self.user.id).all() fetchers = [] for project, tracker, creds in entries: fetcher = get_fetcher(tracker, creds, tracker.logins_mapping) fetcher.fetch_scrum(sprint.name, project.project_selector, project.component_selector) fetchers.append(fetcher) if tracker.type in ('bugzilla', 'rockzilla', 'igozilla'): break start = time() bugs = [] while fetchers: for i, fetcher in enumerate(fetchers): if fetcher.isReady(): fetchers.pop(i) if fetcher.error: WARN(u"Could not fetch bugs from tracker %s: %s" % (fetcher.tracker.name, fetcher.error)) flash(u"Could not fetch bugs from tracker %s" % (fetcher.tracker.name, )) else: for bug in fetcher: bugs.append(bug) break else: # no fetcher is yet ready, give them time if time() - start > MAX_TIMEOUT: WARN(u"Fetchers for trackers %s timed-out" % (u', '.join(fetcher.tracker.name for fetcher in fetchers))) for fetcher in fetchers: pass flash(u"Getting bugs from tracker %s timed out" % (fetcher.tracker.name, )) fetchers = [] else: sleep(0.1) # all bugs were fetched, time to resolve their projects projects = [bug.project_id for bug in bugs] projects = dict((project.id, project) for project in Project.query.filter(Project.id.in_(projects))) # now assign projects to bugs for bug in bugs: bug.project = projects.get(bug.project_id) bugs = self.add_time(bugs, sprint=sprint) memcache.set(SCRUM_BUG_CACHE_KEY % sprint.id, bugs, SCRUM_BUG_CACHE_TIMEOUT) return bugs
def parse_milestones(self, callback, data): milestone_map = {} json_data = json.loads(data) for milestone in json_data: milestone_map[milestone['title']] = str(milestone['number']) memcache.set(self.MILESTONES_KEY, milestone_map, timeout=self.MILESTONES_TIMEOUT) self.milestones = milestone_map callback()
def parse_milestones(self, callback, data): milestone_map = {} json_data = json.loads(data) for milestone in json_data: milestone_map[milestone['title']] = str(milestone['number']) memcache.set( self.MILESTONES_KEY, milestone_map, timeout=self.MILESTONES_TIMEOUT ) self.milestones = milestone_map callback()
def get_sprint(self, sprint): bugs = memcache.get(SCRUM_BUG_CACHE_KEY % sprint.id) if bugs: return bugs query = self.request.db_session.query tracker, creds = query(Tracker, TrackerCredentials)\ .filter(TrackerCredentials.tracker_id==sprint.project.tracker_id)\ .filter(TrackerCredentials.tracker_id==Tracker.id)\ .filter(TrackerCredentials.user_id==self.user.id).one() mapping = TrackerCredentials.get_logins_mapping(tracker) fetcher = get_fetcher(tracker, creds, mapping) fetcher.fetch_scrum(sprint.name, sprint.project.project_selector) start = time() bugs = [] while True: if fetcher.isReady(): if fetcher.error: ERROR(u"Fetcher for tracker %s failed with %r" % (tracker.name, fetcher.error)) break bugs = [ bug for bug in fetcher ] break else: # fetcher is not ready yet if time() - start > MAX_TIMEOUT: ERROR(u'Request timed-out') break else: sleep(0.1) projects = {} for bug in bugs: projects[bug.project_id] = None # query for all project id's projects = dict((project.id, project) for project in Project.query.filter(Project.id.in_(projects.keys()))) # now assign projects to bugs for bug in bugs: if bug.project_id: bug.project = projects.get(bug.project_id) else: bug.project_id = sprint.project_id bug.project = sprint.project bugs = self.add_time(bugs, sprint=sprint) memcache.set(SCRUM_BUG_CACHE_KEY % sprint.id, bugs, SCRUM_BUG_CACHE_TIMEOUT) return bugs
def parse_data(self, callback, data): jdata = json.loads(data) users = self._get_users(jdata["people"]) projects = self._get_projects(jdata["projects"]) components = self._get_components(jdata.get("components", [])) milestones = self._get_milestones(jdata.get("milestones", [])) custom_fields = self._get_custom_fields(jdata.get("custom_field_values", [])) data = { "users": users, "projects": projects, "components": components, "milestones": milestones, "custom_fields": custom_fields, } self.unfuddle_data = data memcache.set(self.UNFUDDLE_DATA_KEY, data, timeout=self.UNFUDDLE_DATA_TIMEOUT) callback()
def access_token(self): access_token = memcache.get(GOOGLE_ACCESS_TOKEN_MEMCACHE_KEY % self.id) if access_token: INFO('Found access token %s for user %s' % (access_token, self.name)) if not access_token: args = dict( client_id=config['GOOGLE_CLIENT_ID'], client_secret=config['GOOGLE_CLIENT_SECRET'], refresh_token=self.refresh_token, grant_type='refresh_token', ) response = requests.post( 'https://accounts.google.com/o/oauth2/token', data=args, verify=False) if 'access_token' not in response.json: ERROR( 'There is no token in google response %s, status_code: %s, refresh_token: %s, user.email: %s' % (response.json, response.status_code, self.refresh_token, self.email)) return None data = response.json INFO('Received response with access_token for user %s: %s' % (self.name, data)) access_token = data['access_token'] expires = data['expires_in'] expires = int(expires / 2) INFO('Saving access_token %s for user %s in memcached for %s s' % ( access_token, self.name, expires, )) memcache.set(GOOGLE_ACCESS_TOKEN_MEMCACHE_KEY % self.id, access_token, expires) return access_token
def consume(self, rpcs): if not isinstance(rpcs, list): rpcs = [rpcs] if not self._auth_data: self._auth_data = self.get_auth() for rpc in rpcs: self.set_auth(rpc.s, self._auth_data) self.add_data(rpc.s) rpc.start() for rpc in rpcs: response = rpc.get_result() self.check_if_failed(response) self._parsed_data.extend(self.parse(response.text)) self._parsed_data = self.after_parsing(self._parsed_data) memcache.set( self._memcached_key, self._parsed_data, self.CACHE_TIMEOUT, )