def access_token(self):
access_token = memcache.get(GOOGLE_ACCESS_TOKEN_MEMCACHE_KEY % self.id)
if access_token:
INFO('Found access token %s for user %s' % (
access_token, self.name
))
if not access_token:
args = dict(
client_id=config['GOOGLE_CLIENT_ID'],
client_secret=config['GOOGLE_CLIENT_SECRET'],
refresh_token=self.refresh_token,
grant_type='refresh_token',
)
response = requests.post('https://accounts.google.com/o/oauth2/token', data=args, verify=False)
if 'access_token' not in response.json:
ERROR('There is no token in google response %s, status_code: %s, refresh_token: %s, user.email: %s' % (response.json, response.status_code, self.refresh_token, self.email))
return None
data = response.json
INFO('Received response with access_token for user %s: %s' % (
self.name, data
))
access_token = data['access_token']
expires = data['expires_in']
expires = int(expires / 2)
INFO('Saving access_token %s for user %s in memcached for %s s' % (
access_token, self.name, expires,
))
memcache.set(
GOOGLE_ACCESS_TOKEN_MEMCACHE_KEY % self.id,
access_token,
expires
)
return access_token
def parse_data(self, callback, data):
jdata = json.loads(data)
users = self._get_users(jdata['people'])
projects = self._get_projects(jdata['projects'])
whiteboard_field_numbers = self._get_whiteboard_number(jdata['projects'])
components = self._get_components(jdata.get('components', []))
milestones = self._get_milestones(jdata.get('milestones', []))
custom_fields = self._get_custom_fields(jdata.get('custom_field_values', []))
data = {
'users': users,
'projects': projects,
'components': components,
'milestones': milestones,
'custom_fields': custom_fields,
'whiteboard_field_numbers': whiteboard_field_numbers,
}
self.unfuddle_data = data
memcache.set(
self.UNFUDDLE_DATA_KEY,
data,
timeout=self.UNFUDDLE_DATA_TIMEOUT
)
callback()
def parse_token(self, callback, data):
key = self.TOKEN_MEMCACHE_KEY.format(tracker_type=self.tracker.type, tracker_id=self.tracker.id, user_id=self.user.id)
data = ET.fromstring(data)
self._token = data.find('guid').text
memcache.set(key, self._token, timeout=self.TOKEN_MEMCACHE_TIMEOUT)
callback()
def __init__(self, tracker):
"""
Creates a selector mapping for given tracker
None -> project_id
project_name -> project_id
(project_name, component_name) -> project_id
"""
self.tracker = tracker
self.by_ticket_id = {}
self.default = None
self.by_project = {} # key: project_name
self.by_component = {} # key: project_name, component_name
self.by_version = {} # key: project_name, version
self.by_component_version = {} # key: project_name, component_name, version
cache_key = SELECTOR_CACHE_KEY % tracker.id
mapping = memcache.get(cache_key)
if mapping:
self.clone(mapping)
return
projects = Project.query.filter(Project.tracker_id == tracker.id) \
.filter(Project.turn_off_selectors == False) \
.filter(Project.active == True)
self.projects = dict([(project.id, project.name) for project in projects])
for project in projects:
self._create_for_project(project)
memcache.set(cache_key, self)
DEBUG('Created selector mapping for tracker %s: %s, %s' % (
tracker.id, pformat(self.by_ticket_id), pformat(self.by_component))
)
def __init__(self, tracker):
"""
Creates a selector mapping for given tracker
None -> project_id
project_name -> project_id
(project_name, component_name) -> project_id
"""
self.tracker = tracker
self.by_ticket_id = {}
self.default = None
self.by_project = {} # key: project_name
self.by_component = {} # key: project_name, component_name
self.by_version = {} # key: project_name, version
self.by_component_version = {} # key: project_name, component_name, version
cache_key = SELECTOR_CACHE_KEY % tracker.id
mapping = memcache.get(cache_key)
if mapping:
self.clone(mapping)
return
projects = Project.query.filter(Project.tracker_id == tracker.id) \
.filter(Project.turn_off_selectors == False) \
.filter(Project.active == True)
self.projects = dict([(project.id, project.name) for project in projects])
for project in projects:
self._create_for_project(project)
memcache.set(cache_key, self)
DEBUG('Created selector mapping for tracker %s: %s, %s' % (
tracker.id, pformat(self.by_ticket_id), pformat(self.by_component))
)
def get_story_points_field_id(self):
story_points_key = STORY_POINTS_KEY.format(self.tracker.id)
story_points_field_id = memcache.get(story_points_key)
if story_points_field_id is None:
story_points_field_id = self.query_story_points_field_id()
memcache.set(story_points_key, story_points_field_id)
return story_points_field_id
def __iter__(self):
""" iterate over fetched tickets """
if self.cache_key and self.error is None: # cache bugs if key was designeated and no error occured
memcache.set(self.cache_key, self.bugs, timeout=self.CACHE_TIMEOUT)
DEBUG(u"Cached %s bugs for key %s" % (len(self.bugs), self.cache_key))
for bug in self.bugs.itervalues():
self.resolve(bug)
yield bug
def parse_users(self, callback, data):
jdata = json.loads(data)
login_mapping = dict([ ( user['id'], user['username'] ) for user in jdata])
login_mapping[None] = 'nobody'
self.unfuddle_login_mapping = login_mapping
memcache.set(self.UNFUDDLE_LOGIN_MAPPING_KEY, login_mapping, timeout=self.UNFUDDLE_LOGIN_MAPPING_TIMEOUT)
callback()
def all(cls, cache=True):
holidays = None
if cache:
holidays = memcache.get(HOLIDAYS_MEMCACHE_KEY)
if holidays is None:
holidays = dict([ (date[0], True) for date in DBSession.query(Holiday.date) ])
memcache.set(HOLIDAYS_MEMCACHE_KEY, holidays, HOLIDAYS_MEMCACHE_TIME)
DEBUG(u"Holidays cached")
return holidays
def get_office_ip(cls):
office_ip = memcache.get(OFFICE_IP_MEMCACHE_KEY)
if not office_ip:
config = cls.get_current_config(allow_empty=True)
if config is None:
return
office_ip = [c.strip() for c in config.office_ip.split(',')]
memcache.set(OFFICE_IP_MEMCACHE_KEY, office_ip)
return office_ip
def get_sprint(self, sprint):
bugs = memcache.get(SCRUM_BUG_CACHE_KEY % sprint.id)
if bugs:
return bugs
query = self.request.db_session.query
project_ids = sprint.bugs_project_ids
entries = query(Project, Tracker, TrackerCredentials) \
.filter(Project.id.in_(project_ids)) \
.filter(Project.tracker_id==Tracker.id) \
.filter(TrackerCredentials.tracker_id==Project.tracker_id) \
.filter(TrackerCredentials.user_id==self.user.id).all()
fetchers = []
for project, tracker, creds in entries:
fetcher = get_fetcher(tracker, creds, tracker.logins_mapping)
fetcher.fetch_scrum(sprint.name, project.project_selector, project.component_selector)
fetchers.append(fetcher)
if tracker.type in ('bugzilla', 'rockzilla', 'igozilla'):
break
start = time()
bugs = []
while fetchers:
for i, fetcher in enumerate(fetchers):
if fetcher.isReady():
fetchers.pop(i)
if fetcher.error:
WARN(u"Could not fetch bugs from tracker %s: %s" % (fetcher.tracker.name, fetcher.error))
flash(u"Could not fetch bugs from tracker %s" % (fetcher.tracker.name, ))
else:
for bug in fetcher:
bugs.append(bug)
break
else:
# no fetcher is yet ready, give them time
if time() - start > MAX_TIMEOUT:
WARN(u"Fetchers for trackers %s timed-out" % (u', '.join(fetcher.tracker.name for fetcher in fetchers)))
for fetcher in fetchers:
pass
flash(u"Getting bugs from tracker %s timed out" % (fetcher.tracker.name, ))
fetchers = []
else:
sleep(0.1)
# all bugs were fetched, time to resolve their projects
projects = [bug.project_id for bug in bugs]
projects = dict((project.id, project) for project in Project.query.filter(Project.id.in_(projects)))
# now assign projects to bugs
for bug in bugs:
bug.project = projects.get(bug.project_id)
bugs = self.add_time(bugs, sprint=sprint)
memcache.set(SCRUM_BUG_CACHE_KEY % sprint.id, bugs, SCRUM_BUG_CACHE_TIMEOUT)
return bugs
def parse_milestones(self, callback, data):
milestone_map = {}
json_data = json.loads(data)
for milestone in json_data:
milestone_map[milestone['title']] = str(milestone['number'])
memcache.set(self.MILESTONES_KEY,
milestone_map,
timeout=self.MILESTONES_TIMEOUT)
self.milestones = milestone_map
callback()
def parse_milestones(self, callback, data):
milestone_map = {}
json_data = json.loads(data)
for milestone in json_data:
milestone_map[milestone['title']] = str(milestone['number'])
memcache.set(
self.MILESTONES_KEY,
milestone_map,
timeout=self.MILESTONES_TIMEOUT
)
self.milestones = milestone_map
callback()
def get_sprint(self, sprint):
bugs = memcache.get(SCRUM_BUG_CACHE_KEY % sprint.id)
if bugs:
return bugs
query = self.request.db_session.query
tracker, creds = query(Tracker, TrackerCredentials)\
.filter(TrackerCredentials.tracker_id==sprint.project.tracker_id)\
.filter(TrackerCredentials.tracker_id==Tracker.id)\
.filter(TrackerCredentials.user_id==self.user.id).one()
mapping = TrackerCredentials.get_logins_mapping(tracker)
fetcher = get_fetcher(tracker, creds, mapping)
fetcher.fetch_scrum(sprint.name, sprint.project.project_selector)
start = time()
bugs = []
while True:
if fetcher.isReady():
if fetcher.error:
ERROR(u"Fetcher for tracker %s failed with %r" % (tracker.name, fetcher.error))
break
bugs = [ bug for bug in fetcher ]
break
else: # fetcher is not ready yet
if time() - start > MAX_TIMEOUT:
ERROR(u'Request timed-out')
break
else:
sleep(0.1)
projects = {}
for bug in bugs:
projects[bug.project_id] = None
# query for all project id's
projects = dict((project.id, project) for project in Project.query.filter(Project.id.in_(projects.keys())))
# now assign projects to bugs
for bug in bugs:
if bug.project_id:
bug.project = projects.get(bug.project_id)
else:
bug.project_id = sprint.project_id
bug.project = sprint.project
bugs = self.add_time(bugs, sprint=sprint)
memcache.set(SCRUM_BUG_CACHE_KEY % sprint.id, bugs, SCRUM_BUG_CACHE_TIMEOUT)
return bugs
def parse_data(self, callback, data):
jdata = json.loads(data)
users = self._get_users(jdata["people"])
projects = self._get_projects(jdata["projects"])
components = self._get_components(jdata.get("components", []))
milestones = self._get_milestones(jdata.get("milestones", []))
custom_fields = self._get_custom_fields(jdata.get("custom_field_values", []))
data = {
"users": users,
"projects": projects,
"components": components,
"milestones": milestones,
"custom_fields": custom_fields,
}
self.unfuddle_data = data
memcache.set(self.UNFUDDLE_DATA_KEY, data, timeout=self.UNFUDDLE_DATA_TIMEOUT)
callback()
def access_token(self):
access_token = memcache.get(GOOGLE_ACCESS_TOKEN_MEMCACHE_KEY % self.id)
if access_token:
INFO('Found access token %s for user %s' %
(access_token, self.name))
if not access_token:
args = dict(
client_id=config['GOOGLE_CLIENT_ID'],
client_secret=config['GOOGLE_CLIENT_SECRET'],
refresh_token=self.refresh_token,
grant_type='refresh_token',
)
response = requests.post(
'https://accounts.google.com/o/oauth2/token',
data=args,
verify=False)
if 'access_token' not in response.json:
ERROR(
'There is no token in google response %s, status_code: %s, refresh_token: %s, user.email: %s'
% (response.json, response.status_code, self.refresh_token,
self.email))
return None
data = response.json
INFO('Received response with access_token for user %s: %s' %
(self.name, data))
access_token = data['access_token']
expires = data['expires_in']
expires = int(expires / 2)
INFO('Saving access_token %s for user %s in memcached for %s s' % (
access_token,
self.name,
expires,
))
memcache.set(GOOGLE_ACCESS_TOKEN_MEMCACHE_KEY % self.id,
access_token, expires)
return access_token
def consume(self, rpcs):
if not isinstance(rpcs, list):
rpcs = [rpcs]
if not self._auth_data:
self._auth_data = self.get_auth()
for rpc in rpcs:
self.set_auth(rpc.s, self._auth_data)
self.add_data(rpc.s)
rpc.start()
for rpc in rpcs:
response = rpc.get_result()
self.check_if_failed(response)
self._parsed_data.extend(self.parse(response.text))
self._parsed_data = self.after_parsing(self._parsed_data)
memcache.set(
self._memcached_key,
self._parsed_data,
self.CACHE_TIMEOUT,
)
