def access_token(self):
access_token = memcache.get(GOOGLE_ACCESS_TOKEN_MEMCACHE_KEY % self.id)
if access_token:
INFO('Found access token %s for user %s' % (
access_token, self.name
))
if not access_token:
args = dict(
client_id=config['GOOGLE_CLIENT_ID'],
client_secret=config['GOOGLE_CLIENT_SECRET'],
refresh_token=self.refresh_token,
grant_type='refresh_token',
)
response = requests.post('https://accounts.google.com/o/oauth2/token', data=args, verify=False)
if 'access_token' not in response.json:
ERROR('There is no token in google response %s, status_code: %s, refresh_token: %s, user.email: %s' % (response.json, response.status_code, self.refresh_token, self.email))
return None
data = response.json
INFO('Received response with access_token for user %s: %s' % (
self.name, data
))
access_token = data['access_token']
expires = data['expires_in']
expires = int(expires / 2)
INFO('Saving access_token %s for user %s in memcached for %s s' % (
access_token, self.name, expires,
))
memcache.set(
GOOGLE_ACCESS_TOKEN_MEMCACHE_KEY % self.id,
access_token,
expires
)
return access_token
def get_story_points_field_id(self):
story_points_key = STORY_POINTS_KEY.format(self.tracker.id)
story_points_field_id = memcache.get(story_points_key)
if story_points_field_id is None:
story_points_field_id = self.query_story_points_field_id()
memcache.set(story_points_key, story_points_field_id)
return story_points_field_id
def __init__(self, tracker):
"""
Creates a selector mapping for given tracker
None -> project_id
project_name -> project_id
(project_name, component_name) -> project_id
"""
self.tracker = tracker
self.by_ticket_id = {}
self.default = None
self.by_project = {} # key: project_name
self.by_component = {} # key: project_name, component_name
self.by_version = {} # key: project_name, version
self.by_component_version = {} # key: project_name, component_name, version
cache_key = SELECTOR_CACHE_KEY % tracker.id
mapping = memcache.get(cache_key)
if mapping:
self.clone(mapping)
return
projects = Project.query.filter(Project.tracker_id == tracker.id) \
.filter(Project.turn_off_selectors == False) \
.filter(Project.active == True)
self.projects = dict([(project.id, project.name) for project in projects])
for project in projects:
self._create_for_project(project)
memcache.set(cache_key, self)
DEBUG('Created selector mapping for tracker %s: %s, %s' % (
tracker.id, pformat(self.by_ticket_id), pformat(self.by_component))
)
def get_story_points_field_id(self):
story_points_key = STORY_POINTS_KEY.format(self.tracker.id)
story_points_field_id = memcache.get(story_points_key)
if story_points_field_id is None:
story_points_field_id = self.query_story_points_field_id()
memcache.set(story_points_key, story_points_field_id)
return story_points_field_id
def __init__(self, tracker):
"""
Creates a selector mapping for given tracker
None -> project_id
project_name -> project_id
(project_name, component_name) -> project_id
"""
self.tracker = tracker
self.by_ticket_id = {}
self.default = None
self.by_project = {} # key: project_name
self.by_component = {} # key: project_name, component_name
self.by_version = {} # key: project_name, version
self.by_component_version = {} # key: project_name, component_name, version
cache_key = SELECTOR_CACHE_KEY % tracker.id
mapping = memcache.get(cache_key)
if mapping:
self.clone(mapping)
return
projects = Project.query.filter(Project.tracker_id == tracker.id) \
.filter(Project.turn_off_selectors == False) \
.filter(Project.active == True)
self.projects = dict([(project.id, project.name) for project in projects])
for project in projects:
self._create_for_project(project)
memcache.set(cache_key, self)
DEBUG('Created selector mapping for tracker %s: %s, %s' % (
tracker.id, pformat(self.by_ticket_id), pformat(self.by_component))
)
def get_sprint(self, sprint):
bugs = memcache.get(SCRUM_BUG_CACHE_KEY % sprint.id)
if bugs:
return bugs
query = self.request.db_session.query
project_ids = sprint.bugs_project_ids
entries = query(Project, Tracker, TrackerCredentials) \
.filter(Project.id.in_(project_ids)) \
.filter(Project.tracker_id==Tracker.id) \
.filter(TrackerCredentials.tracker_id==Project.tracker_id) \
.filter(TrackerCredentials.user_id==self.user.id).all()
fetchers = []
for project, tracker, creds in entries:
fetcher = get_fetcher(tracker, creds, tracker.logins_mapping)
fetcher.fetch_scrum(sprint.name, project.project_selector, project.component_selector)
fetchers.append(fetcher)
if tracker.type in ('bugzilla', 'rockzilla', 'igozilla'):
break
start = time()
bugs = []
while fetchers:
for i, fetcher in enumerate(fetchers):
if fetcher.isReady():
fetchers.pop(i)
if fetcher.error:
WARN(u"Could not fetch bugs from tracker %s: %s" % (fetcher.tracker.name, fetcher.error))
flash(u"Could not fetch bugs from tracker %s" % (fetcher.tracker.name, ))
else:
for bug in fetcher:
bugs.append(bug)
break
else:
# no fetcher is yet ready, give them time
if time() - start > MAX_TIMEOUT:
WARN(u"Fetchers for trackers %s timed-out" % (u', '.join(fetcher.tracker.name for fetcher in fetchers)))
for fetcher in fetchers:
pass
flash(u"Getting bugs from tracker %s timed out" % (fetcher.tracker.name, ))
fetchers = []
else:
sleep(0.1)
# all bugs were fetched, time to resolve their projects
projects = [bug.project_id for bug in bugs]
projects = dict((project.id, project) for project in Project.query.filter(Project.id.in_(projects)))
# now assign projects to bugs
for bug in bugs:
bug.project = projects.get(bug.project_id)
bugs = self.add_time(bugs, sprint=sprint)
memcache.set(SCRUM_BUG_CACHE_KEY % sprint.id, bugs, SCRUM_BUG_CACHE_TIMEOUT)
return bugs
def get_office_ip(cls):
office_ip = memcache.get(OFFICE_IP_MEMCACHE_KEY)
if not office_ip:
config = cls.get_current_config(allow_empty=True)
if config is None:
return
office_ip = [c.strip() for c in config.office_ip.split(',')]
memcache.set(OFFICE_IP_MEMCACHE_KEY, office_ip)
return office_ip
def get_data(self, callback):
if not self.unfuddle_data:
data = memcache.get(self.UNFUDDLE_DATA_KEY)
if not data:
self.fetch_data(callback)
return
else:
self.unfuddle_data = data
callback()
def get(self):
date = self.request.GET.get('date')
if date:
date = datetime.datetime.strptime(date, '%d.%m.%Y').date()
else:
date = datetime.date.today()
current_data_late = memcache.get(MEMCACHED_NOTIFY_KEY % date)
blacklist = self.request.user.notify_blacklist
if current_data_late is not None:
current_data_late['blacklist'] = blacklist
return current_data_late
late_query = DBSession.query(User.id, User.name, Late.id,
Late.late_start, Late.late_end,
Late.explanation, Late.work_from_home)
late_query = late_query.filter(User.id == Late.user_id)\
.filter(Late.date == date)\
.filter(Late.deleted == False)\
.order_by(User.name)
absences = DBSession.query(User.id, User.name, Absence.id,
Absence.date_start, Absence.date_end,
Absence.remarks)
absences = absences.filter(User.id == Absence.user_id)\
.filter(Absence.deleted == False)\
.filter(Absence.date_start <= date)\
.filter(Absence.date_end >= date)\
.order_by(User.name)
current_data_late = dict(
lates=[
dict(id=user_id,
name=user_name,
late_id=late_id,
start=start and start.isoformat()[:5] or None,
end=end and end.isoformat()[:5] or None,
explanation=explanation,
work_from_home=work_from_home) for user_id, user_name,
late_id, start, end, explanation, work_from_home in late_query
],
absences=[
dict(id=user_id,
name=user_name,
absence_id=absence_id,
start=date_start and date_start.strftime('%d.%m.%y')
or None,
end=date_end and date_end.strftime('%d.%m.%y') or None,
remarks=remarks) for user_id, user_name, absence_id,
date_start, date_end, remarks in absences
])
memcache.add(
MEMCACHED_NOTIFY_KEY % date,
current_data_late,
60 * 60 * 24,
)
current_data_late['blacklist'] = blacklist
return current_data_late
def get_users(self, callback):
if not self.unfuddle_login_mapping:
login_mapping = memcache.get(self.UNFUDDLE_LOGIN_MAPPING_KEY)
if not login_mapping:
self.fetch_users(callback)
return
else:
self.unfuddle_login_mapping = login_mapping
callback()
def all(cls, cache=True):
holidays = None
if cache:
holidays = memcache.get(HOLIDAYS_MEMCACHE_KEY)
if holidays is None:
holidays = dict([ (date[0], True) for date in DBSession.query(Holiday.date) ])
memcache.set(HOLIDAYS_MEMCACHE_KEY, holidays, HOLIDAYS_MEMCACHE_TIME)
DEBUG(u"Holidays cached")
return holidays
def get_users(self, callback):
if not self.unfuddle_login_mapping:
login_mapping = memcache.get(self.UNFUDDLE_LOGIN_MAPPING_KEY)
if not login_mapping:
self.fetch_users(callback)
return
else:
self.unfuddle_login_mapping = login_mapping
callback()
def get_office_ip(cls):
office_ip = memcache.get(OFFICE_IP_MEMCACHE_KEY)
if not office_ip:
config = cls.get_current_config(allow_empty=True)
if config is None:
return
office_ip = [c.strip() for c in config.office_ip.split(',')]
memcache.set(OFFICE_IP_MEMCACHE_KEY, office_ip)
return office_ip
def get_token(self, callback):
if not self._token:
key = self.TOKEN_MEMCACHE_KEY.format(tracker_type=self.tracker.type, tracker_id=self.tracker.id, user_id=self.user.id)
token = memcache.get(key)
if not token:
self.fetch_token(callback)
else:
self._token = token
callback()
else:
callback()
def fetcher(this, *args, **kwargs):
query = self.key_generator(*args, **kwargs)
key = u"BUGS_LOGIN-%s_TRACKERID-%s_QUERY-%s" % (this.login, this.tracker.id, query)
bugs = memcache.get(key)
if bugs is None: # fetch as usual
DEBUG(u"Bugs not in cache for key %s" % (key, ))
this.cache_key = key # mark where to cache results
func(this, *args, **kwargs)
else: # bugs got from cache
DEBUG(u"Bugs found in cache for key %s" % (key, ))
this.bugs = bugs
this.success()
def fetcher(this, *args, **kwargs):
query = self.key_generator(*args, **kwargs)
key = u"BUGS_LOGIN-%s_TRACKERID-%s_QUERY-%s" % (this.login, this.tracker.id, query)
bugs = memcache.get(key)
if bugs is None: # fetch as usual
DEBUG(u"Bugs not in cache for key %s" % (key, ))
this.cache_key = key.replace(' ', '') # mark where to cache results
func(this, *args, **kwargs)
else: # bugs got from cache
DEBUG(u"Bugs found in cache for key %s" % (key, ))
this.bugs = bugs
this.success()
def fetch_milestones(self, callback, url):
if self.milestones is None:
self.milestones = memcache.get(self.MILESTONES_KEY)
if self.milestones is None:
url = str(url)
self.request(
url,
self.get_headers(),
partial(self.responded, on_success=partial(self.parse_milestones, callback)),
)
return
else:
callback()
def fetch_milestones(self, callback, url):
if self.milestones is None:
self.milestones = memcache.get(self.MILESTONES_KEY)
if self.milestones is None:
url = str(url)
self.request(
url,
self.get_headers(),
partial(self.responded,
on_success=partial(self.parse_milestones,
callback)),
)
return
else:
callback()
def get_sprint(self, sprint):
bugs = memcache.get(SCRUM_BUG_CACHE_KEY % sprint.id)
if bugs:
return bugs
query = self.request.db_session.query
tracker, creds = query(Tracker, TrackerCredentials)\
.filter(TrackerCredentials.tracker_id==sprint.project.tracker_id)\
.filter(TrackerCredentials.tracker_id==Tracker.id)\
.filter(TrackerCredentials.user_id==self.user.id).one()
mapping = TrackerCredentials.get_logins_mapping(tracker)
fetcher = get_fetcher(tracker, creds, mapping)
fetcher.fetch_scrum(sprint.name, sprint.project.project_selector)
start = time()
bugs = []
while True:
if fetcher.isReady():
if fetcher.error:
ERROR(u"Fetcher for tracker %s failed with %r" %
(tracker.name, fetcher.error))
break
bugs = [bug for bug in fetcher]
break
else: # fetcher is not ready yet
if time() - start > MAX_TIMEOUT:
ERROR(u'Request timed-out')
break
else:
sleep(0.1)
projects = {}
for bug in bugs:
projects[bug.project_id] = None
# query for all project id's
projects = dict((project.id, project)
for project in Project.query.filter(
Project.id.in_(projects.keys())))
# now assign projects to bugs
for bug in bugs:
bug.project = projects.get(bug.project_id)
bugs = self.add_time(bugs, sprint=sprint)
memcache.set(SCRUM_BUG_CACHE_KEY % sprint.id, bugs,
SCRUM_BUG_CACHE_TIMEOUT)
return bugs
def get_sprint(self, sprint):
bugs = memcache.get(SCRUM_BUG_CACHE_KEY % sprint.id)
if bugs:
return bugs
query = self.request.db_session.query
tracker, creds = query(Tracker, TrackerCredentials)\
.filter(TrackerCredentials.tracker_id==sprint.project.tracker_id)\
.filter(TrackerCredentials.tracker_id==Tracker.id)\
.filter(TrackerCredentials.user_id==self.user.id).one()
mapping = TrackerCredentials.get_logins_mapping(tracker)
fetcher = get_fetcher(tracker, creds, mapping)
fetcher.fetch_scrum(sprint.name, sprint.project.project_selector)
start = time()
bugs = []
while True:
if fetcher.isReady():
if fetcher.error:
ERROR(u"Fetcher for tracker %s failed with %r" % (tracker.name, fetcher.error))
break
bugs = [ bug for bug in fetcher ]
break
else: # fetcher is not ready yet
if time() - start > MAX_TIMEOUT:
ERROR(u'Request timed-out')
break
else:
sleep(0.1)
projects = {}
for bug in bugs:
projects[bug.project_id] = None
# query for all project id's
projects = dict((project.id, project) for project in Project.query.filter(Project.id.in_(projects.keys())))
# now assign projects to bugs
for bug in bugs:
if bug.project_id:
bug.project = projects.get(bug.project_id)
else:
bug.project_id = sprint.project_id
bug.project = sprint.project
bugs = self.add_time(bugs, sprint=sprint)
memcache.set(SCRUM_BUG_CACHE_KEY % sprint.id, bugs, SCRUM_BUG_CACHE_TIMEOUT)
return bugs
def func(*args, **kwargs):
self = args[0]
self._memcached_key = mcs._gen_memcached_key(
self.tracker.id,
self.login,
f.func_name,
args[1:],
kwargs,
)
# clear fetcher
self._parsed_data = []
cached = memcache.get(self._memcached_key)
if cached is not None:
DEBUG(u"Bugs found in cache for key %s" % self._memcached_key)
self._parsed_data = cached
else:
# start greenlet
DEBUG(u"Bugs not in cache for key %s" % self._memcached_key)
self.before_fetch()
self._greenlet = Greenlet.spawn(f, *args, **kwargs)
def access_token(self):
access_token = memcache.get(GOOGLE_ACCESS_TOKEN_MEMCACHE_KEY % self.id)
if access_token:
INFO('Found access token %s for user %s' %
(access_token, self.name))
if not access_token:
args = dict(
client_id=config['GOOGLE_CLIENT_ID'],
client_secret=config['GOOGLE_CLIENT_SECRET'],
refresh_token=self.refresh_token,
grant_type='refresh_token',
)
response = requests.post(
'https://accounts.google.com/o/oauth2/token',
data=args,
verify=False)
if 'access_token' not in response.json:
ERROR(
'There is no token in google response %s, status_code: %s, refresh_token: %s, user.email: %s'
% (response.json, response.status_code, self.refresh_token,
self.email))
return None
data = response.json
INFO('Received response with access_token for user %s: %s' %
(self.name, data))
access_token = data['access_token']
expires = data['expires_in']
expires = int(expires / 2)
INFO('Saving access_token %s for user %s in memcached for %s s' % (
access_token,
self.name,
expires,
))
memcache.set(GOOGLE_ACCESS_TOKEN_MEMCACHE_KEY % self.id,
access_token, expires)
return access_token
def func(*args, **kwargs):
self = args[0]
self._memcached_key = mcs._gen_memcached_key(
self.tracker.id,
self.login,
f.func_name,
args[1:],
kwargs,
)
# clear fetcher
self._parsed_data = []
cached = memcache.get(self._memcached_key)
if cached is not None:
DEBUG(u"Bugs found in cache for key %s" % self._memcached_key)
self._parsed_data = cached
else:
# start greenlet
DEBUG(u"Bugs not in cache for key %s" % self._memcached_key)
self._greenlet = Greenlet.spawn(self.before_fetch)
self._greenlet.join(self.MAX_TIMEOUT)
if self._greenlet.successful():
self._greenlet = Greenlet.spawn(f, *args, **kwargs)
def get(self):
date = self.request.GET.get('date')
if date:
date = datetime.datetime.strptime(date, '%d.%m.%Y').date()
else:
date = datetime.date.today()
current_data_late = memcache.get(
MEMCACHED_NOTIFY_KEY % date
)
blacklist = self.request.user.notify_blacklist
if current_data_late is not None:
current_data_late['blacklist'] = blacklist
return current_data_late
late_query = DBSession.query(
User.id,
User.name,
Late.id,
Late.late_start,
Late.late_end,
Late.explanation,
Late.work_from_home
)
late_query = late_query.filter(User.id == Late.user_id)\
.filter(Late.date == date)\
.filter(Late.deleted == False)\
.order_by(User.name)
absences = DBSession.query(
User.id,
User.name,
Absence.id,
Absence.date_start,
Absence.date_end,
Absence.remarks
)
absences = absences.filter(User.id == Absence.user_id)\
.filter(Absence.deleted == False)\
.filter(Absence.date_start <= date)\
.filter(Absence.date_end >= date)\
.order_by(User.name)
current_data_late = dict(
lates=[
dict(
id=user_id,
name=user_name,
late_id=late_id,
start=start and start.isoformat()[:5] or None,
end=end and end.isoformat()[:5] or None,
explanation=explanation,
work_from_home=work_from_home
)for user_id, user_name, late_id, start, end,
explanation, work_from_home in late_query
],
absences=[
dict(
id=user_id,
name=user_name,
absence_id=absence_id,
start=date_start and date_start.strftime('%d.%m.%y') or None,
end=date_end and date_end.strftime('%d.%m.%y') or None,
remarks=remarks
)
for user_id, user_name, absence_id, date_start, date_end, remarks in absences
]
)
memcache.add(
MEMCACHED_NOTIFY_KEY % date,
current_data_late,
60 * 60 * 24,
)
current_data_late['blacklist'] = blacklist
return current_data_late
