def get_project(self, project, resolved=False, credentials=None):
"""
Get bugs for given project
"""
start = time()
bugs = []
tracker = project.tracker
credentials = credentials or self._get_credentials(tracker.id)
if not credentials:
return []
login_mapping = TrackerCredentials.get_logins_mapping(tracker)
fetcher = get_fetcher(tracker, credentials, login_mapping)
(fetcher.fetch_resolved_bugs_for_query if resolved else fetcher.fetch_bugs_for_query)(*project.get_selector_tuple())
while True:
if fetcher.isReady():
if fetcher.error:
WARN(u"Could not fetch bugs from tracker %s: %s" % (tracker.name, fetcher.error))
flash(u"Could not fetch bugs from tracker %s" % (tracker.name, ))
else:
for bug in fetcher:
bug.project = project
bugs.append(bug)
break
elif time() - start > MAX_TIMEOUT:
WARN(u"Fetchers for tracker %s timed-out" % (tracker.name, ))
flash(u"Getting bugs from tracker %s timed out" % (tracker.name, ))
else:
sleep(0.1)
bugs = self.add_time(bugs)
return bugs
def get_project(self, project, resolved=False, credentials=None):
"""
Get bugs for given project
"""
start = time()
bugs = []
tracker = project.tracker
credentials = credentials or self._get_credentials(tracker.id)
if not credentials:
return []
login_mapping = TrackerCredentials.get_logins_mapping(tracker)
fetcher = get_fetcher(tracker, credentials, login_mapping)
(fetcher.fetch_resolved_bugs_for_query if resolved else fetcher.fetch_bugs_for_query)(*project.get_selector_tuple())
while True:
if fetcher.isReady():
if fetcher.error:
WARN(u"Could not fetch bugs from tracker %s: %s" % (tracker.name, fetcher.error))
flash(u"Could not fetch bugs from tracker %s" % (tracker.name, ))
else:
for bug in fetcher:
bug.project = project
bugs.append(bug)
break
elif time() - start > MAX_TIMEOUT:
WARN(u"Fetchers for tracker %s timed-out" % (tracker.name, ))
flash(u"Getting bugs from tracker %s timed out" % (tracker.name, ))
else:
sleep(0.1)
bugs = self.add_time(bugs)
return bugs
def get_sprint(self, sprint):
bugs = memcache.get(SCRUM_BUG_CACHE_KEY % sprint.id)
if bugs:
return bugs
query = self.request.db_session.query
project_ids = sprint.bugs_project_ids
entries = query(Project, Tracker, TrackerCredentials) \
.filter(Project.id.in_(project_ids)) \
.filter(Project.tracker_id==Tracker.id) \
.filter(TrackerCredentials.tracker_id==Project.tracker_id) \
.filter(TrackerCredentials.user_id==self.user.id).all()
fetchers = []
for project, tracker, creds in entries:
fetcher = get_fetcher(tracker, creds, tracker.logins_mapping)
fetcher.fetch_scrum(sprint.name, project.project_selector, project.component_selector)
fetchers.append(fetcher)
if tracker.type in ('bugzilla', 'rockzilla', 'igozilla'):
break
start = time()
bugs = []
while fetchers:
for i, fetcher in enumerate(fetchers):
if fetcher.isReady():
fetchers.pop(i)
if fetcher.error:
WARN(u"Could not fetch bugs from tracker %s: %s" % (fetcher.tracker.name, fetcher.error))
flash(u"Could not fetch bugs from tracker %s" % (fetcher.tracker.name, ))
else:
for bug in fetcher:
bugs.append(bug)
break
else:
# no fetcher is yet ready, give them time
if time() - start > MAX_TIMEOUT:
WARN(u"Fetchers for trackers %s timed-out" % (u', '.join(fetcher.tracker.name for fetcher in fetchers)))
for fetcher in fetchers:
pass
flash(u"Getting bugs from tracker %s timed out" % (fetcher.tracker.name, ))
fetchers = []
else:
sleep(0.1)
# all bugs were fetched, time to resolve their projects
projects = [bug.project_id for bug in bugs]
projects = dict((project.id, project) for project in Project.query.filter(Project.id.in_(projects)))
# now assign projects to bugs
for bug in bugs:
bug.project = projects.get(bug.project_id)
bugs = self.add_time(bugs, sprint=sprint)
memcache.set(SCRUM_BUG_CACHE_KEY % sprint.id, bugs, SCRUM_BUG_CACHE_TIMEOUT)
return bugs
def get_sprint(self, sprint):
project_ids = sprint.bugs_project_ids
entries = DBSession.query(Project, Tracker, TrackerCredentials, User) \
.filter(Project.id.in_(project_ids)) \
.filter(Project.tracker_id==Tracker.id) \
.filter(TrackerCredentials.tracker_id==Project.tracker_id) \
.filter(TrackerCredentials.user_id==User.id)\
.filter(TrackerCredentials.user_id==self.user.id).all()
fetchers = []
for project, tracker, creds, user in entries:
fetcher = get_fetcher(tracker, creds, user, tracker.logins_mapping)
fetcher.fetch_scrum(sprint.name, project.project_selector,
project.component_selector)
fetchers.append(fetcher)
if tracker.type in ('bugzilla', 'rockzilla', 'igozilla'):
break
start = time()
bugs = []
for fetcher in fetchers:
try:
fbugs = fetcher.get_result()
except FetcherTimeout as e:
flash(
'Fetchers for trackers %s timed-out' %
fetcher.tracker.name,
klass='error',
)
continue
except FetcherBadDataError as e:
flash(e, klass='error')
continue
except FetcherBaseException as e:
flash(
'Could not fetch bugs from tracker %s' %
fetcher.tracker.name,
klass='error',
)
continue
bugs.extend(fbugs)
projects = [bug.project_id for bug in bugs]
projects = dict(
(project.id, project)
for project in Project.query.filter(Project.id.in_(projects)))
# now assign projects to bugs
for bug in bugs:
bug.project = projects.get(bug.project_id)
bugs = self.add_time(bugs, sprint=sprint)
return bugs
def _fetch_bugs_titles(self, tracker_ticket):
query = self.session.query
config_obj = ApplicationConfig.get_current_config()
user = query(User).filter(
User.id == config_obj.hours_ticket_user_id).first()
tracker_creds = query(Tracker.id, Tracker, TrackerCredentials)\
.filter(TrackerCredentials.tracker_id==Tracker.id)\
.filter(TrackerCredentials.user_id==User.id)\
.filter(User.id==config_obj.hours_ticket_user_id)
tracker_creds = groupby(tracker_creds, lambda x: x[0], lambda x: x[1:])
fetchers = {}
for tracker_id, ticket_ids in tracker_ticket.iteritems():
ticket_ids = [ticket_id for ticket_id in ticket_ids if ticket_id]
if not tracker_id in tracker_creds:
continue
tracker, creds = tracker_creds[tracker_id][0]
mapping = {creds.login.lower(): user}
for i, portion in enumerate(
partition(ticket_ids, MAX_TICKETS_PER_REQUEST)):
fetcher = get_fetcher(tracker, creds, mapping)
fetcher.fetch_bug_titles_and_depends_on(portion)
fetchers[(tracker.name, i)] = fetcher
ticket_titles = {}
start = time()
while len(fetchers):
for key, fetcher in fetchers.iteritems():
if fetcher.isReady():
if fetcher.error:
ERROR(u"Fetcher for tracker %s failed with %r" %
(key[0], fetcher.error))
del fetchers[key]
break # continue ignoring exception - titles and depends_on will be empty
#raise fetcher.error
for bug_id, bug in fetcher.bugs.iteritems():
if type(bug['title']) == str:
ticket_titles['%s_%s' %
(fetcher.tracker.id, bug_id
)] = bug['title'].decode('utf-8')
else:
ticket_titles['%s_%s' % (fetcher.tracker.id,
bug_id)] = bug['title']
del fetchers[key]
break
else: # no fetcher is ready yet
if time() - start > MAX_TIMEOUT:
ERROR(u'%s requests timed-out' % len(fetchers))
break # continue without exception - titles and depends_on will be empty
# raise FetchException(u'%s requests timed-out for tracker %s' % (len(fetchers), tracker.name))
else:
sleep(0.1)
return ticket_titles
def _get_bugs(self, fetcher_callback, full_mapping=True):
start = time()
fetchers = []
creds_q = DBSession.query(Tracker, TrackerCredentials)\
.filter(Tracker.id==TrackerCredentials.tracker_id)\
.filter(TrackerCredentials.user_id==self.user.id)
for tracker, credentials in creds_q:
if full_mapping:
mapping = TrackerCredentials.get_logins_mapping(tracker)
else:
mapping = {credentials.login.lower(): self.user}
fetcher = get_fetcher(tracker, credentials, mapping)
fetchers.append(fetcher)
fetcher_callback(fetcher) # initialize query
bugs = []
while fetchers:
for i, fetcher in enumerate(fetchers):
if fetcher.isReady():
fetchers.pop(i)
if fetcher.error:
WARN(u"Could not fetch bugs from tracker %s: %s" %
(fetcher.tracker.name, fetcher.error))
flash(u"Could not fetch bugs from tracker %s" %
(fetcher.tracker.name, ))
else:
for bug in fetcher:
bugs.append(bug)
break
else:
# no fetcher is yet ready, give them time
if time() - start > MAX_TIMEOUT:
WARN(u"Fetchers for trackers %s timed-out" %
(u', '.join(fetcher.tracker.name
for fetcher in fetchers)))
for fetcher in fetchers:
pass
flash(u"Getting bugs from tracker %s timed out" %
(fetcher.tracker.name, ))
fetchers = []
else:
sleep(0.1)
# all bugs were fetched, time to resolve their projects
projects = {}
for bug in bugs:
projects[bug.project_id] = None
# query for all project id's
projects = dict((project.id, project)
for project in Project.query.filter(
Project.id.in_(projects.keys())))
# now assign projects to bugs
for bug in bugs:
bug.project = projects.get(bug.project_id)
return bugs
def get_sprint(self, sprint):
project_ids = sprint.bugs_project_ids
entries = DBSession.query(Project, Tracker, TrackerCredentials, User) \
.filter(Project.id.in_(project_ids)) \
.filter(Project.tracker_id==Tracker.id) \
.filter(TrackerCredentials.tracker_id==Project.tracker_id) \
.filter(TrackerCredentials.user_id==User.id)\
.filter(TrackerCredentials.user_id==self.user.id).all()
fetchers = []
for project, tracker, creds, user in entries:
fetcher = get_fetcher(tracker, creds, user, tracker.logins_mapping)
fetcher.fetch_scrum(sprint.name, project.project_selector, project.component_selector)
fetchers.append(fetcher)
if tracker.type in ('bugzilla', 'rockzilla', 'igozilla'):
break
start = time()
bugs = []
for fetcher in fetchers:
try:
fbugs = fetcher.get_result()
except FetcherTimeout as e:
flash(
'Fetchers for trackers %s timed-out' % fetcher.tracker.name,
klass='error',
)
continue
except FetcherBadDataError as e:
flash(e, klass='error')
continue
except FetcherBaseException as e:
flash(
'Could not fetch bugs from tracker %s' % fetcher.tracker.name,
klass='error',
)
continue
bugs.extend(fbugs)
projects = [bug.project_id for bug in bugs]
projects = dict((project.id, project) for project in Project.query.filter(Project.id.in_(projects)))
# now assign projects to bugs
for bug in bugs:
bug.project = projects.get(bug.project_id)
bugs = self.add_time(bugs, sprint=sprint)
return bugs
def _fetch_bugs_titles(self, tracker_ticket):
query = self.session.query
config_obj = ApplicationConfig.get_current_config()
user = query(User).filter(User.id == config_obj.hours_ticket_user_id).first()
tracker_creds = (
query(Tracker.id, Tracker, TrackerCredentials)
.filter(TrackerCredentials.tracker_id == Tracker.id)
.filter(TrackerCredentials.user_id == User.id)
.filter(User.id == config_obj.hours_ticket_user_id)
)
tracker_creds = groupby(tracker_creds, lambda x: x[0], lambda x: x[1:])
fetchers = {}
for tracker_id, ticket_ids in tracker_ticket.iteritems():
ticket_ids = [ticket_id for ticket_id in ticket_ids if ticket_id]
if not tracker_id in tracker_creds:
continue
tracker, creds = tracker_creds[tracker_id][0]
mapping = {creds.login.lower(): user}
for i, portion in enumerate(partition(ticket_ids, MAX_TICKETS_PER_REQUEST)):
fetcher = get_fetcher(tracker, creds, mapping)
fetcher.fetch_bug_titles_and_depends_on(portion)
fetchers[(tracker.name, i)] = fetcher
ticket_titles = {}
start = time()
while len(fetchers):
for key, fetcher in fetchers.iteritems():
if fetcher.isReady():
if fetcher.error:
ERROR(u"Fetcher for tracker %s failed with %r" % (key[0], fetcher.error))
del fetchers[key]
break # continue ignoring exception - titles and depends_on will be empty
# raise fetcher.error
for bug_id, bug in fetcher.bugs.iteritems():
if type(bug["title"]) == str:
ticket_titles["%s_%s" % (fetcher.tracker.id, bug_id)] = bug["title"].decode("utf-8")
else:
ticket_titles["%s_%s" % (fetcher.tracker.id, bug_id)] = bug["title"]
del fetchers[key]
break
else: # no fetcher is ready yet
if time() - start > MAX_TIMEOUT:
ERROR(u"%s requests timed-out" % len(fetchers))
break # continue without exception - titles and depends_on will be empty
# raise FetchException(u'%s requests timed-out for tracker %s' % (len(fetchers), tracker.name))
else:
sleep(0.1)
return ticket_titles
def _get_bugs(self, fetcher_callback, full_mapping=True):
fetchers = []
creds_q = DBSession.query(Tracker, TrackerCredentials, User)\
.filter(Tracker.id==TrackerCredentials.tracker_id) \
.filter(User.id==TrackerCredentials.user_id)\
.filter(TrackerCredentials.user_id==self.user.id).all()
for tracker, credentials, user in creds_q:
if full_mapping:
mapping = TrackerCredentials.get_logins_mapping(tracker)
else:
mapping = {credentials.login.lower(): self.user}
fetcher = get_fetcher(tracker, credentials, user, mapping)
fetchers.append(fetcher)
fetcher_callback(fetcher) # initialize query
bugs = []
for fetcher in fetchers:
try:
fbugs = fetcher.get_result()
bugs.extend(fbugs)
except FetcherTimeout as e:
flash(
'Fetchers for trackers %s timed-out' %
fetcher.tracker.name,
klass='error',
)
except FetcherBadDataError as e:
flash(e, klass='error')
except FetcherBaseException as e:
flash(
'Could not fetch bugs from tracker %s' %
fetcher.tracker.name,
klass='error',
)
projects = {}
for bug in bugs:
projects[bug.project_id] = None
# query for all project id's
projects = dict((project.id, project)
for project in Project.query.filter(
Project.id.in_(projects.keys())))
# now assign projects to bugs
for bug in bugs:
bug.project = projects.get(bug.project_id)
return bugs
def _get_bugs(self, fetcher_callback, full_mapping=True):
start = time()
fetchers = []
creds_q = DBSession.query(Tracker, TrackerCredentials)\
.filter(Tracker.id==TrackerCredentials.tracker_id)\
.filter(TrackerCredentials.user_id==self.user.id)
for tracker, credentials in creds_q:
if full_mapping:
mapping = TrackerCredentials.get_logins_mapping(tracker)
else:
mapping = {credentials.login.lower(): self.user}
fetcher = get_fetcher(tracker, credentials, mapping)
fetchers.append(fetcher)
fetcher_callback(fetcher) # initialize query
bugs = []
while fetchers:
for i, fetcher in enumerate(fetchers):
if fetcher.isReady():
fetchers.pop(i)
if fetcher.error:
WARN(u"Could not fetch bugs from tracker %s: %s" % (fetcher.tracker.name, fetcher.error))
flash(u"Could not fetch bugs from tracker %s" % (fetcher.tracker.name, ))
else:
for bug in fetcher:
bugs.append(bug)
break
else:
# no fetcher is yet ready, give them time
if time() - start > MAX_TIMEOUT:
WARN(u"Fetchers for trackers %s timed-out" % (u', '.join(fetcher.tracker.name for fetcher in fetchers)))
for fetcher in fetchers:
pass
flash(u"Getting bugs from tracker %s timed out" % (fetcher.tracker.name, ))
fetchers = []
else:
sleep(0.1)
# all bugs were fetched, time to resolve their projects
projects = {}
for bug in bugs:
projects[bug.project_id] = None
# query for all project id's
projects = dict((project.id, project) for project in Project.query.filter(Project.id.in_(projects.keys())))
# now assign projects to bugs
for bug in bugs:
bug.project = projects.get(bug.project_id)
return bugs
def get_sprint(self, sprint):
bugs = memcache.get(SCRUM_BUG_CACHE_KEY % sprint.id)
if bugs:
return bugs
query = self.request.db_session.query
tracker, creds = query(Tracker, TrackerCredentials)\
.filter(TrackerCredentials.tracker_id==sprint.project.tracker_id)\
.filter(TrackerCredentials.tracker_id==Tracker.id)\
.filter(TrackerCredentials.user_id==self.user.id).one()
mapping = TrackerCredentials.get_logins_mapping(tracker)
fetcher = get_fetcher(tracker, creds, mapping)
fetcher.fetch_scrum(sprint.name, sprint.project.project_selector)
start = time()
bugs = []
while True:
if fetcher.isReady():
if fetcher.error:
ERROR(u"Fetcher for tracker %s failed with %r" % (tracker.name, fetcher.error))
break
bugs = [ bug for bug in fetcher ]
break
else: # fetcher is not ready yet
if time() - start > MAX_TIMEOUT:
ERROR(u'Request timed-out')
break
else:
sleep(0.1)
projects = {}
for bug in bugs:
projects[bug.project_id] = None
# query for all project id's
projects = dict((project.id, project) for project in Project.query.filter(Project.id.in_(projects.keys())))
# now assign projects to bugs
for bug in bugs:
if bug.project_id:
bug.project = projects.get(bug.project_id)
else:
bug.project_id = sprint.project_id
bug.project = sprint.project
bugs = self.add_time(bugs, sprint=sprint)
memcache.set(SCRUM_BUG_CACHE_KEY % sprint.id, bugs, SCRUM_BUG_CACHE_TIMEOUT)
return bugs
def get_sprint(self, sprint):
bugs = memcache.get(SCRUM_BUG_CACHE_KEY % sprint.id)
if bugs:
return bugs
query = self.request.db_session.query
tracker, creds = query(Tracker, TrackerCredentials)\
.filter(TrackerCredentials.tracker_id==sprint.project.tracker_id)\
.filter(TrackerCredentials.tracker_id==Tracker.id)\
.filter(TrackerCredentials.user_id==self.user.id).one()
mapping = TrackerCredentials.get_logins_mapping(tracker)
fetcher = get_fetcher(tracker, creds, mapping)
fetcher.fetch_scrum(sprint.name, sprint.project.project_selector)
start = time()
bugs = []
while True:
if fetcher.isReady():
if fetcher.error:
ERROR(u"Fetcher for tracker %s failed with %r" %
(tracker.name, fetcher.error))
break
bugs = [bug for bug in fetcher]
break
else: # fetcher is not ready yet
if time() - start > MAX_TIMEOUT:
ERROR(u'Request timed-out')
break
else:
sleep(0.1)
projects = {}
for bug in bugs:
projects[bug.project_id] = None
# query for all project id's
projects = dict((project.id, project)
for project in Project.query.filter(
Project.id.in_(projects.keys())))
# now assign projects to bugs
for bug in bugs:
bug.project = projects.get(bug.project_id)
bugs = self.add_time(bugs, sprint=sprint)
memcache.set(SCRUM_BUG_CACHE_KEY % sprint.id, bugs,
SCRUM_BUG_CACHE_TIMEOUT)
return bugs
def _get_bugs(self, fetcher_callback, full_mapping=True):
fetchers = []
creds_q = DBSession.query(Tracker, TrackerCredentials, User)\
.filter(Tracker.id==TrackerCredentials.tracker_id) \
.filter(User.id==TrackerCredentials.user_id)\
.filter(TrackerCredentials.user_id==self.user.id).all()
for tracker, credentials, user in creds_q:
if full_mapping:
mapping = TrackerCredentials.get_logins_mapping(tracker)
else:
mapping = {credentials.login.lower(): self.user}
fetcher = get_fetcher(tracker, credentials, user, mapping)
fetchers.append(fetcher)
fetcher_callback(fetcher) # initialize query
bugs = []
for fetcher in fetchers:
try:
fbugs = fetcher.get_result()
bugs.extend(fbugs)
except FetcherTimeout as e:
flash(
'Fetchers for trackers %s timed-out' % fetcher.tracker.name,
klass='error',
)
except FetcherBadDataError as e:
flash(e, klass='error')
except FetcherBaseException as e:
flash(
'Could not fetch bugs from tracker %s' % fetcher.tracker.name,
klass='error',
)
projects = {}
for bug in bugs:
projects[bug.project_id] = None
# query for all project id's
projects = dict((project.id, project) for project in Project.query.filter(Project.id.in_(projects.keys())))
# now assign projects to bugs
for bug in bugs:
bug.project = projects.get(bug.project_id)
return bugs
def get_project(self, project, resolved=False, credentials=None):
"""
Get bugs for given project
"""
start = time()
bugs = []
tracker = project.tracker
credentials = credentials or self._get_credentials(tracker.id)
if not credentials:
return []
login_mapping = TrackerCredentials.get_logins_mapping(tracker)
fetcher = get_fetcher(tracker, credentials, self.user, login_mapping)
fetcher.fetch_bugs_for_query(*project.get_selector_tuple(),
resolved=resolved)
bugs = []
try:
for bug in fetcher.get_result():
bug.project = project
bugs.append(bug)
except FetcherTimeout as e:
flash(
'Fetchers for trackers %s timed-out' % fetcher.tracker.name,
klass='error',
)
except FetcherBadDataError as e:
flash(e, klass='error')
except FetcherBaseException as e:
flash(
'Could not fetch bugs from tracker %s' % fetcher.tracker.name,
klass='error',
)
bugs = self.add_time(bugs)
return bugs
def get_project(self, project, resolved=False, credentials=None):
"""
Get bugs for given project
"""
start = time()
bugs = []
tracker = project.tracker
credentials = credentials or self._get_credentials(tracker.id)
if not credentials:
return []
login_mapping = TrackerCredentials.get_logins_mapping(tracker)
fetcher = get_fetcher(tracker, credentials, self.user, login_mapping)
fetcher.fetch_bugs_for_query(*project.get_selector_tuple(), resolved=resolved)
bugs = []
try:
for bug in fetcher.get_result():
bug.project = project
bugs.append(bug)
except FetcherTimeout as e:
flash(
'Fetchers for trackers %s timed-out' % fetcher.tracker.name,
klass='error',
)
except FetcherBadDataError as e:
flash(e, klass='error')
except FetcherBaseException as e:
flash(
'Could not fetch bugs from tracker %s' % fetcher.tracker.name,
klass='error',
)
bugs = self.add_time(bugs)
return bugs