def sort(self, elem_links, url):
fex = Faup()
f = Filters()
f.load()
self.r.switchDB(1)
extend = True
domainfilter = True
schemefilter = True
try:
for link in elem_links:
new_url = link
self.r.switchDB(2)
if not self.r.get(new_url) and new_url:
self.r.switchDB(1)
if not self.r.get(new_url):
fex.decode(new_url)
domain = fex.get_host()
if f.isfilteredscheme(fex.get_scheme()):
self.r.switchDB(2)
self.r.put(new_url, new_url)
schemefilter = False
if f.isfiltereddomains(domain):
self.r.switchDB(2)
self.r.put(new_url, new_url)
domainfilter = False
if f.isfilteredextention(fex.get_resource_path()):
extend = False
self.r.switchDB(2)
self.r.put(new_url, new_url)
if extend and domainfilter and schemefilter:
self.r.switchDB(1)
self.r.rpush('crawl', new_url)
self.queue.append(new_url)
except TypeError as e:
print "TypeError"
def sort(self, elem_links, url):
fex = Faup()
f = Filters()
f.load()
self.r.switchDB(1)
extend = True
domainfilter = True
schemefilter = True
try:
for link in elem_links:
new_url = link
self.r.switchDB(2)
if not self.r.get(new_url) and new_url:
self.r.switchDB(1)
if not self.r.get(new_url):
fex.decode(new_url)
domain = fex.get_host()
if f.isfilteredscheme(fex.get_scheme()):
self.r.switchDB(2)
self.r.put(new_url, new_url)
schemefilter = False
if f.isfiltereddomains(domain):
self.r.switchDB(2)
self.r.put(new_url, new_url)
domainfilter = False
if f.isfilteredextention(fex.get_resource_path()):
extend = False
self.r.switchDB(2)
self.r.put(new_url, new_url)
if extend and domainfilter and schemefilter:
self.r.switchDB(1)
self.r.rpush('crawl', new_url)
self.queue.append(new_url)
except TypeError as e:
print "TypeError"
def addReviewFilters(db, creator, user, review, reviewer_directory_ids, reviewer_file_ids, watcher_directory_ids, watcher_file_ids):
cursor = db.cursor()
cursor.execute("INSERT INTO reviewassignmentstransactions (review, assigner) VALUES (%s, %s) RETURNING id", (review.id, creator.id))
transaction_id = cursor.fetchone()[0]
def add(filter_type, directory_ids, file_ids):
for directory_id, file_id in izip(directory_ids, file_ids):
cursor.execute("""SELECT id, type
FROM reviewfilters
WHERE review=%s
AND uid=%s
AND directory=%s
AND file=%s""",
(review.id, user.id, directory_id, file_id))
row = cursor.fetchone()
if row:
old_filter_id, old_filter_type = row
if old_filter_type == filter_type:
continue
else:
cursor.execute("""DELETE FROM reviewfilters
WHERE id=%s""",
(old_filter_id,))
cursor.execute("""INSERT INTO reviewfilterchanges (transaction, uid, directory, file, type, created)
VALUES (%s, %s, %s, %s, %s, false)""",
(transaction_id, user.id, directory_id, file_id, old_filter_type))
cursor.execute("""INSERT INTO reviewfilters (review, uid, directory, file, type, creator)
VALUES (%s, %s, %s, %s, %s, %s)""",
(review.id, user.id, directory_id, file_id, filter_type, creator.id))
cursor.execute("""INSERT INTO reviewfilterchanges (transaction, uid, directory, file, type, created)
VALUES (%s, %s, %s, %s, %s, true)""",
(transaction_id, user.id, directory_id, file_id, filter_type))
add("reviewer", reviewer_directory_ids, repeat(0))
add("reviewer", repeat(0), reviewer_file_ids)
add("watcher", watcher_directory_ids, repeat(0))
add("watcher", repeat(0), watcher_file_ids)
filters = Filters()
filters.load(db, review=review, user=user)
if user not in review.reviewers and user not in review.watchers and user not in review.owners:
cursor.execute("""INSERT INTO reviewusers (review, uid, type)
VALUES (%s, %s, 'manual')""",
(review.id, user.id,))
delete_files = set()
insert_files = set()
if watcher_directory_ids or watcher_file_ids:
# Unassign changes currently assigned to the affected user.
cursor.execute("""SELECT reviewfiles.id, reviewfiles.file
FROM reviewfiles
JOIN reviewuserfiles ON (reviewuserfiles.file=reviewfiles.id)
WHERE reviewfiles.review=%s
AND reviewuserfiles.uid=%s""",
(review.id, user.id))
for review_file_id, file_id in cursor:
if not filters.isReviewer(db, user.id, file_id):
delete_files.add(review_file_id)
if reviewer_directory_ids or reviewer_file_ids:
# Assign changes currently not assigned to the affected user.
cursor.execute("""SELECT reviewfiles.id, reviewfiles.file
FROM reviewfiles
JOIN changesets ON (changesets.id=reviewfiles.changeset)
JOIN commits ON (commits.id=changesets.child)
JOIN gitusers ON (gitusers.id=commits.author_gituser)
LEFT OUTER JOIN usergitemails USING (email)
LEFT OUTER JOIN reviewuserfiles ON (reviewuserfiles.file=reviewfiles.id AND reviewuserfiles.uid=%s)
WHERE reviewfiles.review=%s
AND (usergitemails.uid IS NULL OR usergitemails.uid!=%s)
AND reviewuserfiles.uid IS NULL""",
(user.id, review.id, user.id))
for review_file_id, file_id in cursor:
if filters.isReviewer(db, user.id, file_id):
insert_files.add(review_file_id)
if delete_files:
cursor.executemany("DELETE FROM reviewuserfiles WHERE file=%s AND uid=%s",
izip(delete_files, repeat(user.id)))
cursor.executemany("INSERT INTO reviewassignmentchanges (transaction, file, uid, assigned) VALUES (%s, %s, %s, false)",
izip(repeat(transaction_id), delete_files, repeat(user.id)))
if insert_files:
cursor.executemany("INSERT INTO reviewuserfiles (file, uid) VALUES (%s, %s)",
izip(insert_files, repeat(user.id)))
cursor.executemany("INSERT INTO reviewassignmentchanges (transaction, file, uid, assigned) VALUES (%s, %s, %s, true)",
izip(repeat(transaction_id), insert_files, repeat(user.id)))
return generateMailsForAssignmentsTransaction(db, transaction_id)
def getReviewersAndWatchers(db, repository, commits=None, changesets=None, reviewfilters=None, applyfilters=True, applyparentfilters=False, parentfiltersonly=False):
"""getReviewersAndWatchers(db, commits=None, changesets=None) -> tuple
Returns a tuple containing two dictionaries, each mapping file IDs to
dictionaries mapping user IDs to sets of changeset IDs. The first dictionary
defines the reviwers of each file, the second dictionary defines the watchers of
each file. For any changes in a file for which no reviewer is identified, None
is used as a key in the dictionary instead of a real user ID."""
if changesets is None:
changesets = []
changeset_utils.createChangesets(db, repository, commits)
for commit in commits:
changesets.extend(changeset_utils.createChangeset(db, None, repository, commit, do_highlight=False))
cursor = db.cursor()
filters = Filters()
if applyfilters:
if parentfiltersonly:
filters.load(db, repository=repository.parent, recursive=True)
else:
filters.load(db, repository=repository, recursive=applyparentfilters)
if reviewfilters:
filters.addFilters(db, reviewfilters, sort=True)
reviewers = {}
watchers = {}
for changeset in changesets:
author_user_id = changeset.child.author.getUserId(db) if changeset.child else None
cursor.execute("SELECT DISTINCT file FROM fileversions WHERE changeset=%s", (changeset.id,))
for (file_id,) in cursor:
reviewers_found = False
for user_id, (filter_type, delegate) in filters.listUsers(db, file_id).items():
try: assert isinstance(user_id, int)
except: raise Exception, repr(filters.listUsers(db, file_id))
if filter_type == 'reviewer':
if author_user_id != user_id:
reviewer_user_ids = [user_id]
elif delegate:
reviewer_user_ids = []
for delegate_user_name in delegate.split(","):
delegate_user = dbutils.User.fromName(db, delegate_user_name)
if delegate_user: reviewer_user_ids.append(delegate_user.id)
else: raise Exception, repr((user_id, delegate_user_name, file_id))
else:
reviewer_user_ids = []
for reviewer_user_id in reviewer_user_ids:
reviewers.setdefault(file_id, {}).setdefault(reviewer_user_id, set()).add(changeset.id)
reviewers_found = True
else:
watchers.setdefault(file_id, {}).setdefault(user_id, set()).add(changeset.id)
if not reviewers_found:
reviewers.setdefault(file_id, {}).setdefault(None, set()).add(changeset.id)
return reviewers, watchers
def addReviewFilters(db, creator, user, review, reviewer_directory_ids,
reviewer_file_ids, watcher_directory_ids,
watcher_file_ids):
cursor = db.cursor()
cursor.execute(
"INSERT INTO reviewassignmentstransactions (review, assigner) VALUES (%s, %s) RETURNING id",
(review.id, creator.id))
transaction_id = cursor.fetchone()[0]
def add(filter_type, directory_ids, file_ids):
for directory_id, file_id in izip(directory_ids, file_ids):
cursor.execute(
"""SELECT id, type
FROM reviewfilters
WHERE review=%s
AND uid=%s
AND directory=%s
AND file=%s""",
(review.id, user.id, directory_id, file_id))
row = cursor.fetchone()
if row:
old_filter_id, old_filter_type = row
if old_filter_type == filter_type:
continue
else:
cursor.execute(
"""DELETE FROM reviewfilters
WHERE id=%s""", (old_filter_id, ))
cursor.execute(
"""INSERT INTO reviewfilterchanges (transaction, uid, directory, file, type, created)
VALUES (%s, %s, %s, %s, %s, false)""",
(transaction_id, user.id, directory_id, file_id,
old_filter_type))
cursor.execute(
"""INSERT INTO reviewfilters (review, uid, directory, file, type, creator)
VALUES (%s, %s, %s, %s, %s, %s)""",
(review.id, user.id, directory_id, file_id, filter_type,
creator.id))
cursor.execute(
"""INSERT INTO reviewfilterchanges (transaction, uid, directory, file, type, created)
VALUES (%s, %s, %s, %s, %s, true)""",
(transaction_id, user.id, directory_id, file_id, filter_type))
add("reviewer", reviewer_directory_ids, repeat(0))
add("reviewer", repeat(0), reviewer_file_ids)
add("watcher", watcher_directory_ids, repeat(0))
add("watcher", repeat(0), watcher_file_ids)
filters = Filters()
filters.load(db, review=review, user=user)
if user not in review.reviewers and user not in review.watchers and user not in review.owners:
cursor.execute(
"""INSERT INTO reviewusers (review, uid, type)
VALUES (%s, %s, 'manual')""", (
review.id,
user.id,
))
delete_files = set()
insert_files = set()
if watcher_directory_ids or watcher_file_ids:
# Unassign changes currently assigned to the affected user.
cursor.execute(
"""SELECT reviewfiles.id, reviewfiles.file
FROM reviewfiles
JOIN reviewuserfiles ON (reviewuserfiles.file=reviewfiles.id)
WHERE reviewfiles.review=%s
AND reviewuserfiles.uid=%s""",
(review.id, user.id))
for review_file_id, file_id in cursor:
if not filters.isReviewer(db, user.id, file_id):
delete_files.add(review_file_id)
if reviewer_directory_ids or reviewer_file_ids:
# Assign changes currently not assigned to the affected user.
cursor.execute(
"""SELECT reviewfiles.id, reviewfiles.file
FROM reviewfiles
JOIN changesets ON (changesets.id=reviewfiles.changeset)
JOIN commits ON (commits.id=changesets.child)
JOIN gitusers ON (gitusers.id=commits.author_gituser)
LEFT OUTER JOIN usergitemails USING (email)
LEFT OUTER JOIN reviewuserfiles ON (reviewuserfiles.file=reviewfiles.id AND reviewuserfiles.uid=%s)
WHERE reviewfiles.review=%s
AND (usergitemails.uid IS NULL OR usergitemails.uid!=%s)
AND reviewuserfiles.uid IS NULL""",
(user.id, review.id, user.id))
for review_file_id, file_id in cursor:
if filters.isReviewer(db, user.id, file_id):
insert_files.add(review_file_id)
if delete_files:
cursor.executemany(
"DELETE FROM reviewuserfiles WHERE file=%s AND uid=%s",
izip(delete_files, repeat(user.id)))
cursor.executemany(
"INSERT INTO reviewassignmentchanges (transaction, file, uid, assigned) VALUES (%s, %s, %s, false)",
izip(repeat(transaction_id), delete_files, repeat(user.id)))
if insert_files:
cursor.executemany(
"INSERT INTO reviewuserfiles (file, uid) VALUES (%s, %s)",
izip(insert_files, repeat(user.id)))
cursor.executemany(
"INSERT INTO reviewassignmentchanges (transaction, file, uid, assigned) VALUES (%s, %s, %s, true)",
izip(repeat(transaction_id), insert_files, repeat(user.id)))
return generateMailsForAssignmentsTransaction(db, transaction_id)
def getReviewersAndWatchers(db,
repository,
commits=None,
changesets=None,
reviewfilters=None,
applyfilters=True,
applyparentfilters=False,
parentfiltersonly=False):
"""getReviewersAndWatchers(db, commits=None, changesets=None) -> tuple
Returns a tuple containing two dictionaries, each mapping file IDs to
dictionaries mapping user IDs to sets of changeset IDs. The first dictionary
defines the reviwers of each file, the second dictionary defines the watchers of
each file. For any changes in a file for which no reviewer is identified, None
is used as a key in the dictionary instead of a real user ID."""
if changesets is None:
changesets = []
changeset_utils.createChangesets(db, repository, commits)
for commit in commits:
changesets.extend(
changeset_utils.createChangeset(db,
None,
repository,
commit,
do_highlight=False))
cursor = db.cursor()
filters = Filters()
if applyfilters:
if parentfiltersonly:
filters.load(db, repository=repository.parent, recursive=True)
else:
filters.load(db,
repository=repository,
recursive=applyparentfilters)
if reviewfilters:
filters.addFilters(db, reviewfilters, sort=True)
reviewers = {}
watchers = {}
for changeset in changesets:
author_user_id = changeset.child.author.getUserId(
db) if changeset.child else None
cursor.execute(
"SELECT DISTINCT file FROM fileversions WHERE changeset=%s",
(changeset.id, ))
for (file_id, ) in cursor:
reviewers_found = False
for user_id, (filter_type,
delegate) in filters.listUsers(db, file_id).items():
try:
assert isinstance(user_id, int)
except:
raise Exception, repr(filters.listUsers(db, file_id))
if filter_type == 'reviewer':
if author_user_id != user_id:
reviewer_user_ids = [user_id]
elif delegate:
reviewer_user_ids = []
for delegate_user_name in delegate.split(","):
delegate_user = dbutils.User.fromName(
db, delegate_user_name)
if delegate_user:
reviewer_user_ids.append(delegate_user.id)
else:
raise Exception, repr(
(user_id, delegate_user_name, file_id))
else:
reviewer_user_ids = []
for reviewer_user_id in reviewer_user_ids:
reviewers.setdefault(file_id, {}).setdefault(
reviewer_user_id, set()).add(changeset.id)
reviewers_found = True
else:
watchers.setdefault(file_id,
{}).setdefault(user_id,
set()).add(changeset.id)
if not reviewers_found:
reviewers.setdefault(file_id,
{}).setdefault(None,
set()).add(changeset.id)
return reviewers, watchers
