def sort(self, elem_links, url): fex = Faup() f = Filters() f.load() self.r.switchDB(1) extend = True domainfilter = True schemefilter = True try: for link in elem_links: new_url = link self.r.switchDB(2) if not self.r.get(new_url) and new_url: self.r.switchDB(1) if not self.r.get(new_url): fex.decode(new_url) domain = fex.get_host() if f.isfilteredscheme(fex.get_scheme()): self.r.switchDB(2) self.r.put(new_url, new_url) schemefilter = False if f.isfiltereddomains(domain): self.r.switchDB(2) self.r.put(new_url, new_url) domainfilter = False if f.isfilteredextention(fex.get_resource_path()): extend = False self.r.switchDB(2) self.r.put(new_url, new_url) if extend and domainfilter and schemefilter: self.r.switchDB(1) self.r.rpush('crawl', new_url) self.queue.append(new_url) except TypeError as e: print "TypeError"
def addReviewFilters(db, creator, user, review, reviewer_directory_ids, reviewer_file_ids, watcher_directory_ids, watcher_file_ids): cursor = db.cursor() cursor.execute("INSERT INTO reviewassignmentstransactions (review, assigner) VALUES (%s, %s) RETURNING id", (review.id, creator.id)) transaction_id = cursor.fetchone()[0] def add(filter_type, directory_ids, file_ids): for directory_id, file_id in izip(directory_ids, file_ids): cursor.execute("""SELECT id, type FROM reviewfilters WHERE review=%s AND uid=%s AND directory=%s AND file=%s""", (review.id, user.id, directory_id, file_id)) row = cursor.fetchone() if row: old_filter_id, old_filter_type = row if old_filter_type == filter_type: continue else: cursor.execute("""DELETE FROM reviewfilters WHERE id=%s""", (old_filter_id,)) cursor.execute("""INSERT INTO reviewfilterchanges (transaction, uid, directory, file, type, created) VALUES (%s, %s, %s, %s, %s, false)""", (transaction_id, user.id, directory_id, file_id, old_filter_type)) cursor.execute("""INSERT INTO reviewfilters (review, uid, directory, file, type, creator) VALUES (%s, %s, %s, %s, %s, %s)""", (review.id, user.id, directory_id, file_id, filter_type, creator.id)) cursor.execute("""INSERT INTO reviewfilterchanges (transaction, uid, directory, file, type, created) VALUES (%s, %s, %s, %s, %s, true)""", (transaction_id, user.id, directory_id, file_id, filter_type)) add("reviewer", reviewer_directory_ids, repeat(0)) add("reviewer", repeat(0), reviewer_file_ids) add("watcher", watcher_directory_ids, repeat(0)) add("watcher", repeat(0), watcher_file_ids) filters = Filters() filters.load(db, review=review, user=user) if user not in review.reviewers and user not in review.watchers and user not in review.owners: cursor.execute("""INSERT INTO reviewusers (review, uid, type) VALUES (%s, %s, 'manual')""", (review.id, user.id,)) delete_files = set() insert_files = set() if watcher_directory_ids or watcher_file_ids: # Unassign changes currently assigned to the affected user. cursor.execute("""SELECT reviewfiles.id, reviewfiles.file FROM reviewfiles JOIN reviewuserfiles ON (reviewuserfiles.file=reviewfiles.id) WHERE reviewfiles.review=%s AND reviewuserfiles.uid=%s""", (review.id, user.id)) for review_file_id, file_id in cursor: if not filters.isReviewer(db, user.id, file_id): delete_files.add(review_file_id) if reviewer_directory_ids or reviewer_file_ids: # Assign changes currently not assigned to the affected user. cursor.execute("""SELECT reviewfiles.id, reviewfiles.file FROM reviewfiles JOIN changesets ON (changesets.id=reviewfiles.changeset) JOIN commits ON (commits.id=changesets.child) JOIN gitusers ON (gitusers.id=commits.author_gituser) LEFT OUTER JOIN usergitemails USING (email) LEFT OUTER JOIN reviewuserfiles ON (reviewuserfiles.file=reviewfiles.id AND reviewuserfiles.uid=%s) WHERE reviewfiles.review=%s AND (usergitemails.uid IS NULL OR usergitemails.uid!=%s) AND reviewuserfiles.uid IS NULL""", (user.id, review.id, user.id)) for review_file_id, file_id in cursor: if filters.isReviewer(db, user.id, file_id): insert_files.add(review_file_id) if delete_files: cursor.executemany("DELETE FROM reviewuserfiles WHERE file=%s AND uid=%s", izip(delete_files, repeat(user.id))) cursor.executemany("INSERT INTO reviewassignmentchanges (transaction, file, uid, assigned) VALUES (%s, %s, %s, false)", izip(repeat(transaction_id), delete_files, repeat(user.id))) if insert_files: cursor.executemany("INSERT INTO reviewuserfiles (file, uid) VALUES (%s, %s)", izip(insert_files, repeat(user.id))) cursor.executemany("INSERT INTO reviewassignmentchanges (transaction, file, uid, assigned) VALUES (%s, %s, %s, true)", izip(repeat(transaction_id), insert_files, repeat(user.id))) return generateMailsForAssignmentsTransaction(db, transaction_id)
def getReviewersAndWatchers(db, repository, commits=None, changesets=None, reviewfilters=None, applyfilters=True, applyparentfilters=False, parentfiltersonly=False): """getReviewersAndWatchers(db, commits=None, changesets=None) -> tuple Returns a tuple containing two dictionaries, each mapping file IDs to dictionaries mapping user IDs to sets of changeset IDs. The first dictionary defines the reviwers of each file, the second dictionary defines the watchers of each file. For any changes in a file for which no reviewer is identified, None is used as a key in the dictionary instead of a real user ID.""" if changesets is None: changesets = [] changeset_utils.createChangesets(db, repository, commits) for commit in commits: changesets.extend(changeset_utils.createChangeset(db, None, repository, commit, do_highlight=False)) cursor = db.cursor() filters = Filters() if applyfilters: if parentfiltersonly: filters.load(db, repository=repository.parent, recursive=True) else: filters.load(db, repository=repository, recursive=applyparentfilters) if reviewfilters: filters.addFilters(db, reviewfilters, sort=True) reviewers = {} watchers = {} for changeset in changesets: author_user_id = changeset.child.author.getUserId(db) if changeset.child else None cursor.execute("SELECT DISTINCT file FROM fileversions WHERE changeset=%s", (changeset.id,)) for (file_id,) in cursor: reviewers_found = False for user_id, (filter_type, delegate) in filters.listUsers(db, file_id).items(): try: assert isinstance(user_id, int) except: raise Exception, repr(filters.listUsers(db, file_id)) if filter_type == 'reviewer': if author_user_id != user_id: reviewer_user_ids = [user_id] elif delegate: reviewer_user_ids = [] for delegate_user_name in delegate.split(","): delegate_user = dbutils.User.fromName(db, delegate_user_name) if delegate_user: reviewer_user_ids.append(delegate_user.id) else: raise Exception, repr((user_id, delegate_user_name, file_id)) else: reviewer_user_ids = [] for reviewer_user_id in reviewer_user_ids: reviewers.setdefault(file_id, {}).setdefault(reviewer_user_id, set()).add(changeset.id) reviewers_found = True else: watchers.setdefault(file_id, {}).setdefault(user_id, set()).add(changeset.id) if not reviewers_found: reviewers.setdefault(file_id, {}).setdefault(None, set()).add(changeset.id) return reviewers, watchers
def addReviewFilters(db, creator, user, review, reviewer_directory_ids, reviewer_file_ids, watcher_directory_ids, watcher_file_ids): cursor = db.cursor() cursor.execute( "INSERT INTO reviewassignmentstransactions (review, assigner) VALUES (%s, %s) RETURNING id", (review.id, creator.id)) transaction_id = cursor.fetchone()[0] def add(filter_type, directory_ids, file_ids): for directory_id, file_id in izip(directory_ids, file_ids): cursor.execute( """SELECT id, type FROM reviewfilters WHERE review=%s AND uid=%s AND directory=%s AND file=%s""", (review.id, user.id, directory_id, file_id)) row = cursor.fetchone() if row: old_filter_id, old_filter_type = row if old_filter_type == filter_type: continue else: cursor.execute( """DELETE FROM reviewfilters WHERE id=%s""", (old_filter_id, )) cursor.execute( """INSERT INTO reviewfilterchanges (transaction, uid, directory, file, type, created) VALUES (%s, %s, %s, %s, %s, false)""", (transaction_id, user.id, directory_id, file_id, old_filter_type)) cursor.execute( """INSERT INTO reviewfilters (review, uid, directory, file, type, creator) VALUES (%s, %s, %s, %s, %s, %s)""", (review.id, user.id, directory_id, file_id, filter_type, creator.id)) cursor.execute( """INSERT INTO reviewfilterchanges (transaction, uid, directory, file, type, created) VALUES (%s, %s, %s, %s, %s, true)""", (transaction_id, user.id, directory_id, file_id, filter_type)) add("reviewer", reviewer_directory_ids, repeat(0)) add("reviewer", repeat(0), reviewer_file_ids) add("watcher", watcher_directory_ids, repeat(0)) add("watcher", repeat(0), watcher_file_ids) filters = Filters() filters.load(db, review=review, user=user) if user not in review.reviewers and user not in review.watchers and user not in review.owners: cursor.execute( """INSERT INTO reviewusers (review, uid, type) VALUES (%s, %s, 'manual')""", ( review.id, user.id, )) delete_files = set() insert_files = set() if watcher_directory_ids or watcher_file_ids: # Unassign changes currently assigned to the affected user. cursor.execute( """SELECT reviewfiles.id, reviewfiles.file FROM reviewfiles JOIN reviewuserfiles ON (reviewuserfiles.file=reviewfiles.id) WHERE reviewfiles.review=%s AND reviewuserfiles.uid=%s""", (review.id, user.id)) for review_file_id, file_id in cursor: if not filters.isReviewer(db, user.id, file_id): delete_files.add(review_file_id) if reviewer_directory_ids or reviewer_file_ids: # Assign changes currently not assigned to the affected user. cursor.execute( """SELECT reviewfiles.id, reviewfiles.file FROM reviewfiles JOIN changesets ON (changesets.id=reviewfiles.changeset) JOIN commits ON (commits.id=changesets.child) JOIN gitusers ON (gitusers.id=commits.author_gituser) LEFT OUTER JOIN usergitemails USING (email) LEFT OUTER JOIN reviewuserfiles ON (reviewuserfiles.file=reviewfiles.id AND reviewuserfiles.uid=%s) WHERE reviewfiles.review=%s AND (usergitemails.uid IS NULL OR usergitemails.uid!=%s) AND reviewuserfiles.uid IS NULL""", (user.id, review.id, user.id)) for review_file_id, file_id in cursor: if filters.isReviewer(db, user.id, file_id): insert_files.add(review_file_id) if delete_files: cursor.executemany( "DELETE FROM reviewuserfiles WHERE file=%s AND uid=%s", izip(delete_files, repeat(user.id))) cursor.executemany( "INSERT INTO reviewassignmentchanges (transaction, file, uid, assigned) VALUES (%s, %s, %s, false)", izip(repeat(transaction_id), delete_files, repeat(user.id))) if insert_files: cursor.executemany( "INSERT INTO reviewuserfiles (file, uid) VALUES (%s, %s)", izip(insert_files, repeat(user.id))) cursor.executemany( "INSERT INTO reviewassignmentchanges (transaction, file, uid, assigned) VALUES (%s, %s, %s, true)", izip(repeat(transaction_id), insert_files, repeat(user.id))) return generateMailsForAssignmentsTransaction(db, transaction_id)
def getReviewersAndWatchers(db, repository, commits=None, changesets=None, reviewfilters=None, applyfilters=True, applyparentfilters=False, parentfiltersonly=False): """getReviewersAndWatchers(db, commits=None, changesets=None) -> tuple Returns a tuple containing two dictionaries, each mapping file IDs to dictionaries mapping user IDs to sets of changeset IDs. The first dictionary defines the reviwers of each file, the second dictionary defines the watchers of each file. For any changes in a file for which no reviewer is identified, None is used as a key in the dictionary instead of a real user ID.""" if changesets is None: changesets = [] changeset_utils.createChangesets(db, repository, commits) for commit in commits: changesets.extend( changeset_utils.createChangeset(db, None, repository, commit, do_highlight=False)) cursor = db.cursor() filters = Filters() if applyfilters: if parentfiltersonly: filters.load(db, repository=repository.parent, recursive=True) else: filters.load(db, repository=repository, recursive=applyparentfilters) if reviewfilters: filters.addFilters(db, reviewfilters, sort=True) reviewers = {} watchers = {} for changeset in changesets: author_user_id = changeset.child.author.getUserId( db) if changeset.child else None cursor.execute( "SELECT DISTINCT file FROM fileversions WHERE changeset=%s", (changeset.id, )) for (file_id, ) in cursor: reviewers_found = False for user_id, (filter_type, delegate) in filters.listUsers(db, file_id).items(): try: assert isinstance(user_id, int) except: raise Exception, repr(filters.listUsers(db, file_id)) if filter_type == 'reviewer': if author_user_id != user_id: reviewer_user_ids = [user_id] elif delegate: reviewer_user_ids = [] for delegate_user_name in delegate.split(","): delegate_user = dbutils.User.fromName( db, delegate_user_name) if delegate_user: reviewer_user_ids.append(delegate_user.id) else: raise Exception, repr( (user_id, delegate_user_name, file_id)) else: reviewer_user_ids = [] for reviewer_user_id in reviewer_user_ids: reviewers.setdefault(file_id, {}).setdefault( reviewer_user_id, set()).add(changeset.id) reviewers_found = True else: watchers.setdefault(file_id, {}).setdefault(user_id, set()).add(changeset.id) if not reviewers_found: reviewers.setdefault(file_id, {}).setdefault(None, set()).add(changeset.id) return reviewers, watchers