def get_implicit_gpl_checkouts(builder): """Find all the checkouts to which GPL-ness propagates. Returns a tuple, (result, because), where: * 'result' is a set of the checkout labels that are implicitly made "GPL" by propagation, and * 'because' is a dictionary linking each such label to a set of strings explaining the reason for the labels inclusion """ # There are clearly two ways we can do this: # # 1. For each checkout, follow its dependencies until we find something # that is non-system GPL, or we don't (obviously, finding one such # is enough). # # 2. For each non-system GPL checkout, find everything that depends upon # it and mark it as propagated-to # # In either case, it is definitely worth checking to see if there are # *any* non-system GPL checkouts. # # If we do (1) then we may need to traverse the entire dependency tree # for each and every checkout in it (e.g., if there are no non-system # GPL licensed checkouts). # # If we do (2), then we do nothing if there are no non-system GPL # checkouts. For each that there is, we do need to traverse the entire # dependency tree, but we can hope that this is for significantly fewer # cases than in (1). # # Also, it is possible that we may have "blockers" inserted into the tree, # which truncate such a traversal (I'm not 100% sure about this yet). # # Regardless, approach (2) seems the more sensible. all_gpl_checkouts = get_gpl_checkouts(builder) # Localise for our loop get_checkout_license = builder.db.get_checkout_license get_license_not_affected_by = builder.db.get_license_not_affected_by get_nothing_builds_against = builder.db.get_nothing_builds_against ruleset = builder.ruleset DEBUG = False def add_if_not_us_or_gpl(our_co, this_co, result, because, reason): """Add 'this_co' to 'result' if it is not 'our_co' and not GPL itself. In which case, also add 'this_co':'reason' to 'because' Relies on 'our_co' having a wildcarded label tag. """ if our_co.just_match(this_co): # OK, that's just some variant on ourselves if DEBUG: print 'BUT %s is our_co' % this_co return this_license = get_checkout_license(this_co, absent_is_None=True) if this_license and this_license.is_gpl(): if DEBUG: print 'BUT %s is already GPL' % this_co return lbl = this_co.copy_with_tag('*') result.add(lbl) if lbl in because: because[lbl].add(reason) else: because[lbl] = set([reason]) if DEBUG: print 'ADD %s' % lbl result = set() # Checkouts implicitly affected because = {} # checkout -> what it depended on that did so if DEBUG: print print 'Finding implicit GPL checkouts' for co_label in all_gpl_checkouts: if DEBUG: print '.. %s' % co_label license = get_checkout_license(co_label) if not license.propagates(): if DEBUG: print ' has a link-exception of some sort - ignoring it' continue if get_nothing_builds_against(co_label): if DEBUG: print ' nothing builds against this - ignoring it' continue depend_on_this = required_by(ruleset, co_label) for this_label in depend_on_this: # We should have a bunch of package labels (possibly the same # package present with different tags), plus quite likely some # variants on our own checkout label, and sometimes other stuff if DEBUG: print ' %s' % this_label, if this_label.type == LabelType.Package: not_affected_by = get_license_not_affected_by(this_label) if co_label in not_affected_by: if DEBUG: print 'NOT against %s' % co_label continue # OK, what checkouts does that imply? pkg_checkouts = builder.checkouts_for_package(this_label) if DEBUG: print 'EXPANDS to %s' % ( label_list_to_string(pkg_checkouts)) for this_co in pkg_checkouts: if DEBUG: print ' %s' % this_label, not_affected_by = get_license_not_affected_by(this_co) if co_label in not_affected_by: if DEBUG: print 'NOT against %s' % co_label continue # We know that our original 'co_label' has type '/*` add_if_not_us_or_gpl( co_label, this_co, result, because, '%s depends on %s' % (this_label.copy_with_tag('*'), co_label)) elif this_label.type == LabelType.Checkout: # We know that our original 'co_label' has type '/*` add_if_not_us_or_gpl( co_label, this_label, result, because, '%s depends on %s' % (this_label.copy_with_tag('*'), co_label)) else: # Deployments don't build stuff, so we can ignore them if DEBUG: print 'IGNORE' continue return result, because
def get_implicit_gpl_checkouts(builder): """Find all the checkouts to which GPL-ness propagates. Returns a tuple, (result, because), where: * 'result' is a set of the checkout labels that are implicitly made "GPL" by propagation, and * 'because' is a dictionary linking each such label to a set of strings explaining the reason for the labels inclusion """ # There are clearly two ways we can do this: # # 1. For each checkout, follow its dependencies until we find something # that is non-system GPL, or we don't (obviously, finding one such # is enough). # # 2. For each non-system GPL checkout, find everything that depends upon # it and mark it as propagated-to # # In either case, it is definitely worth checking to see if there are # *any* non-system GPL checkouts. # # If we do (1) then we may need to traverse the entire dependency tree # for each and every checkout in it (e.g., if there are no non-system # GPL licensed checkouts). # # If we do (2), then we do nothing if there are no non-system GPL # checkouts. For each that there is, we do need to traverse the entire # dependency tree, but we can hope that this is for significantly fewer # cases than in (1). # # Also, it is possible that we may have "blockers" inserted into the tree, # which truncate such a traversal (I'm not 100% sure about this yet). # # Regardless, approach (2) seems the more sensible. all_gpl_checkouts = get_gpl_checkouts(builder) # Localise for our loop get_checkout_license = builder.db.get_checkout_license get_license_not_affected_by = builder.db.get_license_not_affected_by get_nothing_builds_against = builder.db.get_nothing_builds_against ruleset = builder.ruleset DEBUG = False def add_if_not_us_or_gpl(our_co, this_co, result, because, reason): """Add 'this_co' to 'result' if it is not 'our_co' and not GPL itself. In which case, also add 'this_co':'reason' to 'because' Relies on 'our_co' having a wildcarded label tag. """ if our_co.just_match(this_co): # OK, that's just some variant on ourselves if DEBUG: print 'BUT %s is our_co'%this_co return this_license = get_checkout_license(this_co, absent_is_None=True) if this_license and this_license.is_gpl(): if DEBUG: print 'BUT %s is already GPL'%this_co return lbl = this_co.copy_with_tag('*') result.add(lbl) if lbl in because: because[lbl].add(reason) else: because[lbl] = set([reason]) if DEBUG: print 'ADD %s'%lbl result = set() # Checkouts implicitly affected because = {} # checkout -> what it depended on that did so if DEBUG: print print 'Finding implicit GPL checkouts' for co_label in all_gpl_checkouts: if DEBUG: print '.. %s'%co_label license = get_checkout_license(co_label) if not license.propagates(): if DEBUG: print ' has a link-exception of some sort - ignoring it' continue if get_nothing_builds_against(co_label): if DEBUG: print ' nothing builds against this - ignoring it' continue depend_on_this = required_by(ruleset, co_label) for this_label in depend_on_this: # We should have a bunch of package labels (possibly the same # package present with different tags), plus quite likely some # variants on our own checkout label, and sometimes other stuff if DEBUG: print ' %s'%this_label, if this_label.type == LabelType.Package: not_affected_by = get_license_not_affected_by(this_label) if co_label in not_affected_by: if DEBUG: print 'NOT against %s'%co_label continue # OK, what checkouts does that imply? pkg_checkouts = builder.checkouts_for_package(this_label) if DEBUG: print 'EXPANDS to %s'%(label_list_to_string(pkg_checkouts)) for this_co in pkg_checkouts: if DEBUG: print ' %s'%this_label, not_affected_by = get_license_not_affected_by(this_co) if co_label in not_affected_by: if DEBUG: print 'NOT against %s'%co_label continue # We know that our original 'co_label' has type '/*` add_if_not_us_or_gpl(co_label, this_co, result, because, '%s depends on %s'%(this_label.copy_with_tag('*'), co_label)) elif this_label.type == LabelType.Checkout: # We know that our original 'co_label' has type '/*` add_if_not_us_or_gpl(co_label, this_label, result, because, '%s depends on %s'%(this_label.copy_with_tag('*'), co_label)) else: # Deployments don't build stuff, so we can ignore them if DEBUG: print 'IGNORE' continue return result, because
def label_domain_sort(): """Test sorting labels with domain names in them. """ # Yes, apparently these are all legitimate label names # The domain names are the same as those used in the docstring for # utils.sort_domains() labels = [ Label.from_string('checkout:(a)fred/*'), Label.from_string('checkout:(+(1))fred/*'), Label.from_string('checkout:(-(2))fred/*'), Label.from_string('checkout:(a(b(c2)))fred/*'), Label.from_string('checkout:(a(b(c1)))fred/*'), Label.from_string('checkout:(+(1(+2(+4(+4)))))fred/*'), Label.from_string('checkout:(b(b))fred/*'), Label.from_string('checkout:(b)fred/*'), Label.from_string('checkout:(b(a))fred/*'), Label.from_string('checkout:(a(a))fred/*'), Label.from_string('checkout:(+(1(+2)))fred/*'), Label.from_string('checkout:(+(1(+2(+4))))fred/*'), Label.from_string('checkout:(+(1(+3)))fred/*'), ] sorted_labels = sorted(labels) string_labels = map(str, labels) string_labels.sort() # Our properly sorted labels have changed order from that given assert sorted_labels != labels # It's not the same order as we'd get by sorting the labels as strings assert map(str, sorted_labels) != string_labels # It is this order... assert depend.label_list_to_string(sorted_labels) == ( "checkout:(+(1))fred/*" " checkout:(+(1(+2)))fred/*" " checkout:(+(1(+2(+4))))fred/*" " checkout:(+(1(+2(+4(+4)))))fred/*" " checkout:(+(1(+3)))fred/*" " checkout:(-(2))fred/*" " checkout:(a)fred/*" " checkout:(a(a))fred/*" " checkout:(a(b(c1)))fred/*" " checkout:(a(b(c2)))fred/*" " checkout:(b)fred/*" " checkout:(b(a))fred/*" " checkout:(b(b))fred/*") # A specific test, which we originally got wrong labels = [ Label.from_string('checkout:(sub1)builds/checked_out'), Label.from_string('checkout:(sub1(sub4))builds/checked_out'), Label.from_string('checkout:(sub1(sub5))builds/checked_out'), Label.from_string('checkout:(sub2)builds/checked_out'), Label.from_string('checkout:(sub1(sub4))co0/checked_out'), Label.from_string('checkout:(sub1(sub5))co0/checked_out'), Label.from_string('checkout:(sub2(sub3))builds/checked_out'), Label.from_string('checkout:(sub2(sub3))co0/checked_out'), ] sorted_labels = sorted(labels) string_labels = map(str, labels) string_labels.sort() # Our properly sorted labels have changed order from that given assert sorted_labels != labels # It's not the same order as we'd get by sorting the labels as strings assert map(str, sorted_labels) != string_labels #print 'xxx' #for label in sorted_labels: # print ' ', str(label) #print 'xxx' # It is this order... assert depend.label_list_to_string(sorted_labels) == ( "checkout:(sub1)builds/checked_out" " checkout:(sub1(sub4))builds/checked_out" " checkout:(sub1(sub4))co0/checked_out" " checkout:(sub1(sub5))builds/checked_out" " checkout:(sub1(sub5))co0/checked_out" " checkout:(sub2)builds/checked_out" " checkout:(sub2(sub3))builds/checked_out" " checkout:(sub2(sub3))co0/checked_out") # Another originally erroneous case l1 = Label.from_string("checkout:(subdomain2(subdomain3))main_co/checked_out") l2 = Label.from_string("checkout:first_co/checked_out") print print 'xx', l1 print 'xx', l2 print 'xx l2 < l1', l2 < l1 assert l2 < l1 labels = [ Label.from_string("checkout:main_co/checked_out"), Label.from_string("checkout:(subdomain1)first_co/checked_out"), Label.from_string("checkout:(subdomain1)main_co/checked_out"), Label.from_string("checkout:(subdomain1)second_co/checked_out"), Label.from_string("checkout:(subdomain1(subdomain3))first_co/checked_out"), Label.from_string("checkout:(subdomain1(subdomain3))main_co/checked_out"), Label.from_string("checkout:(subdomain1(subdomain3))second_co/checked_out"), Label.from_string("checkout:(subdomain2)first_co/checked_out"), Label.from_string("checkout:(subdomain2)main_co/checked_out"), Label.from_string("checkout:(subdomain2(subdomain3))main_co/checked_out"), Label.from_string("checkout:first_co/checked_out"), Label.from_string("checkout:second_co/checked_out"), Label.from_string("checkout:(subdomain2)second_co/checked_out"), Label.from_string("checkout:(subdomain2(subdomain3))first_co/checked_out"), Label.from_string("checkout:(subdomain2(subdomain3))second_co/checked_out"), Label.from_string("checkout:(subdomain2(subdomain4))first_co/checked_out"), Label.from_string("checkout:(subdomain2(subdomain4))main_co/checked_out"), Label.from_string("checkout:(subdomain2(subdomain4))second_co/checked_out"), ] sorted_labels = sorted(labels) string_labels = map(str, labels) string_labels.sort() # Our properly sorted labels have changed order from that given assert sorted_labels != labels # It's not the same order as we'd get by sorting the labels as strings assert map(str, sorted_labels) != string_labels # It is this order... assert depend.label_list_to_string(sorted_labels) == ( "checkout:first_co/checked_out" " checkout:main_co/checked_out" " checkout:second_co/checked_out" " checkout:(subdomain1)first_co/checked_out" " checkout:(subdomain1)main_co/checked_out" " checkout:(subdomain1)second_co/checked_out" " checkout:(subdomain1(subdomain3))first_co/checked_out" " checkout:(subdomain1(subdomain3))main_co/checked_out" " checkout:(subdomain1(subdomain3))second_co/checked_out" " checkout:(subdomain2)first_co/checked_out" " checkout:(subdomain2)main_co/checked_out" " checkout:(subdomain2)second_co/checked_out" " checkout:(subdomain2(subdomain3))first_co/checked_out" " checkout:(subdomain2(subdomain3))main_co/checked_out" " checkout:(subdomain2(subdomain3))second_co/checked_out" " checkout:(subdomain2(subdomain4))first_co/checked_out" " checkout:(subdomain2(subdomain4))main_co/checked_out" " checkout:(subdomain2(subdomain4))second_co/checked_out" )
def label_domain_sort(): """Test sorting labels with domain names in them. """ # Yes, apparently these are all legitimate label names # The domain names are the same as those used in the docstring for # utils.sort_domains() labels = [ Label.from_string('checkout:(a)fred/*'), Label.from_string('checkout:(+(1))fred/*'), Label.from_string('checkout:(-(2))fred/*'), Label.from_string('checkout:(a(b(c2)))fred/*'), Label.from_string('checkout:(a(b(c1)))fred/*'), Label.from_string('checkout:(+(1(+2(+4(+4)))))fred/*'), Label.from_string('checkout:(b(b))fred/*'), Label.from_string('checkout:(b)fred/*'), Label.from_string('checkout:(b(a))fred/*'), Label.from_string('checkout:(a(a))fred/*'), Label.from_string('checkout:(+(1(+2)))fred/*'), Label.from_string('checkout:(+(1(+2(+4))))fred/*'), Label.from_string('checkout:(+(1(+3)))fred/*'), ] sorted_labels = sorted(labels) string_labels = map(str, labels) string_labels.sort() # Our properly sorted labels have changed order from that given assert sorted_labels != labels # It's not the same order as we'd get by sorting the labels as strings assert map(str, sorted_labels) != string_labels # It is this order... assert depend.label_list_to_string(sorted_labels) == ( "checkout:(+(1))fred/*" " checkout:(+(1(+2)))fred/*" " checkout:(+(1(+2(+4))))fred/*" " checkout:(+(1(+2(+4(+4)))))fred/*" " checkout:(+(1(+3)))fred/*" " checkout:(-(2))fred/*" " checkout:(a)fred/*" " checkout:(a(a))fred/*" " checkout:(a(b(c1)))fred/*" " checkout:(a(b(c2)))fred/*" " checkout:(b)fred/*" " checkout:(b(a))fred/*" " checkout:(b(b))fred/*") # A specific test, which we originally got wrong labels = [ Label.from_string('checkout:(sub1)builds/checked_out'), Label.from_string('checkout:(sub1(sub4))builds/checked_out'), Label.from_string('checkout:(sub1(sub5))builds/checked_out'), Label.from_string('checkout:(sub2)builds/checked_out'), Label.from_string('checkout:(sub1(sub4))co0/checked_out'), Label.from_string('checkout:(sub1(sub5))co0/checked_out'), Label.from_string('checkout:(sub2(sub3))builds/checked_out'), Label.from_string('checkout:(sub2(sub3))co0/checked_out'), ] sorted_labels = sorted(labels) string_labels = map(str, labels) string_labels.sort() # Our properly sorted labels have changed order from that given assert sorted_labels != labels # It's not the same order as we'd get by sorting the labels as strings assert map(str, sorted_labels) != string_labels #print 'xxx' #for label in sorted_labels: # print ' ', str(label) #print 'xxx' # It is this order... assert depend.label_list_to_string(sorted_labels) == ( "checkout:(sub1)builds/checked_out" " checkout:(sub1(sub4))builds/checked_out" " checkout:(sub1(sub4))co0/checked_out" " checkout:(sub1(sub5))builds/checked_out" " checkout:(sub1(sub5))co0/checked_out" " checkout:(sub2)builds/checked_out" " checkout:(sub2(sub3))builds/checked_out" " checkout:(sub2(sub3))co0/checked_out") # Another originally erroneous case l1 = Label.from_string( "checkout:(subdomain2(subdomain3))main_co/checked_out") l2 = Label.from_string("checkout:first_co/checked_out") print print 'xx', l1 print 'xx', l2 print 'xx l2 < l1', l2 < l1 assert l2 < l1 labels = [ Label.from_string("checkout:main_co/checked_out"), Label.from_string("checkout:(subdomain1)first_co/checked_out"), Label.from_string("checkout:(subdomain1)main_co/checked_out"), Label.from_string("checkout:(subdomain1)second_co/checked_out"), Label.from_string( "checkout:(subdomain1(subdomain3))first_co/checked_out"), Label.from_string( "checkout:(subdomain1(subdomain3))main_co/checked_out"), Label.from_string( "checkout:(subdomain1(subdomain3))second_co/checked_out"), Label.from_string("checkout:(subdomain2)first_co/checked_out"), Label.from_string("checkout:(subdomain2)main_co/checked_out"), Label.from_string( "checkout:(subdomain2(subdomain3))main_co/checked_out"), Label.from_string("checkout:first_co/checked_out"), Label.from_string("checkout:second_co/checked_out"), Label.from_string("checkout:(subdomain2)second_co/checked_out"), Label.from_string( "checkout:(subdomain2(subdomain3))first_co/checked_out"), Label.from_string( "checkout:(subdomain2(subdomain3))second_co/checked_out"), Label.from_string( "checkout:(subdomain2(subdomain4))first_co/checked_out"), Label.from_string( "checkout:(subdomain2(subdomain4))main_co/checked_out"), Label.from_string( "checkout:(subdomain2(subdomain4))second_co/checked_out"), ] sorted_labels = sorted(labels) string_labels = map(str, labels) string_labels.sort() # Our properly sorted labels have changed order from that given assert sorted_labels != labels # It's not the same order as we'd get by sorting the labels as strings assert map(str, sorted_labels) != string_labels # It is this order... assert depend.label_list_to_string(sorted_labels) == ( "checkout:first_co/checked_out" " checkout:main_co/checked_out" " checkout:second_co/checked_out" " checkout:(subdomain1)first_co/checked_out" " checkout:(subdomain1)main_co/checked_out" " checkout:(subdomain1)second_co/checked_out" " checkout:(subdomain1(subdomain3))first_co/checked_out" " checkout:(subdomain1(subdomain3))main_co/checked_out" " checkout:(subdomain1(subdomain3))second_co/checked_out" " checkout:(subdomain2)first_co/checked_out" " checkout:(subdomain2)main_co/checked_out" " checkout:(subdomain2)second_co/checked_out" " checkout:(subdomain2(subdomain3))first_co/checked_out" " checkout:(subdomain2(subdomain3))main_co/checked_out" " checkout:(subdomain2(subdomain3))second_co/checked_out" " checkout:(subdomain2(subdomain4))first_co/checked_out" " checkout:(subdomain2(subdomain4))main_co/checked_out" " checkout:(subdomain2(subdomain4))second_co/checked_out")