def neargroups(self, blocknames):
        """Given a list or set of block names, finds groups of 'near' blocks.  Blocks are assigned the same group
        if they are neighbours, or share a neighbour."""
        blocknames = list(set(blocknames))
        groups = []
        for blk in blocknames:
            groups.append(set([blk]))
        from copy import copy

        done = False
        while not done:
            done = True
            for i, g in enumerate(groups):
                ng = copy(g)
                for blk in g:
                    ng = ng | self.block[blk].neighbour_name
                if i < len(groups) - 1:
                    for g2 in groups[i + 1 :]:
                        ng2 = copy(g2)
                        for blk in g2:
                            ng2 = ng2 | self.block[blk].neighbour_name
                        if ng & ng2:
                            g.update(g2)
                            groups.remove(g2)
                            done = False
                            break
                    if not done:
                        break
        return groups
	def _load(self, src, text_src):
		if isinstance(src, PredictionResult):
			pass
		elif isinstance(src, str):
			result = PredictionResult()
			result.load(src)
		else:
			raise Exception('"result" should be PredictionResult or string.')
	
		if not result.analyzable():
			raise ValueError('The given result is not analyzable.')
	
		# +++ Need to move to another place.			   
		#if self.model._hashcode != result.model_id:
		#	sys.stderr.write('Warning: model ID is different from that in the predicted result. Do you use a different model to analyze?\n')
	
		if text_src is None:
			self.filepath = result.text_src
		else:
			self.filepath = text_src
		self.extra_svm_files = result.extra_svm_files
		predicted_y = result.predicted_y
		self.acc = result.get_accuracy()
		decvals = result.decvals
		true_y = result.true_y
				   
		self.insts, self.true_labels, self.predict_labels = [], set(), set()
		for idx in range(len(true_y)):
			self.insts += [TextInstance(idx, true_y = true_y[idx], predicted_y = predicted_y[idx], decvals = list(decvals[idx]))]
			self.true_labels.add(true_y[idx])
			self.predict_labels.add(predicted_y[idx])
def testStandingsBeforeMatches():
    """
    Test to ensure players are properly represented in standings prior
    to any matches being reported.
    """
    deleteMatches()
    deletePlayers()
    registerPlayer("Melpomene Murray")
    registerPlayer("Randy Schwartz")
    standings = playerStandings()
    if len(standings) < 2:
        raise ValueError("Players should appear in playerStandings even before "
                         "they have played any matches.")
    elif len(standings) > 2:
        raise ValueError("Only registered players should appear in standings.")
    if len(standings[0]) != 4:
        raise ValueError("Each playerStandings row should have four columns.")
    [(id1, name1, wins1, matches1), (id2, name2, wins2, matches2)] = standings
    if matches1 != 0 or matches2 != 0 or wins1 != 0 or wins2 != 0:
        raise ValueError(
            "Newly registered players should have no matches or wins.")
    if set([name1, name2]) != set(["Melpomene Murray", "Randy Schwartz"]):
        raise ValueError("Registered players' names should appear in standings, "
                         "even if they have no matches played.")
    print "6. Newly registered players appear in the standings with no matches."
Example #4
0
 def get_analysis_analysisID_dataStage01ResequencingAnalysis(self,analysis_id_I):
     '''Query rows that are used from the analysis'''
     try:
         data = self.session.query(data_stage01_resequencing_analysis).filter(
                 data_stage01_resequencing_analysis.analysis_id.like(analysis_id_I),
                 data_stage01_resequencing_analysis.used_.is_(True)).all();
         analysis_id_O = []
         experiment_id_O = []
         lineage_name_O = []
         sample_name_O = []
         analysis_type_O = []
         analysis_O = {};
         if data: 
             for d in data:
                 analysis_id_O.append(d.analysis_id);
                 experiment_id_O.append(d.experiment_id);
                 lineage_name_O.append(d.lineage_name);
                 sample_name_O.append(d.sample_name);
                 analysis_type_O.append(d.analysis_type);
             analysis_id_O = list(set(analysis_id_O))
             experiment_id_O = list(set(experiment_id_O))
             lineage_name_O = list(set(lineage_name_O))
             sample_name_O = list(set(sample_name_O))
             analysis_type_O = list(set(analysis_type_O))
             analysis_O={
                     'analysis_id':analysis_id_O,
                     'experiment_id':experiment_id_O,
                     'lineage_name':lineage_name_O,
                     'sample_name':sample_name_O,
                     'analysis_type':analysis_type_O};
             
         return analysis_O;
     except SQLAlchemyError as e:
         print(e);
Example #5
0
 def __init__(self, deviceRef):
     # Check that we've got a valid IOHIDDevice.
     assert(deviceRef)
     assert(cf.CFGetTypeID(deviceRef) == iokit.IOHIDDeviceGetTypeID())
     _device_lookup[deviceRef.value] = self
     self.deviceRef = deviceRef
     # Set attributes from device properties.
     self.transport = self.get_property("Transport")
     self.vendorID = self.get_property("VendorID")
     self.vendorIDSource = self.get_property("VendorIDSource")
     self.productID = self.get_property("ProductID")
     self.versionNumber = self.get_property("VersionNumber")
     self.manufacturer = self.get_property("Manufacturer")
     self.product = self.get_property("Product")
     self.serialNumber = self.get_property("SerialNumber")  # always returns None; apple bug?
     self.locationID = self.get_property("LocationID")
     self.primaryUsage = self.get_property("PrimaryUsage")
     self.primaryUsagePage = self.get_property("PrimaryUsagePage")
     # Populate self.elements with our device elements.
     self.get_elements()        
     # Set up callback functions.
     self.value_observers = set()
     self.removal_observers = set()
     self.register_removal_callback()
     self.register_input_value_callback()
Example #6
0
def times(*combined):
  """Generate a product of N sets of combinations.

  times(combine(a=[1,2]), combine(b=[3,4])) == combine(a=[1,2], b=[3,4])

  Args:
    *combined: N lists of dictionaries that specify combinations.

  Returns:
    a list of dictionaries for each combination.

  Raises:
    ValueError: if some of the inputs have overlapping keys.
  """
  assert combined

  if len(combined) == 1:
    return combined[0]

  first = combined[0]
  rest_combined = times(*combined[1:])

  combined_results = []
  for a in first:
    for b in rest_combined:
      if set(a.keys()).intersection(set(b.keys())):
        raise ValueError("Keys need to not overlap: {} vs {}".format(
            a.keys(), b.keys()))

      combined_results.append(OrderedDict(list(a.items()) + list(b.items())))
  return combined_results
Example #7
0
    def __init__(self, config):
        self.populate_logger()

        self.config = config

        mozinfo.find_and_update_from_json(config.topobjdir)

        # Python 2.6 doesn't allow unicode keys to be used for keyword
        # arguments. This gross hack works around the problem until we
        # rid ourselves of 2.6.
        self.info = {}
        for k, v in mozinfo.info.items():
            if isinstance(k, unicode):
                k = k.encode('ascii')
            self.info[k] = v

        self._libs = OrderedDefaultDict(list)
        self._binaries = OrderedDict()
        self._linkage = []
        self._static_linking_shared = set()

        # Keep track of external paths (third party build systems), starting
        # from what we run a subconfigure in. We'll eliminate some directories
        # as we traverse them with moz.build (e.g. js/src).
        subconfigures = os.path.join(self.config.topobjdir, 'subconfigures')
        paths = []
        if os.path.exists(subconfigures):
            paths = open(subconfigures).read().splitlines()
        self._external_paths = set(mozpath.normsep(d) for d in paths)
        # Add security/nss manually, since it doesn't have a subconfigure.
        self._external_paths.add('security/nss')
def load_data(name, plotdir, print_out=True):
    "Read data and split into train, test data."
    df = read_data(name)
    train, test = train_test_split(df, test_size=0.3)
#   plot_scatter_matrix(train, plotdir)  # takes a while, not that useful 
    yvars = ['risk', 'Y']
    train_y = train[yvars]
    test_y  = test[yvars]
#   train_r = train['risk']    # for five-way multi-class classification
    train = train.drop(['risk', 'Y'], axis=1)
    test  = test.drop(['risk', 'Y'],  axis=1)
    if print_out:
        print("train test types %s %s %s %s" % (type(train), type(test), type(train_y), type(test_y)))
        print("train test shapes %s %s %s %s" % (train.shape, test.shape, train_y.shape, test_y.shape))
        print("train head\n%s" % (train[:3]))
        print("test head\n%s" % (test[:3]))
        print("train_y set %s, test_y set %s" % (set(train_y['Y']), set(test_y['Y'])))
        print("train_y stats\n%s\ntest_y stats\n%s" % (train_y.describe(), test_y.describe()))

#   drop_col = ['b_sugar_up']
#   print('dropping high std/mean columns', drop_col)
#   train = train.drop(drop_col, axis=1)
#   test  = test.drop(drop_col, axis=1)
#   drop_col = ['age','exer_slope']
#   print('dropping low importance columns', drop_col)
#   train = train.drop(drop_col, axis=1)
#   test  = test.drop(drop_col, axis=1)
    return train, test, train_y, test_y
Example #9
0
  def _computeConflicts( self ):
    self.conflicts = []
    self.warnings = []
    nonterminalUsageMap = {N: list() for N in self.nonterminals} # maps nonterminal to rules that use this nonterminal in its production

    for R in self.expandedRules:
      for M in R.production.morphemes:
        if isinstance(M, NonTerminal):
          nonterminalUsageMap[M].append(R)

    for N in self.nonterminals:
      if self._empty in self.first[N] and len(self.first[N].intersection(self.follow[N])):
        self.conflicts.append( FirstFollowConflict( N, self.first[N], self.follow[N] ) )

      if not len(nonterminalUsageMap[N]) and not N.generated:
        self.warnings.append(UnusedNonterminalWarning(N))

      NR = self.getExpandedRules( N )
      if len(NR) == 0:
        self.conflicts.append( UndefinedNonterminalConflict(N) )
      for x in range(len(NR)):
        for y in range(len(NR)):
          if x == y:
            continue

          xR = self._pfirst(NR[x].production)
          yR = self._pfirst(NR[y].production)
          intersection = xR.intersection(yR.difference({self._empty}))
          if intersection != set():
            self.conflicts.append( FirstFirstConflict(NR[x], NR[y], self) )
    for macro in self.macros:
      if isinstance(macro, MorphemeListMacro):
        if self.first[macro.morpheme].intersection(self.follow[macro]) != set():
          self.conflicts.append( ListFirstFollowConflict(macro, self.first[macro.nonterminal], self.follow[macro]) )
    return self.conflicts
    def extract_features(self, tweet_message):

        if len(self.bag_of_words) == 0:
            printf('Bag-of-Words empty!')
            return None

        tweet_words = [word.lower() for word, tag in tweet_message if word not in stopwords and not word.isdigit()]
        tweet_tags = [tag[:2] for word, tag in tweet_message if word not in stopwords and not word.isdigit()]

        feature_set = {}

        # 1st set of features: bag-of-words
        for word in self.bag_of_words:
            feature_set['has_'+word] = (word in tweet_words)

        # 2nd set of features: the tags present in the message
        for tag in ['NN','VG','CD','JJ','CC','RB']:
            feature_set['has_'+tag] = (tag in tweet_tags)

        # 3rd feature: negation is present?
        negators = set(['not', 'none', 'nobody', 'never', 'nothing', 'lack', 't','n\'t','dont', 'no'])
        if len(negators.intersection(set(tweet_words))) > 0:
            feature_set['has_negator'] = True

        return feature_set
Example #11
0
def copy_apps(hemps_path, testcase_path, apps_name_list):
    #--------------  COPIES ALL APP SOURCE FILES RELATED INTO TESTCASE FILE ----------------
    source_app_path = hemps_path+"/applications/"
    testcase_app_path = testcase_path+"/applications/"
    
    create_ifn_exists(testcase_app_path)
        
    #for each app described into testcase file
    for app_name in apps_name_list:

        source_app_dir = source_app_path + app_name
        target_app_dir = testcase_app_path + app_name
        
        generic_copy(source_app_dir, target_app_dir, [".svn"])
        
    apps_in_testcase = []
    
    #List as directories from applications directory
    for tc_app in os.listdir(testcase_app_path):
        if os.path.isdir(testcase_app_path+tc_app):
            apps_in_testcase.append(tc_app)
        
    #Remove the apps already present into testcase 
    to_remove_apps = list ( set(apps_in_testcase) - set(apps_name_list) )
    
    for to_remove_app in to_remove_apps:
        delete_if_exists(testcase_app_path + to_remove_app)
Example #12
0
def dedup_value(body, ctype, action="dedup_value", prop=None):
    '''
    Service that accepts a JSON document and enriches the prop field of that document by:

    a) Removing duplicates
    '''

    if prop:
        try:
            data = json.loads(body)
        except:
            response.code = 500
            response.add_header('content-type', 'text/plain')
            return "Unable to parse body as JSON"

    try:
        data = json.loads(body)
    except:
        response.code = 500
        response.add_header('content-type', 'text/plain')
        return "Unable to parse body as JSON"

    for p in prop.split(","):
        if exists(data, p):
            v = getprop(data, p)
            if isinstance(v, list):
                # Remove whitespace, periods, parens, brackets
                clone = [re.sub("[ \.\(\)\[\]\{\}]", "", s).lower() for s in v]
                # Get index of unique values
                index = list(set([clone.index(s) for s in list(set(clone))]))
            
                setprop(data, p, [v[i] for i in index])

    return json.dumps(data)
def test_parameter_grid():
    """Test basic properties of ParameterGrid."""
    params1 = {"foo": [1, 2, 3]}
    grid1 = ParameterGrid(params1)
    assert_true(isinstance(grid1, Iterable))
    assert_true(isinstance(grid1, Sized))
    assert_equal(len(grid1), 3)

    params2 = {"foo": [4, 2],
               "bar": ["ham", "spam", "eggs"]}
    grid2 = ParameterGrid(params2)
    assert_equal(len(grid2), 6)

    # loop to assert we can iterate over the grid multiple times
    for i in xrange(2):
        # tuple + chain transforms {"a": 1, "b": 2} to ("a", 1, "b", 2)
        points = set(tuple(chain(*(sorted(p.items())))) for p in grid2)
        assert_equal(points,
                     set(("bar", x, "foo", y)
                         for x, y in product(params2["bar"], params2["foo"])))

    # Special case: empty grid (useful to get default estimator settings)
    empty = ParameterGrid({})
    assert_equal(len(empty), 1)
    assert_equal(list(empty), [{}])

    has_empty = ParameterGrid([{'C': [1, 10]}, {}])
    assert_equal(len(has_empty), 3)
    assert_equal(list(has_empty), [{'C': 1}, {'C': 10}, {}])
Example #14
0
    def get_sendable_users(self, project):
        conf_key = self.get_conf_key()

        alert_settings = dict(
            (o.user_id, int(o.value))
            for o in UserOption.objects.filter(
                project=project,
                key='%s:alert' % conf_key,
            )
        )

        disabled = set(u for u, v in alert_settings.iteritems() if v == 0)

        member_set = set(project.member_set.exclude(
            user__in=disabled,
        ).values_list('user', flat=True))

        # determine members default settings
        members_to_check = set(u for u in member_set if u not in alert_settings)
        if members_to_check:
            disabled = set(UserOption.objects.filter(
                key='subscribe_by_default',
                value='0',
                user__in=members_to_check,
            ).values_list('user', flat=True))
            member_set = filter(lambda x: x not in disabled, member_set)

        return member_set
Example #15
0
def plot_overtime(data_file):
    data = performance.load_score_dict(data_file)

    avg_sim = []
    std_sim = []

    # Lets compute the average fraction of matching paths for each case
    for index, time_step in enumerate(data):
        if index == 0:
            continue
        prev_step = data[index - 1] 


        sim_list = []

        
        for pair_index, pair in enumerate(time_step):

            curr_chain = set([x[0] for x in pair])
            print curr_chain
            prev_chain = set([x[0] for x in prev_step[pair_index]])

            if len(curr_chain) == 0 or len(prev_chain) == 0:
                continue

            sim = float(len(curr_chain & prev_chain)) / len(curr_chain)
            
            sim_list.append(sim)

        avg_sim.append(np.mean(sim_list)) 
        std_sim.append(np.std(sim_list))

        print "Next Time Step!"

    plotting.overtime_plot(avg_sim, std_sim)  
Example #16
0
def includes_for_type(idl_type):
    idl_type = idl_type.preprocessed_type

    # Composite types
    if idl_type.native_array_element_type:
        return includes_for_type(idl_type)

    # Simple types
    base_idl_type = idl_type.base_type
    if base_idl_type in INCLUDES_FOR_TYPE:
        return INCLUDES_FOR_TYPE[base_idl_type]
    if idl_type.is_basic_type:
        return set()
    if idl_type.is_typed_array_type:
        # Typed array factory methods are already provided by DartUtilities.h.
        return set([])
    if base_idl_type.endswith('ConstructorConstructor'):
        # FIXME: rename to NamedConstructor
        # FIXME: replace with a [NamedConstructorAttribute] extended attribute
        # Ending with 'ConstructorConstructor' indicates a named constructor,
        # and these do not have header files, as they are part of the generated
        # bindings for the interface
        return set()
    if base_idl_type.endswith('Constructor'):
        # FIXME: replace with a [ConstructorAttribute] extended attribute
        base_idl_type = idl_type.constructor_type_name
    if base_idl_type not in component_dir:
        return set()
    return set(['gen/sky/bindings/Dart%s.h' % base_idl_type])
Example #17
0
  def decorated(self, **kwargs):
    """A wrapped test method that treats some arguments in a special way."""
    mode = kwargs.pop("mode", "graph")

    distribution = kwargs.get("distribution", None)
    required_tpu = kwargs.pop("required_tpu", False)
    required_gpus = kwargs.pop("required_gpus", None)

    if distribution:
      assert required_gpus is None, (
          "Do not use `required_gpus` and `distribution` together.")
      assert required_tpu is False, (
          "Do not use `required_tpu` and `distribution` together.")
      required_gpus = distribution.required_gpus
      required_tpu = distribution.required_tpu

    if required_tpu and not TPU_TEST:
      self.skipTest("Test requires a TPU, but it's not available.")
    if not required_tpu and TPU_TEST:
      self.skipTest("Test that doesn't require a TPU.")

    if not required_gpus:
      if GPU_TEST:
        self.skipTest("Test that doesn't require GPUs.")
    elif context.num_gpus() < required_gpus:
      self.skipTest(
          "{} GPUs are not available for this test. {} GPUs are available".
          format(required_gpus, context.num_gpus()))

    # At this point, `kwargs` doesn't have `required_gpus` or `required_tpu`
    # that the user might have specified.  `kwargs` still has `mode`, which
    # the test is allowed to accept or ignore.
    requested_arguments = tf_inspect.getfullargspec(test_method).args
    missing_arguments = set(list(kwargs.keys()) + ["self"]).difference(
        set(requested_arguments + ["mode"]))
    if missing_arguments:
      raise ValueError("The test is missing arguments {} .".format(
          missing_arguments))

    kwargs_to_pass = {}
    for arg in requested_arguments:
      if arg == "self":
        kwargs_to_pass[arg] = self
      else:
        kwargs_to_pass[arg] = kwargs[arg]

    if mode == "eager":
      with ops.Graph().as_default(), context.eager_mode():
        if distribution:
          kwargs_to_pass["distribution"] = distribution.strategy
        test_method(**kwargs_to_pass)
    elif mode == "graph":
      with ops.Graph().as_default(), context.graph_mode():
        if distribution:
          kwargs_to_pass["distribution"] = distribution.strategy
        test_method(**kwargs_to_pass)
    else:
      raise ValueError(
          "'mode' has to be either 'eager' or 'graph' and not {}".format(
              mode))
    def test00_pg_hba_conf_file(self):
        os.environ[self.GP_COMMAND_FAULT_POINT] = 'gpexpand tar segment template'

        cmd = Command(name='run gpexpand', cmdStr='gpexpand -D %s -i %s' % (self.TEST_DB, self.EXPANSION_INPUT_FILE))
        with self.assertRaisesRegexp(ExecutionError, 'Fault Injection'):
            cmd.run(validateAfter=True)
        
        #Read from the pg_hba.conf file and ensure that 
        #The address of the new hosts is present.
        cmd = Command(name='get the temp pg_hba.conf file', 
                      cmdStr="ls %s" % os.path.join(os.path.dirname(self.MASTER_DATA_DIRECTORY),
                                                    'gpexpand*',
                                                    'pg_hba.conf'))
        cmd.run(validateAfter=True)
        results = cmd.get_results()
        temp_pg_hba_conf = results.stdout.strip() 

        actual_values = set()
        expected_values = set([self.primary_host_address, self.mirror_host_address])
        with open(temp_pg_hba_conf) as f:
            for line in f:
                if line.strip() == '# %s' % self.primary_host_name or\
                   line.strip() == '# %s' % self.mirror_host_name:
                    address = f.next().strip().split()[3]
                    address = address[:address.rfind('/')]
                    actual_values.add(address)

        self.assertEqual(actual_values, expected_values)

        GpStart(name='start the database in master only mode', masterOnly=True).run(validateAfter=True)
        Command(name='rollback the expansion', cmdStr='gpexpand -r -D %s' % self.TEST_DB).run(validateAfter=True)
        GpStart(name='start the database').run(validateAfter=True)
Example #19
0
def get_all_group_lines(import_groups):
  if not import_groups:
    return []

  def get_group_lines(group):
    def comparator(x, y):
      # These shenanigans are used to properly order imports for inner classes.
      # So we get ordering like:
      # import com.foo.Bar;
      # import com.foo.Bar.Baz;
      # (this is not lexicographical, so normal sort won't suffice)
      x_m = IMPORT_CLASS_RE.match(x)
      y_m = IMPORT_CLASS_RE.match(y)
      if x_m.group('outer') == y_m.group('outer'):
        return cmp(x_m.group('inners'), y_m.group('inners'))
      else:
        return cmp(x, y)
    lines = sorted(import_groups[group], comparator)
    lines.append('')
    return lines

  all_lines = []
  explicit_groups = ['java', 'javax', 'scala', 'com', 'net', 'org']
  for group in explicit_groups:
    if group in import_groups:
      all_lines += get_group_lines(group)

  # Gather remaining groups.
  remaining_groups = sorted(set(import_groups.keys()) - set(explicit_groups))
  for group in remaining_groups:
    all_lines += get_group_lines(group)
  return all_lines
Example #20
0
def _dict_diff(a, b):
    """A one way dictionary diff.

    a: a dictionary
    b: a dictionary

    Returns: True if the dictionaries are different
    """
    # Only things the master has which the slave lacks matter
    if set(a.keys()) - set(b.keys()):
        LOG.debug('metadata diff -- master has extra keys: %(keys)s',
                  {'keys': ' '.join(set(a.keys()) - set(b.keys()))})
        return True

    for key in a:
        if str(a[key]) != str(b[key]):
            LOG.debug('metadata diff -- value differs for key '
                      '%(key)s: master "%(master_value)s" vs '
                      'slave "%(slave_value)s"',
                      {'key': key,
                       'master_value': a[key],
                       'slave_value': b[key]})
            return True

    return False
Example #21
0
def get_adjacency_lists(in_file):
    edges = {}
    verts = {}
    edge_count = 0
    with open(in_file) as f:
        for line in f.readlines():
            vertex = line.split()
            v1 = int(vertex[0])
            for v2_s in vertex[1:]:
                v2 = int(v2_s)
                if v2 > v1:
                    # avoid adding duplicated edges in the loaded graph
                    try:
                        verts[v1].add(edge_count)  # edges in v1
                    except KeyError:
                        verts[v1] = set()
                        verts[v1].add(edge_count)
                    try:
                        verts[v2].add(edge_count)  # edges in v2
                    except KeyError:
                        verts[v2] = set()
                        verts[v2].add(edge_count)

                    edges[edge_count] = [v1, v2]
                    edge_count += 1

    return edges, verts
Example #22
0
def calculateSparseDictCOO(data_set, data_label_hash, jump=1, valid_flag=False):
	row = []
	col = []
	data = []
	row_valid = []
	col_valid = []
	data_valid = []

	doc_ids = set(sorted(map(lambda row:int(row[0]), data_set)))
	base_ids_list = filter(lambda ids: ids % jump == 0, doc_ids)
	train_ids = base_ids_list
	valid_ids = set()
	if valid_flag:
		valid_index = filter(lambda ids: ids % validation_perc == 0, range(len(base_ids_list)))
		valid_ids = [base_ids_list[i] for i in valid_index]
		base_ids = set(base_ids_list)
		train_ids = sorted(base_ids - set(valid_ids))

	labels = map(lambda trid: int(data_label_hash[trid]), train_ids)
	labels_valid = map(lambda vlid: int(data_label_hash[vlid]), valid_ids)
	for i in range(len(data_set)):
		if int(data_set[i][0]) in train_ids:
			row.append(int(data_set[i][0]))
			col.append(int(data_set[i][1])-1)
			data.append(int(data_set[i][2]))
			# labels.append(int(data_label_hash[int(data_set[i][0])]))
		elif int(data_set[i][0]) in valid_ids:
			row_valid.append(int(data_set[i][0]))
			col_valid.append(int(data_set[i][1])-1)
			data_valid.append(int(data_set[i][2]))
			# labels_valid.append(int(data_label_hash[int(data_set[i][0])]))

	train = translate(row), col, data, labels
	valid = translate(row_valid), col_valid, data_valid, labels_valid
	return train, valid
Example #23
0
    def _init_settings(cls, matrix):
        # Проверяем наличие ошибок неправильного заполнения таблицы свойств
        if not cls._check_shape(fact=matrix.shape,
                                req=cls._required_settings_shape):
            raise WrongShapeException(fact=matrix.shape,
                                      req=cls._required_settings_shape,
                                      name="Проверка размерности таблицы с общими настройками",
                                      aud=cls.outer_name)
        if not cls._check_nans(fact=matrix):
            raise NansInMatrixException(name="Проверка наличия отсутствующих значений в общих настройках",
                                        aud=cls.outer_name)
        # Чтобы проверить саму табличку надо проделать несколько махинаций, ведь по умолчанию все в виде матриц
        settings = pd.DataFrame(matrix[1:], columns=matrix[0])
        settings.columns = cls._standard_settings_column_names
        settings.set_index("key", inplace=True)
        # Проверяем все ли настнойки внесены в табличку
        if not cls._check_settings(fact=set(settings.index),
                                   req=cls._required_settings_options):
            raise NotEnoughSettings(fact=set(settings.index),
                                    req=cls._required_settings_options,
                                    name="Проверка вхождения всех необходимых\
переменных по ключу в общих настройках",
                                    aud=cls.outer_name)
        # Проверяем, что это именно то, что мы ожидали получить на входе
        if not cls._check_values_condition(fact=settings["code"].to_dict(),
                                           req=cls._required_settings_values_condition):
            raise ValuesConditionException(fact=settings["code"].to_dict(),
                                           req=cls._required_settings_values_condition,
                                           name="Проверка валидности ввода настроек в таблицу с общими настройками",
                                           aud=cls.outer_name)
        cls.settings = settings["code"].to_dict()
Example #24
0
def ind_complement(v, ind):
    if isinstance(ind, _INDEXTYPES):
        ind = [ind]
    elif type(ind) is slice:
        ind = range(*ind.indices(len(v)))
    l = len(v)
    return sorted(set(range(l)) - set(i if i >= 0 else l+i for i in ind))
Example #25
0
    def __init__(self, data_dir_name):
        '''
        Read meta-data for test data
        and set table to load based on a given order or on schema files in
        input data
        @param data_dir_name: path to directory containing test data and
                              configuration files
        '''
        self.log = logging.getLogger(__name__)
        self.dataDir = data_dir_name

        _topLevelConfigFile = os.path.join(self.dataDir, "description.yaml")
        with io.open(_topLevelConfigFile, 'r') as f:
            self.update(yaml.load(f))

        self.log.debug("Data configuration : %s" % self)

        fromFileTables = self._tableFromSchemaFile()
        # a specific load order on a restricted number of tables
        # ca be specified in yaml
        if not self['tables'].get('load-order'):
            self['tables']['load-order'] = fromFileTables
            self.notLoadedTables = []
        else:
            self.notLoadedTables = list(set(fromFileTables) -
                                        set(self.orderedTables))
        self.log.debug("Tables to load : %s", self.orderedTables)
Example #26
0
 def getHoster(self):     
     # If no accounts are available there will be no hosters available
     if not self.account or not self.account.canUse():
         return []
     
     # Get account data
     (user, data) = self.account.selectAccount()
     
     # Get supported hosters list from premiumize.me using the json API v1 (see https://secure.premiumize.me/?show=api)
     answer = getURL("https://api.premiumize.me/pm-api/v1.php?method=hosterlist&params[login]=%s&params[pass]=%s" % (user, data['password']))
     data = json_loads(answer)
     
     
     # If account is not valid thera are no hosters available
     if data['status'] != 200:
         return []
     
     # Extract hosters from json file 
     hosters = set(data['result']['hosterlist'])
 
             
     # Read config to check if certain hosters should not be handled
     configMode = self.getConfig('hosterListMode')
     if configMode in ("listed", "unlisted"):
         configList = set(self.getConfig('hosterList').strip().lower().replace('|',',').replace(';',',').split(','))
         configList.discard(u'')
         if configMode == "listed":
             hosters &= configList
         else:
             hosters -= configList
     
     return list(hosters)      
Example #27
0
    def _get_router_ids_for_agent(self, context, agent_db, router_ids):
        result_set = set(super(L3_DVRsch_db_mixin,
                            self)._get_router_ids_for_agent(
            context, agent_db, router_ids))
        router_ids = set(router_ids or [])
        if router_ids and result_set == router_ids:
            # no need for extra dvr checks if requested routers are
            # explicitly scheduled to the agent
            return list(result_set)

        # dvr routers are not explicitly scheduled to agents on hosts with
        # dvr serviceable ports, so need special handling
        if (self._get_agent_mode(agent_db) in
            [n_const.L3_AGENT_MODE_DVR,
             n_const.L3_AGENT_MODE_DVR_NO_EXTERNAL,
             n_const.L3_AGENT_MODE_DVR_SNAT]):
            if not router_ids:
                result_set |= set(self._get_dvr_router_ids_for_host(
                    context, agent_db['host']))
            else:
                for router_id in (router_ids - result_set):
                    subnet_ids = self.get_subnet_ids_on_router(
                        context, router_id)
                    if (subnet_ids and
                            self._check_dvr_serviceable_ports_on_host(
                                    context, agent_db['host'],
                                    list(subnet_ids))):
                        result_set.add(router_id)

        return list(result_set)
Example #28
0
    def _get_episode_search_strings(self, ep_obj, add_string=''):

        search_string = {'Episode': []}

        if not ep_obj:
            return []

        if self.show.air_by_date:
            for show_name in set(show_name_helpers.allPossibleShowNames(self.show)):
                ep_string = sanitizeSceneName(show_name) + '.' + \
                            str(ep_obj.airdate).replace('-', '|')
                search_string['Episode'].append(ep_string)
        elif self.show.sports:
            for show_name in set(show_name_helpers.allPossibleShowNames(self.show)):
                ep_string = sanitizeSceneName(show_name) + '.' + \
                            str(ep_obj.airdate).replace('-', '|') + '|' + \
                            ep_obj.airdate.strftime('%b')
                search_string['Episode'].append(ep_string)
        elif self.show.anime:
            for show_name in set(show_name_helpers.allPossibleShowNames(self.show)):
                ep_string = sanitizeSceneName(show_name) + '.' + \
                            "%i" % int(ep_obj.scene_absolute_number)
                search_string['Episode'].append(ep_string)
        else:
            for show_name in set(show_name_helpers.allPossibleShowNames(self.show)):
                ep_string = show_name_helpers.sanitizeSceneName(show_name) + '.' + \
                            sickbeard.config.naming_ep_type[2] % {'seasonnumber': ep_obj.scene_season,
                                                                  'episodenumber': ep_obj.scene_episode} + ' %s' % add_string

                search_string['Episode'].append(re.sub('\s+', '.', ep_string))

        return [search_string]
Example #29
0
	def route(self, minz):
		try:
			self.paths = []
			self.sub_terminal_collision_lines()
			visited = set()
			for index in xrange(1, len(self.terminals)):
				visited |= set([(int(self.terminals[index - 1][2][0]+0.5), int(self.terminals[index - 1][2][1]+0.5), z) for z in xrange(self.pcb.depth)])
				ends = [(int(self.terminals[index][2][0]+0.5), int(self.terminals[index][2][1]+0.5), z) for z in xrange(self.pcb.depth)]
				self.pcb.mark_distances(self.pcb.routing_flood_vectors, self.radius, self.via, self.gap, visited, ends)
				ends = [(self.pcb.get_node(node), node) for node in ends]
				ends.sort()
				_, end = ends[0]
				path = [end]
				while path[-1] not in visited:
					nearer_nodes = self.pcb.all_not_shorting(self.pcb.all_nearer_sorted, \
								(self.pcb.routing_path_vectors, path[-1], end, self.pcb.dfunc), path[-1], self.radius, self.via, self.gap)
					next_node = next(nearer_nodes)
					if minz:
						for node in nearer_nodes:
							if node[2] == path[-1][2]:
								next_node = node
								break
					path.append(next_node)
				visited |= set(path)
				self.paths.append(path)
				self.pcb.unmark_distances()
			self.paths = self.optimise_paths(self.paths)
			self.add_paths_collision_lines()
			self.add_terminal_collision_lines()
			return True
		except StopIteration:
			self.pcb.unmark_distances()
			self.remove()
			return False
def main():
    args = args_parse()
    out1 = open("%s.TE.fa"%str(args.fasta).split(".")[0],"w")
    out2 = open("%s.CON.fa"%str(args.fasta).split(".")[0],"w")
    seqslen = get_len(args.fasta)
    seqs = get_seqs(args.fasta)
    TE = get_TE_ids(args.TE)
    pfam_ids = get_pfam(args.infile)
    TE_ids = []

    for line in open(args.infile,"r"):
        if "#" not in line and len(line.strip().split()) == 15 :
            ids = line.strip().split()[0]
            start = int(line.strip().split()[1])
            end = int(line.strip().split()[2])
            hmm = line.strip().split()[5]

            if hmm.split(".")[0] in TE:
                per = (float(end)-float(start)+float(1))/float(seqslen.get(ids))
                if per >= float(0.3):
                    TE_ids.append(ids)
                    out1.write(">%s\t%f\n%s\n"%(ids,per,seqs.get(ids)))
    set1 = set(TE_ids)
    set2 = set(seqs.keys())
    for retain_ids in set2 - set1:
        try:
            out2.write(">%s\t%s\n%s\n"%(retain_ids,pfam_ids[retain_ids],seqs.get(retain_ids)))
        except KeyError:
            out2.write(">%s\tNon-domains\n%s\n"%(retain_ids,seqs.get(retain_ids)))